{
  "best_metric": 1.188714861869812,
  "best_model_checkpoint": "models/llama3.2-3b-orpo-finegrained/checkpoint-5000",
  "epoch": 0.36407325153820946,
  "eval_steps": 5000,
  "global_step": 5000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.000728146503076419,
      "grad_norm": 14.924340824624208,
      "learning_rate": 8e-07,
      "log_odds_chosen": 1.0834228992462158,
      "log_odds_ratio": -0.545361340045929,
      "logits/chosen": -1.100000023841858,
      "logits/rejected": -1.107031226158142,
      "logps/chosen": -2.315234422683716,
      "logps/rejected": -3.3148436546325684,
      "loss": 2.444,
      "nll_loss": 2.298046827316284,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.23154297471046448,
      "rewards/margins": 0.10004577785730362,
      "rewards/rejected": -0.33173829317092896,
      "step": 10
    },
    {
      "epoch": 0.001456293006152838,
      "grad_norm": 9.501986434443848,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 0.542285144329071,
      "log_odds_ratio": -0.7381836175918579,
      "logits/chosen": -1.164453148841858,
      "logits/rejected": -1.117773413658142,
      "logps/chosen": -2.229687452316284,
      "logps/rejected": -2.7249999046325684,
      "loss": 2.3392,
      "nll_loss": 2.376171827316284,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.22294922173023224,
      "rewards/margins": 0.04965515062212944,
      "rewards/rejected": -0.2725585997104645,
      "step": 20
    },
    {
      "epoch": 0.002184439509229257,
      "grad_norm": 3.957850921214621,
      "learning_rate": 2.4e-06,
      "log_odds_chosen": 0.2750610411167145,
      "log_odds_ratio": -0.762890636920929,
      "logits/chosen": -1.276953101158142,
      "logits/rejected": -1.2453124523162842,
      "logps/chosen": -1.824609398841858,
      "logps/rejected": -2.0972657203674316,
      "loss": 1.995,
      "nll_loss": 1.919921875,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.18251952528953552,
      "rewards/margins": 0.02733764611184597,
      "rewards/rejected": -0.21000976860523224,
      "step": 30
    },
    {
      "epoch": 0.002912586012305676,
      "grad_norm": 3.1498721961214073,
      "learning_rate": 3.2e-06,
      "log_odds_chosen": 0.37443846464157104,
      "log_odds_ratio": -0.695996105670929,
      "logits/chosen": -1.064062476158142,
      "logits/rejected": -1.00390625,
      "logps/chosen": -1.605859398841858,
      "logps/rejected": -1.9660155773162842,
      "loss": 1.8378,
      "nll_loss": 1.766015648841858,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.16059570014476776,
      "rewards/margins": 0.03613433986902237,
      "rewards/rejected": -0.19663086533546448,
      "step": 40
    },
    {
      "epoch": 0.003640732515382095,
      "grad_norm": 2.8961977700437127,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.45396727323532104,
      "log_odds_ratio": -0.658496081829071,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -1.48046875,
      "logps/rejected": -1.8855469226837158,
      "loss": 1.7232,
      "nll_loss": 1.7078125476837158,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.1480712890625,
      "rewards/margins": 0.04048766940832138,
      "rewards/rejected": -0.18852539360523224,
      "step": 50
    },
    {
      "epoch": 0.004368879018458514,
      "grad_norm": 2.604769874255877,
      "learning_rate": 4.8e-06,
      "log_odds_chosen": 0.37847900390625,
      "log_odds_ratio": -0.6636718511581421,
      "logits/chosen": -0.991015613079071,
      "logits/rejected": -0.9378906488418579,
      "logps/chosen": -1.3503906726837158,
      "logps/rejected": -1.666015625,
      "loss": 1.6685,
      "nll_loss": 1.569921851158142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13503417372703552,
      "rewards/margins": 0.03156738355755806,
      "rewards/rejected": -0.16657714545726776,
      "step": 60
    },
    {
      "epoch": 0.005097025521534933,
      "grad_norm": 2.547787098331564,
      "learning_rate": 5.6e-06,
      "log_odds_chosen": 0.501782238483429,
      "log_odds_ratio": -0.605273425579071,
      "logits/chosen": -1.0753905773162842,
      "logits/rejected": -1.015625,
      "logps/chosen": -1.255859375,
      "logps/rejected": -1.694921851158142,
      "loss": 1.6009,
      "nll_loss": 1.5011718273162842,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.12558594346046448,
      "rewards/margins": 0.04382934421300888,
      "rewards/rejected": -0.16943359375,
      "step": 70
    },
    {
      "epoch": 0.005825172024611352,
      "grad_norm": 2.4866096381563847,
      "learning_rate": 6.4e-06,
      "log_odds_chosen": 0.49003905057907104,
      "log_odds_ratio": -0.6048828363418579,
      "logits/chosen": -1.0732421875,
      "logits/rejected": -0.9947265386581421,
      "logps/chosen": -1.1552734375,
      "logps/rejected": -1.5490233898162842,
      "loss": 1.5947,
      "nll_loss": 1.41796875,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.11552734673023224,
      "rewards/margins": 0.039276123046875,
      "rewards/rejected": -0.15485839545726776,
      "step": 80
    },
    {
      "epoch": 0.006553318527687771,
      "grad_norm": 2.5896986661136934,
      "learning_rate": 7.2e-06,
      "log_odds_chosen": 0.41322022676467896,
      "log_odds_ratio": -0.633105456829071,
      "logits/chosen": -1.0517578125,
      "logits/rejected": -1.012304663658142,
      "logps/chosen": -1.205078125,
      "logps/rejected": -1.5320312976837158,
      "loss": 1.5336,
      "nll_loss": 1.4734375476837158,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12058105319738388,
      "rewards/margins": 0.03266448900103569,
      "rewards/rejected": -0.15310057997703552,
      "step": 90
    },
    {
      "epoch": 0.00728146503076419,
      "grad_norm": 2.520791586033232,
      "learning_rate": 8e-06,
      "log_odds_chosen": 0.500213623046875,
      "log_odds_ratio": -0.58251953125,
      "logits/chosen": -1.0818359851837158,
      "logits/rejected": -1.022070288658142,
      "logps/chosen": -1.125,
      "logps/rejected": -1.508398413658142,
      "loss": 1.517,
      "nll_loss": 1.423828125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11247558891773224,
      "rewards/margins": 0.03836822509765625,
      "rewards/rejected": -0.15095214545726776,
      "step": 100
    },
    {
      "epoch": 0.008009611533840609,
      "grad_norm": 1.8027476277737309,
      "learning_rate": 7.627700713964738e-06,
      "log_odds_chosen": 0.45947265625,
      "log_odds_ratio": -0.6207031011581421,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.9388672113418579,
      "logps/chosen": -1.134179711341858,
      "logps/rejected": -1.5144531726837158,
      "loss": 1.4702,
      "nll_loss": 1.419531226158142,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1134033203125,
      "rewards/margins": 0.03805847093462944,
      "rewards/rejected": -0.151611328125,
      "step": 110
    },
    {
      "epoch": 0.008737758036917028,
      "grad_norm": 1.857753363468467,
      "learning_rate": 7.3029674334022146e-06,
      "log_odds_chosen": 0.46162110567092896,
      "log_odds_ratio": -0.612109363079071,
      "logits/chosen": -1.001562476158142,
      "logits/rejected": -0.933398425579071,
      "logps/chosen": -1.054101586341858,
      "logps/rejected": -1.426171898841858,
      "loss": 1.4526,
      "nll_loss": 1.361914038658142,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.10541991889476776,
      "rewards/margins": 0.03725586086511612,
      "rewards/rejected": -0.14265136420726776,
      "step": 120
    },
    {
      "epoch": 0.009465904539993446,
      "grad_norm": 1.617828052395909,
      "learning_rate": 7.016464154456233e-06,
      "log_odds_chosen": 0.529052734375,
      "log_odds_ratio": -0.559521496295929,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.9828125238418579,
      "logps/chosen": -1.009374976158142,
      "logps/rejected": -1.4015624523162842,
      "loss": 1.4068,
      "nll_loss": 1.36328125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.10087890923023224,
      "rewards/margins": 0.03919372707605362,
      "rewards/rejected": -0.14016112685203552,
      "step": 130
    },
    {
      "epoch": 0.010194051043069866,
      "grad_norm": 1.4449249885781161,
      "learning_rate": 6.7612340378281325e-06,
      "log_odds_chosen": 0.6396728754043579,
      "log_odds_ratio": -0.6016601324081421,
      "logits/chosen": -1.080664038658142,
      "logits/rejected": -1.0224609375,
      "logps/chosen": -1.013085961341858,
      "logps/rejected": -1.5509765148162842,
      "loss": 1.3947,
      "nll_loss": 1.263671875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.10129394382238388,
      "rewards/margins": 0.05373077467083931,
      "rewards/rejected": -0.15500488877296448,
      "step": 140
    },
    {
      "epoch": 0.010922197546146284,
      "grad_norm": 1.509210963153022,
      "learning_rate": 6.531972647421809e-06,
      "log_odds_chosen": 0.505322277545929,
      "log_odds_ratio": -0.5859375,
      "logits/chosen": -0.9404296875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -1.0480468273162842,
      "logps/rejected": -1.4265625476837158,
      "loss": 1.3667,
      "nll_loss": 1.3279297351837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10483398288488388,
      "rewards/margins": 0.03777313232421875,
      "rewards/rejected": -0.14262695610523224,
      "step": 150
    },
    {
      "epoch": 0.011650344049222704,
      "grad_norm": 1.7720026658744639,
      "learning_rate": 6.3245553203367575e-06,
      "log_odds_chosen": 0.7771972417831421,
      "log_odds_ratio": -0.520800769329071,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.928906261920929,
      "logps/chosen": -0.968554675579071,
      "logps/rejected": -1.564453125,
      "loss": 1.3618,
      "nll_loss": 1.2130858898162842,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0968017578125,
      "rewards/margins": 0.05971679836511612,
      "rewards/rejected": -0.15654297173023224,
      "step": 160
    },
    {
      "epoch": 0.012378490552299122,
      "grad_norm": 1.6888274544149533,
      "learning_rate": 6.135719910778963e-06,
      "log_odds_chosen": 0.55780029296875,
      "log_odds_ratio": -0.603515625,
      "logits/chosen": -0.874804675579071,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -1.0343749523162842,
      "logps/rejected": -1.4695312976837158,
      "loss": 1.3592,
      "nll_loss": 1.359765648841858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10349120944738388,
      "rewards/margins": 0.04346618801355362,
      "rewards/rejected": -0.14692382514476776,
      "step": 170
    },
    {
      "epoch": 0.013106637055375542,
      "grad_norm": 1.9132061638687,
      "learning_rate": 5.962847939999439e-06,
      "log_odds_chosen": 0.536724865436554,
      "log_odds_ratio": -0.616406261920929,
      "logits/chosen": -0.9156249761581421,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -1.0412108898162842,
      "logps/rejected": -1.448828101158142,
      "loss": 1.33,
      "nll_loss": 1.4132812023162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.10410156100988388,
      "rewards/margins": 0.04075469821691513,
      "rewards/rejected": -0.14492186903953552,
      "step": 180
    },
    {
      "epoch": 0.01383478355845196,
      "grad_norm": 1.6575069796743211,
      "learning_rate": 5.803810000880094e-06,
      "log_odds_chosen": 0.541333019733429,
      "log_odds_ratio": -0.5892578363418579,
      "logits/chosen": -0.914843738079071,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.9683593511581421,
      "logps/rejected": -1.3828125,
      "loss": 1.2763,
      "nll_loss": 1.2560546398162842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09685058891773224,
      "rewards/margins": 0.04147796705365181,
      "rewards/rejected": -0.1383056640625,
      "step": 190
    },
    {
      "epoch": 0.01456293006152838,
      "grad_norm": 1.6203662182799872,
      "learning_rate": 5.65685424949238e-06,
      "log_odds_chosen": 0.550671398639679,
      "log_odds_ratio": -0.6015625,
      "logits/chosen": -0.9144531488418579,
      "logits/rejected": -0.816210925579071,
      "logps/chosen": -0.9501953125,
      "logps/rejected": -1.389257788658142,
      "loss": 1.3354,
      "nll_loss": 1.3134765625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09499511867761612,
      "rewards/margins": 0.04401855543255806,
      "rewards/rejected": -0.13896484673023224,
      "step": 200
    },
    {
      "epoch": 0.0152910765646048,
      "grad_norm": 1.5962074750146626,
      "learning_rate": 5.5205244747388325e-06,
      "log_odds_chosen": 0.5894775390625,
      "log_odds_ratio": -0.5653320550918579,
      "logits/chosen": -0.8978515863418579,
      "logits/rejected": -0.8148437738418579,
      "logps/chosen": -0.931640625,
      "logps/rejected": -1.365625023841858,
      "loss": 1.2597,
      "nll_loss": 1.1833984851837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09321288764476776,
      "rewards/margins": 0.0435333251953125,
      "rewards/rejected": -0.13657227158546448,
      "step": 210
    },
    {
      "epoch": 0.016019223067681217,
      "grad_norm": 1.551764270448319,
      "learning_rate": 5.393598899705936e-06,
      "log_odds_chosen": 0.6410766839981079,
      "log_odds_ratio": -0.551464855670929,
      "logits/chosen": -0.8765624761581421,
      "logits/rejected": -0.8154296875,
      "logps/chosen": -0.9251953363418579,
      "logps/rejected": -1.4226562976837158,
      "loss": 1.2736,
      "nll_loss": 1.267187476158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.09248046576976776,
      "rewards/margins": 0.04987030103802681,
      "rewards/rejected": -0.14248046278953552,
      "step": 220
    },
    {
      "epoch": 0.016747369570757637,
      "grad_norm": 1.7722325738548714,
      "learning_rate": 5.275043787166296e-06,
      "log_odds_chosen": 0.654980480670929,
      "log_odds_ratio": -0.5464843511581421,
      "logits/chosen": -0.9458984136581421,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.904492199420929,
      "logps/rejected": -1.3845703601837158,
      "loss": 1.2292,
      "nll_loss": 1.145117163658142,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09033203125,
      "rewards/margins": 0.0479736328125,
      "rewards/rejected": -0.1383056640625,
      "step": 230
    },
    {
      "epoch": 0.017475516073834057,
      "grad_norm": 1.5661623534113884,
      "learning_rate": 5.163977794943223e-06,
      "log_odds_chosen": 0.707958996295929,
      "log_odds_ratio": -0.5506836175918579,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.8558593988418579,
      "logps/chosen": -0.916210949420929,
      "logps/rejected": -1.4636719226837158,
      "loss": 1.2352,
      "nll_loss": 1.1785156726837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.09162597358226776,
      "rewards/margins": 0.05474090576171875,
      "rewards/rejected": -0.14633789658546448,
      "step": 240
    },
    {
      "epoch": 0.018203662576910473,
      "grad_norm": 1.7274597512775318,
      "learning_rate": 5.059644256269407e-06,
      "log_odds_chosen": 0.6816650629043579,
      "log_odds_ratio": -0.567089855670929,
      "logits/chosen": -0.9320312738418579,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -0.8521484136581421,
      "logps/rejected": -1.3771483898162842,
      "loss": 1.2294,
      "nll_loss": 1.1886718273162842,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08527831733226776,
      "rewards/margins": 0.05253143236041069,
      "rewards/rejected": -0.13771972060203552,
      "step": 250
    },
    {
      "epoch": 0.018931809079986893,
      "grad_norm": 7.311067524981038,
      "learning_rate": 4.961389383568338e-06,
      "log_odds_chosen": 0.4852294921875,
      "log_odds_ratio": -0.618359386920929,
      "logits/chosen": -0.8931640386581421,
      "logits/rejected": -0.8724609613418579,
      "logps/chosen": -0.873046875,
      "logps/rejected": -1.239648461341858,
      "loss": 1.243,
      "nll_loss": 1.2013671398162842,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.08732910454273224,
      "rewards/margins": 0.03660430759191513,
      "rewards/rejected": -0.12399902194738388,
      "step": 260
    },
    {
      "epoch": 0.019659955583063313,
      "grad_norm": 1.5967286666896203,
      "learning_rate": 4.8686449556014755e-06,
      "log_odds_chosen": 0.614990234375,
      "log_odds_ratio": -0.580371081829071,
      "logits/chosen": -0.9281250238418579,
      "logits/rejected": -0.854296863079071,
      "logps/chosen": -0.834765613079071,
      "logps/rejected": -1.306249976158142,
      "loss": 1.2239,
      "nll_loss": 1.158203125,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08347167819738388,
      "rewards/margins": 0.04712829738855362,
      "rewards/rejected": -0.13056640326976776,
      "step": 270
    },
    {
      "epoch": 0.020388102086139732,
      "grad_norm": 1.3620859355944803,
      "learning_rate": 4.780914437337574e-06,
      "log_odds_chosen": 0.7137451171875,
      "log_odds_ratio": -0.53955078125,
      "logits/chosen": -0.854687511920929,
      "logits/rejected": -0.8218749761581421,
      "logps/chosen": -0.792187511920929,
      "logps/rejected": -1.2917969226837158,
      "loss": 1.2133,
      "nll_loss": 1.2140624523162842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0792236328125,
      "rewards/margins": 0.05008697509765625,
      "rewards/rejected": -0.12922362983226776,
      "step": 280
    },
    {
      "epoch": 0.02111624858921615,
      "grad_norm": 1.5653654891183468,
      "learning_rate": 4.697761756117627e-06,
      "log_odds_chosen": 0.6397460699081421,
      "log_odds_ratio": -0.564648449420929,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.921093761920929,
      "logps/chosen": -0.8109375238418579,
      "logps/rejected": -1.2634766101837158,
      "loss": 1.2222,
      "nll_loss": 1.2216796875,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0810546875,
      "rewards/margins": 0.04528503492474556,
      "rewards/rejected": -0.12629394233226776,
      "step": 290
    },
    {
      "epoch": 0.02184439509229257,
      "grad_norm": 1.4517479073848938,
      "learning_rate": 4.618802153517006e-06,
      "log_odds_chosen": 0.4509033262729645,
      "log_odds_ratio": -0.622851550579071,
      "logits/chosen": -0.9271484613418579,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.784375011920929,
      "logps/rejected": -1.097265601158142,
      "loss": 1.2147,
      "nll_loss": 1.162695288658142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07844237983226776,
      "rewards/margins": 0.03138275071978569,
      "rewards/rejected": -0.10979004204273224,
      "step": 300
    },
    {
      "epoch": 0.02257254159536899,
      "grad_norm": 1.5184986157394886,
      "learning_rate": 4.543694673976518e-06,
      "log_odds_chosen": 0.789929211139679,
      "log_odds_ratio": -0.524218738079071,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.900585949420929,
      "logps/chosen": -0.780468761920929,
      "logps/rejected": -1.345703125,
      "loss": 1.191,
      "nll_loss": 1.178320288658142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.078125,
      "rewards/margins": 0.05646514892578125,
      "rewards/rejected": -0.13461914658546448,
      "step": 310
    },
    {
      "epoch": 0.023300688098445408,
      "grad_norm": 1.4159165186889227,
      "learning_rate": 4.472135954999579e-06,
      "log_odds_chosen": 0.585253894329071,
      "log_odds_ratio": -0.6026366949081421,
      "logits/chosen": -0.937304675579071,
      "logits/rejected": -0.87353515625,
      "logps/chosen": -0.825390636920929,
      "logps/rejected": -1.265039086341858,
      "loss": 1.2088,
      "nll_loss": 1.2177734375,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08256836235523224,
      "rewards/margins": 0.043930817395448685,
      "rewards/rejected": -0.12651367485523224,
      "step": 320
    },
    {
      "epoch": 0.024028834601521828,
      "grad_norm": 1.3004630972387219,
      "learning_rate": 4.403855060505443e-06,
      "log_odds_chosen": 0.644482433795929,
      "log_odds_ratio": -0.5538085699081421,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.931835949420929,
      "logps/chosen": -0.7730468511581421,
      "logps/rejected": -1.2265625,
      "loss": 1.2037,
      "nll_loss": 1.1396484375,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.077392578125,
      "rewards/margins": 0.04529418796300888,
      "rewards/rejected": -0.12260742485523224,
      "step": 330
    },
    {
      "epoch": 0.024756981104598244,
      "grad_norm": 1.3149112607847993,
      "learning_rate": 4.338609156373123e-06,
      "log_odds_chosen": 0.652636706829071,
      "log_odds_ratio": -0.5506836175918579,
      "logits/chosen": -0.9886718988418579,
      "logits/rejected": -0.9322265386581421,
      "logps/chosen": -0.769726574420929,
      "logps/rejected": -1.222070336341858,
      "loss": 1.206,
      "nll_loss": 1.1501953601837158,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07703857123851776,
      "rewards/margins": 0.04521484300494194,
      "rewards/rejected": -0.12226562201976776,
      "step": 340
    },
    {
      "epoch": 0.025485127607674664,
      "grad_norm": 1.5387878523139185,
      "learning_rate": 4.27617987059879e-06,
      "log_odds_chosen": 0.6639648675918579,
      "log_odds_ratio": -0.531933605670929,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.940234363079071,
      "logps/chosen": -0.7919921875,
      "logps/rejected": -1.234960913658142,
      "loss": 1.1721,
      "nll_loss": 1.2236328125,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0792236328125,
      "rewards/margins": 0.044281005859375,
      "rewards/rejected": -0.12346191704273224,
      "step": 350
    },
    {
      "epoch": 0.026213274110751084,
      "grad_norm": 1.5632123406053722,
      "learning_rate": 4.216370213557839e-06,
      "log_odds_chosen": 0.5484374761581421,
      "log_odds_ratio": -0.6241210699081421,
      "logits/chosen": -0.9794921875,
      "logits/rejected": -0.9242187738418579,
      "logps/chosen": -0.8414062261581421,
      "logps/rejected": -1.216406226158142,
      "loss": 1.1985,
      "nll_loss": 1.181054711341858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08405761420726776,
      "rewards/margins": 0.037567138671875,
      "rewards/rejected": -0.12165527045726776,
      "step": 360
    },
    {
      "epoch": 0.026941420613827503,
      "grad_norm": 1.3539809998909873,
      "learning_rate": 4.15900195928029e-06,
      "log_odds_chosen": 0.6247314214706421,
      "log_odds_ratio": -0.570996105670929,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.922656238079071,
      "logps/chosen": -0.7626953125,
      "logps/rejected": -1.187890648841858,
      "loss": 1.178,
      "nll_loss": 1.0378906726837158,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07622070610523224,
      "rewards/margins": 0.04259643703699112,
      "rewards/rejected": -0.11882324516773224,
      "step": 370
    },
    {
      "epoch": 0.02766956711690392,
      "grad_norm": 1.4817193256750378,
      "learning_rate": 4.103913408340617e-06,
      "log_odds_chosen": 0.675366222858429,
      "log_odds_ratio": -0.555859386920929,
      "logits/chosen": -1.0001952648162842,
      "logits/rejected": -0.951953113079071,
      "logps/chosen": -0.754687488079071,
      "logps/rejected": -1.2218749523162842,
      "loss": 1.1963,
      "nll_loss": 1.111328125,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07546386867761612,
      "rewards/margins": 0.04680938646197319,
      "rewards/rejected": -0.12226562201976776,
      "step": 380
    },
    {
      "epoch": 0.02839771361998034,
      "grad_norm": 1.5387865278625523,
      "learning_rate": 4.050957468334666e-06,
      "log_odds_chosen": 0.6034179925918579,
      "log_odds_ratio": -0.5555664300918579,
      "logits/chosen": -0.9828125238418579,
      "logits/rejected": -0.8999999761581421,
      "logps/chosen": -0.807421863079071,
      "logps/rejected": -1.2265625,
      "loss": 1.1811,
      "nll_loss": 1.120703101158142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0806884765625,
      "rewards/margins": 0.0419769287109375,
      "rewards/rejected": -0.12272949516773224,
      "step": 390
    },
    {
      "epoch": 0.02912586012305676,
      "grad_norm": 1.4355835455366375,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.48689574003219604,
      "log_odds_ratio": -0.63525390625,
      "logits/chosen": -0.953320324420929,
      "logits/rejected": -0.93359375,
      "logps/chosen": -0.843945324420929,
      "logps/rejected": -1.197265625,
      "loss": 1.2055,
      "nll_loss": 1.080078125,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.08439941704273224,
      "rewards/margins": 0.03527374193072319,
      "rewards/rejected": -0.11967773735523224,
      "step": 400
    },
    {
      "epoch": 0.02985400662613318,
      "grad_norm": 1.5027831622710668,
      "learning_rate": 3.950918386598359e-06,
      "log_odds_chosen": 0.6747802495956421,
      "log_odds_ratio": -0.559374988079071,
      "logits/chosen": -0.9576171636581421,
      "logits/rejected": -0.8882812261581421,
      "logps/chosen": -0.75,
      "logps/rejected": -1.2404296398162842,
      "loss": 1.1956,
      "nll_loss": 1.090234398841858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07502441108226776,
      "rewards/margins": 0.04903869703412056,
      "rewards/rejected": -0.12404785305261612,
      "step": 410
    },
    {
      "epoch": 0.0305821531292096,
      "grad_norm": 1.4280512968140957,
      "learning_rate": 3.903600291794132e-06,
      "log_odds_chosen": 0.668042004108429,
      "log_odds_ratio": -0.5658203363418579,
      "logits/chosen": -0.9326171875,
      "logits/rejected": -0.8863281011581421,
      "logps/chosen": -0.789257824420929,
      "logps/rejected": -1.259179711341858,
      "loss": 1.1769,
      "nll_loss": 1.068750023841858,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07901611179113388,
      "rewards/margins": 0.046973418444395065,
      "rewards/rejected": -0.12587890028953552,
      "step": 420
    },
    {
      "epoch": 0.031310299632286015,
      "grad_norm": 1.4730733041346917,
      "learning_rate": 3.857942577363297e-06,
      "log_odds_chosen": 0.850781261920929,
      "log_odds_ratio": -0.4913085997104645,
      "logits/chosen": -0.9478515386581421,
      "logits/rejected": -0.912304699420929,
      "logps/chosen": -0.7509765625,
      "logps/rejected": -1.326171875,
      "loss": 1.1831,
      "nll_loss": 1.0851562023162842,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.0750732421875,
      "rewards/margins": 0.05753784254193306,
      "rewards/rejected": -0.1326904296875,
      "step": 430
    },
    {
      "epoch": 0.032038446135362435,
      "grad_norm": 1.451709668646297,
      "learning_rate": 3.813850356982369e-06,
      "log_odds_chosen": 0.62457275390625,
      "log_odds_ratio": -0.5673828125,
      "logits/chosen": -0.9271484613418579,
      "logits/rejected": -0.8500000238418579,
      "logps/chosen": -0.721875011920929,
      "logps/rejected": -1.1496093273162842,
      "loss": 1.1566,
      "nll_loss": 1.0578124523162842,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07222900539636612,
      "rewards/margins": 0.04280243068933487,
      "rewards/rejected": -0.11495361477136612,
      "step": 440
    },
    {
      "epoch": 0.032766592638438855,
      "grad_norm": 1.5000831711435392,
      "learning_rate": 3.7712361663282537e-06,
      "log_odds_chosen": 0.7065674066543579,
      "log_odds_ratio": -0.599316418170929,
      "logits/chosen": -0.990234375,
      "logits/rejected": -0.9150390625,
      "logps/chosen": -0.8130859136581421,
      "logps/rejected": -1.350195288658142,
      "loss": 1.1826,
      "nll_loss": 1.15625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.08134765923023224,
      "rewards/margins": 0.05374603345990181,
      "rewards/rejected": -0.13510742783546448,
      "step": 450
    },
    {
      "epoch": 0.033494739141515274,
      "grad_norm": 3.8241943804343226,
      "learning_rate": 3.730019232961255e-06,
      "log_odds_chosen": 0.7632812261581421,
      "log_odds_ratio": -0.558398425579071,
      "logits/chosen": -0.9671875238418579,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.7974609136581421,
      "logps/rejected": -1.317968726158142,
      "loss": 1.1724,
      "nll_loss": 1.1091797351837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07976074516773224,
      "rewards/margins": 0.05216064304113388,
      "rewards/rejected": -0.13190917670726776,
      "step": 460
    },
    {
      "epoch": 0.034222885644591694,
      "grad_norm": 1.4037887232429922,
      "learning_rate": 3.6901248321155403e-06,
      "log_odds_chosen": 0.656494140625,
      "log_odds_ratio": -0.55126953125,
      "logits/chosen": -1.002343773841858,
      "logits/rejected": -0.9361327886581421,
      "logps/chosen": -0.755078136920929,
      "logps/rejected": -1.2111327648162842,
      "loss": 1.1963,
      "nll_loss": 1.075585961341858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0755615234375,
      "rewards/margins": 0.04556732252240181,
      "rewards/rejected": -0.12116698920726776,
      "step": 470
    },
    {
      "epoch": 0.034951032147668114,
      "grad_norm": 14.473120499223526,
      "learning_rate": 3.6514837167011073e-06,
      "log_odds_chosen": 0.670764148235321,
      "log_odds_ratio": -0.55126953125,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.782421886920929,
      "logps/rejected": -1.243554711341858,
      "loss": 1.1542,
      "nll_loss": 1.081445336341858,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0782470703125,
      "rewards/margins": 0.04622802883386612,
      "rewards/rejected": -0.1243896484375,
      "step": 480
    },
    {
      "epoch": 0.03567917865074453,
      "grad_norm": 1.4021631646997694,
      "learning_rate": 3.6140316116210052e-06,
      "log_odds_chosen": 0.6236938238143921,
      "log_odds_ratio": -0.5826171636581421,
      "logits/chosen": -0.927050769329071,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.801953136920929,
      "logps/rejected": -1.217382788658142,
      "loss": 1.1497,
      "nll_loss": 1.1140625476837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0802001953125,
      "rewards/margins": 0.04158172756433487,
      "rewards/rejected": -0.12180175632238388,
      "step": 490
    },
    {
      "epoch": 0.036407325153820946,
      "grad_norm": 1.4504280665517582,
      "learning_rate": 3.5777087639996634e-06,
      "log_odds_chosen": 0.7083984613418579,
      "log_odds_ratio": -0.5472656488418579,
      "logits/chosen": -0.979687511920929,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.7665039300918579,
      "logps/rejected": -1.2509765625,
      "loss": 1.1663,
      "nll_loss": 1.1181640625,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07667236030101776,
      "rewards/margins": 0.04839477688074112,
      "rewards/rejected": -0.12507323920726776,
      "step": 500
    },
    {
      "epoch": 0.037135471656897366,
      "grad_norm": 1.6116253479627094,
      "learning_rate": 3.5424595421603814e-06,
      "log_odds_chosen": 0.6789916753768921,
      "log_odds_ratio": -0.5692383050918579,
      "logits/chosen": -1.037500023841858,
      "logits/rejected": -0.960742175579071,
      "logps/chosen": -0.77734375,
      "logps/rejected": -1.2580077648162842,
      "loss": 1.1452,
      "nll_loss": 1.0730469226837158,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0777587890625,
      "rewards/margins": 0.04798584058880806,
      "rewards/rejected": -0.12578125298023224,
      "step": 510
    },
    {
      "epoch": 0.037863618159973786,
      "grad_norm": 1.539494298635483,
      "learning_rate": 3.5082320772281165e-06,
      "log_odds_chosen": 0.747753918170929,
      "log_odds_ratio": -0.528613269329071,
      "logits/chosen": -1.0373046398162842,
      "logits/rejected": -0.9320312738418579,
      "logps/chosen": -0.7099609375,
      "logps/rejected": -1.2082030773162842,
      "loss": 1.1636,
      "nll_loss": 1.0947265625,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07093505561351776,
      "rewards/margins": 0.04978637769818306,
      "rewards/rejected": -0.12082519382238388,
      "step": 520
    },
    {
      "epoch": 0.038591764663050206,
      "grad_norm": 1.4585989044057364,
      "learning_rate": 3.474977942104555e-06,
      "log_odds_chosen": 0.7982422113418579,
      "log_odds_ratio": -0.4876953065395355,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.9144531488418579,
      "logps/chosen": -0.758984386920929,
      "logps/rejected": -1.274999976158142,
      "loss": 1.1488,
      "nll_loss": 1.159765601158142,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07589111477136612,
      "rewards/margins": 0.05152587965130806,
      "rewards/rejected": -0.12739257514476776,
      "step": 530
    },
    {
      "epoch": 0.039319911166126625,
      "grad_norm": 1.4080621583665085,
      "learning_rate": 3.442651863295481e-06,
      "log_odds_chosen": 0.6578613519668579,
      "log_odds_ratio": -0.56298828125,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.938281238079071,
      "logps/chosen": -0.765625,
      "logps/rejected": -1.209375023841858,
      "loss": 1.1478,
      "nll_loss": 1.089453101158142,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07663574069738388,
      "rewards/margins": 0.04428863525390625,
      "rewards/rejected": -0.12104491889476776,
      "step": 540
    },
    {
      "epoch": 0.040048057669203045,
      "grad_norm": 1.4487680890750678,
      "learning_rate": 3.4112114616897665e-06,
      "log_odds_chosen": 0.7410644292831421,
      "log_odds_ratio": -0.532910168170929,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.7835937738418579,
      "logps/rejected": -1.29296875,
      "loss": 1.1611,
      "nll_loss": 1.1779296398162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07835693657398224,
      "rewards/margins": 0.05080718919634819,
      "rewards/rejected": -0.12912598252296448,
      "step": 550
    },
    {
      "epoch": 0.040776204172279465,
      "grad_norm": 1.4460105953307048,
      "learning_rate": 3.3806170189140663e-06,
      "log_odds_chosen": 0.839404284954071,
      "log_odds_ratio": -0.518310546875,
      "logits/chosen": -0.978320300579071,
      "logits/rejected": -0.9076172113418579,
      "logps/chosen": -0.728222668170929,
      "logps/rejected": -1.279687523841858,
      "loss": 1.1673,
      "nll_loss": 1.0771484375,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07281494140625,
      "rewards/margins": 0.05517425388097763,
      "rewards/rejected": -0.1280517578125,
      "step": 560
    },
    {
      "epoch": 0.041504350675355885,
      "grad_norm": 1.3270439290769895,
      "learning_rate": 3.350831266333564e-06,
      "log_odds_chosen": 0.826708972454071,
      "log_odds_ratio": -0.509960949420929,
      "logits/chosen": -0.9378906488418579,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.781445324420929,
      "logps/rejected": -1.3515625,
      "loss": 1.1297,
      "nll_loss": 1.1443359851837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07811279594898224,
      "rewards/margins": 0.05701599270105362,
      "rewards/rejected": -0.135009765625,
      "step": 570
    },
    {
      "epoch": 0.0422324971784323,
      "grad_norm": 1.4505105770320856,
      "learning_rate": 3.3218191941495984e-06,
      "log_odds_chosen": 0.88323974609375,
      "log_odds_ratio": -0.4817871153354645,
      "logits/chosen": -0.8863281011581421,
      "logits/rejected": -0.814648449420929,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.314062476158142,
      "loss": 1.1539,
      "nll_loss": 1.095703125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07156982272863388,
      "rewards/margins": 0.05988464504480362,
      "rewards/rejected": -0.1314697265625,
      "step": 580
    },
    {
      "epoch": 0.04296064368150872,
      "grad_norm": 1.4158594488139011,
      "learning_rate": 3.293547878370473e-06,
      "log_odds_chosen": 0.7425292730331421,
      "log_odds_ratio": -0.558789074420929,
      "logits/chosen": -1.029296875,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.7777343988418579,
      "logps/rejected": -1.300390601158142,
      "loss": 1.1698,
      "nll_loss": 1.0867187976837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07773437350988388,
      "rewards/margins": 0.05230712890625,
      "rewards/rejected": -0.13017578423023224,
      "step": 590
    },
    {
      "epoch": 0.04368879018458514,
      "grad_norm": 1.2635665666685174,
      "learning_rate": 3.2659863237109044e-06,
      "log_odds_chosen": 0.766796886920929,
      "log_odds_ratio": -0.525195300579071,
      "logits/chosen": -0.926953136920929,
      "logits/rejected": -0.8707031011581421,
      "logps/chosen": -0.747265636920929,
      "logps/rejected": -1.2433593273162842,
      "loss": 1.1475,
      "nll_loss": 1.129296898841858,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07471923530101776,
      "rewards/margins": 0.04962768405675888,
      "rewards/rejected": -0.12424316257238388,
      "step": 600
    },
    {
      "epoch": 0.04441693668766156,
      "grad_norm": 1.365323069745603,
      "learning_rate": 3.239105320715664e-06,
      "log_odds_chosen": 0.6534179449081421,
      "log_odds_ratio": -0.5713866949081421,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.8384765386581421,
      "logps/chosen": -0.758984386920929,
      "logps/rejected": -1.2023437023162842,
      "loss": 1.13,
      "nll_loss": 1.08203125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07584228366613388,
      "rewards/margins": 0.04430847242474556,
      "rewards/rejected": -0.12014160305261612,
      "step": 610
    },
    {
      "epoch": 0.04514508319073798,
      "grad_norm": 1.4386908687736795,
      "learning_rate": 3.2128773156099956e-06,
      "log_odds_chosen": 0.521655261516571,
      "log_odds_ratio": -0.615527331829071,
      "logits/chosen": -0.9248046875,
      "logits/rejected": -0.8724609613418579,
      "logps/chosen": -0.8402343988418579,
      "logps/rejected": -1.213281273841858,
      "loss": 1.1664,
      "nll_loss": 1.1300780773162842,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.08408202975988388,
      "rewards/margins": 0.03731994703412056,
      "rewards/rejected": -0.12141112983226776,
      "step": 620
    },
    {
      "epoch": 0.045873229693814396,
      "grad_norm": 1.4435488382072212,
      "learning_rate": 3.187276291558383e-06,
      "log_odds_chosen": 0.8658691644668579,
      "log_odds_ratio": -0.4942382872104645,
      "logits/chosen": -1.0087890625,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -0.7435547113418579,
      "logps/rejected": -1.3240234851837158,
      "loss": 1.1026,
      "nll_loss": 1.0828125476837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07431640475988388,
      "rewards/margins": 0.05805664137005806,
      "rewards/rejected": -0.13242188096046448,
      "step": 630
    },
    {
      "epoch": 0.046601376196890816,
      "grad_norm": 1.482787682541737,
      "learning_rate": 3.1622776601683788e-06,
      "log_odds_chosen": 0.8796631097793579,
      "log_odds_ratio": -0.49311524629592896,
      "logits/chosen": -0.9281250238418579,
      "logits/rejected": -0.874218761920929,
      "logps/chosen": -0.7124999761581421,
      "logps/rejected": -1.279687523841858,
      "loss": 1.1436,
      "nll_loss": 1.1337890625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07125244289636612,
      "rewards/margins": 0.05678100511431694,
      "rewards/rejected": -0.12810058891773224,
      "step": 640
    },
    {
      "epoch": 0.047329522699967236,
      "grad_norm": 1.5473980483876462,
      "learning_rate": 3.1378581622109444e-06,
      "log_odds_chosen": 0.9545654058456421,
      "log_odds_ratio": -0.45380860567092896,
      "logits/chosen": -1.016992211341858,
      "logits/rejected": -0.9339843988418579,
      "logps/chosen": -0.7142578363418579,
      "logps/rejected": -1.357031226158142,
      "loss": 1.1105,
      "nll_loss": 1.082617163658142,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07135009765625,
      "rewards/margins": 0.06436462700366974,
      "rewards/rejected": -0.13576659560203552,
      "step": 650
    },
    {
      "epoch": 0.048057669203043656,
      "grad_norm": 1.4079702343392544,
      "learning_rate": 3.113995776646092e-06,
      "log_odds_chosen": 0.83203125,
      "log_odds_ratio": -0.50146484375,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.7357422113418579,
      "logps/rejected": -1.3035156726837158,
      "loss": 1.1481,
      "nll_loss": 1.113671898841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07359619438648224,
      "rewards/margins": 0.056671142578125,
      "rewards/rejected": -0.13026122748851776,
      "step": 660
    },
    {
      "epoch": 0.04878581570612007,
      "grad_norm": 1.4702021710737754,
      "learning_rate": 3.090669637145023e-06,
      "log_odds_chosen": 0.72796630859375,
      "log_odds_ratio": -0.530957043170929,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.712890625,
      "logps/rejected": -1.212499976158142,
      "loss": 1.128,
      "nll_loss": 1.1613280773162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07121582329273224,
      "rewards/margins": 0.04995117336511612,
      "rewards/rejected": -0.12127685546875,
      "step": 670
    },
    {
      "epoch": 0.04951396220919649,
      "grad_norm": 1.559008568221807,
      "learning_rate": 3.0678599553894814e-06,
      "log_odds_chosen": 1.0158202648162842,
      "log_odds_ratio": -0.4886230528354645,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.875,
      "logps/chosen": -0.748828113079071,
      "logps/rejected": -1.473046898841858,
      "loss": 1.1259,
      "nll_loss": 1.086523413658142,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07485351711511612,
      "rewards/margins": 0.0723876953125,
      "rewards/rejected": -0.14731445908546448,
      "step": 680
    },
    {
      "epoch": 0.05024210871227291,
      "grad_norm": 1.7932886969400792,
      "learning_rate": 3.0455479505075235e-06,
      "log_odds_chosen": 0.76513671875,
      "log_odds_ratio": -0.5536133050918579,
      "logits/chosen": -0.9830077886581421,
      "logits/rejected": -0.9150390625,
      "logps/chosen": -0.7515624761581421,
      "logps/rejected": -1.309179663658142,
      "loss": 1.1313,
      "nll_loss": 1.025976538658142,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07515869289636612,
      "rewards/margins": 0.055755615234375,
      "rewards/rejected": -0.13076171278953552,
      "step": 690
    },
    {
      "epoch": 0.05097025521534933,
      "grad_norm": 1.3812739282921287,
      "learning_rate": 3.0237157840738173e-06,
      "log_odds_chosen": 0.7027343511581421,
      "log_odds_ratio": -0.5433593988418579,
      "logits/chosen": -0.9306640625,
      "logits/rejected": -0.872265636920929,
      "logps/chosen": -0.7744140625,
      "logps/rejected": -1.233007788658142,
      "loss": 1.1238,
      "nll_loss": 1.1443359851837158,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07749023288488388,
      "rewards/margins": 0.04592742770910263,
      "rewards/rejected": -0.123291015625,
      "step": 700
    },
    {
      "epoch": 0.05169840171842575,
      "grad_norm": 1.4638094361730472,
      "learning_rate": 3.002346500163206e-06,
      "log_odds_chosen": 0.8546386957168579,
      "log_odds_ratio": -0.4844726622104645,
      "logits/chosen": -0.9876953363418579,
      "logits/rejected": -0.936718761920929,
      "logps/chosen": -0.70361328125,
      "logps/rejected": -1.2419922351837158,
      "loss": 1.1139,
      "nll_loss": 1.0587890148162842,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07028808444738388,
      "rewards/margins": 0.05384521558880806,
      "rewards/rejected": -0.12429199367761612,
      "step": 710
    },
    {
      "epoch": 0.05242654822150217,
      "grad_norm": 1.5176926052780488,
      "learning_rate": 2.9814239699997195e-06,
      "log_odds_chosen": 0.8565429449081421,
      "log_odds_ratio": -0.5035644769668579,
      "logits/chosen": -1.0478515625,
      "logits/rejected": -0.961132824420929,
      "logps/chosen": -0.734375,
      "logps/rejected": -1.3244140148162842,
      "loss": 1.1277,
      "nll_loss": 1.076562523841858,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07342529296875,
      "rewards/margins": 0.05902709811925888,
      "rewards/rejected": -0.13239745795726776,
      "step": 720
    },
    {
      "epoch": 0.05315469472457859,
      "grad_norm": 1.4510014022690885,
      "learning_rate": 2.9609328407904207e-06,
      "log_odds_chosen": 0.6505126953125,
      "log_odds_ratio": -0.551562488079071,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.9556640386581421,
      "logps/chosen": -0.7339843511581421,
      "logps/rejected": -1.146875023841858,
      "loss": 1.1182,
      "nll_loss": 1.1130859851837158,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07340087741613388,
      "rewards/margins": 0.041185759007930756,
      "rewards/rejected": -0.11457519233226776,
      "step": 730
    },
    {
      "epoch": 0.05388284122765501,
      "grad_norm": 1.5201286658330744,
      "learning_rate": 2.940858488375231e-06,
      "log_odds_chosen": 0.6685546636581421,
      "log_odds_ratio": -0.549267590045929,
      "logits/chosen": -1.053320288658142,
      "logits/rejected": -0.9599609375,
      "logps/chosen": -0.7748047113418579,
      "logps/rejected": -1.248632788658142,
      "loss": 1.1478,
      "nll_loss": 1.1228516101837158,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07745361328125,
      "rewards/margins": 0.04729461669921875,
      "rewards/rejected": -0.12482909858226776,
      "step": 740
    },
    {
      "epoch": 0.054610987730731426,
      "grad_norm": 1.5232617658745549,
      "learning_rate": 2.9211869733608857e-06,
      "log_odds_chosen": 0.8666747808456421,
      "log_odds_ratio": -0.5086914300918579,
      "logits/chosen": -1.0291016101837158,
      "logits/rejected": -0.9716796875,
      "logps/chosen": -0.7669922113418579,
      "logps/rejected": -1.3732421398162842,
      "loss": 1.1273,
      "nll_loss": 1.015234351158142,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07661132514476776,
      "rewards/margins": 0.0606536865234375,
      "rewards/rejected": -0.13735350966453552,
      "step": 750
    },
    {
      "epoch": 0.05533913423380784,
      "grad_norm": 1.554150267019514,
      "learning_rate": 2.901905000440047e-06,
      "log_odds_chosen": 1.002893090248108,
      "log_odds_ratio": -0.4734863340854645,
      "logits/chosen": -1.052148461341858,
      "logits/rejected": -0.989062488079071,
      "logps/chosen": -0.774707019329071,
      "logps/rejected": -1.4660155773162842,
      "loss": 1.1179,
      "nll_loss": 1.0564453601837158,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07745361328125,
      "rewards/margins": 0.06914367526769638,
      "rewards/rejected": -0.1466064453125,
      "step": 760
    },
    {
      "epoch": 0.05606728073688426,
      "grad_norm": 1.6235967684077466,
      "learning_rate": 2.8829998806257885e-06,
      "log_odds_chosen": 0.967846691608429,
      "log_odds_ratio": -0.502246081829071,
      "logits/chosen": -1.0460937023162842,
      "logits/rejected": -0.9306640625,
      "logps/chosen": -0.719921886920929,
      "logps/rejected": -1.4093749523162842,
      "loss": 1.108,
      "nll_loss": 0.981640636920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07194824516773224,
      "rewards/margins": 0.0689239501953125,
      "rewards/rejected": -0.14094237983226776,
      "step": 770
    },
    {
      "epoch": 0.05679542723996068,
      "grad_norm": 1.6284312497330824,
      "learning_rate": 2.8644594961577314e-06,
      "log_odds_chosen": 0.686022937297821,
      "log_odds_ratio": -0.58935546875,
      "logits/chosen": -1.0148437023162842,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.8052734136581421,
      "logps/rejected": -1.2841796875,
      "loss": 1.105,
      "nll_loss": 1.0955078601837158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.08050537109375,
      "rewards/margins": 0.04795227199792862,
      "rewards/rejected": -0.12849120795726776,
      "step": 780
    },
    {
      "epoch": 0.0575235737430371,
      "grad_norm": 1.4199339043529113,
      "learning_rate": 2.84627226785928e-06,
      "log_odds_chosen": 0.8512207269668579,
      "log_odds_ratio": -0.481201171875,
      "logits/chosen": -0.987109363079071,
      "logits/rejected": -0.8896484375,
      "logps/chosen": -0.673144519329071,
      "logps/rejected": -1.225195288658142,
      "loss": 1.1308,
      "nll_loss": 1.0949218273162842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06732177734375,
      "rewards/margins": 0.055206298828125,
      "rewards/rejected": -0.12246093899011612,
      "step": 790
    },
    {
      "epoch": 0.05825172024611352,
      "grad_norm": 1.3919340886975955,
      "learning_rate": 2.82842712474619e-06,
      "log_odds_chosen": 0.902099609375,
      "log_odds_ratio": -0.4999023377895355,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.9033203125,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.313867211341858,
      "loss": 1.0997,
      "nll_loss": 1.0578124523162842,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07080078125,
      "rewards/margins": 0.06048583984375,
      "rewards/rejected": -0.13129882514476776,
      "step": 800
    },
    {
      "epoch": 0.05897986674918994,
      "grad_norm": 1.4695419553082982,
      "learning_rate": 2.810913475705226e-06,
      "log_odds_chosen": 1.1593749523162842,
      "log_odds_ratio": -0.4581054747104645,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.944531261920929,
      "logps/chosen": -0.7826172113418579,
      "logps/rejected": -1.597265601158142,
      "loss": 1.0938,
      "nll_loss": 1.076757788658142,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0782470703125,
      "rewards/margins": 0.08135986328125,
      "rewards/rejected": -0.15969237685203552,
      "step": 810
    },
    {
      "epoch": 0.05970801325226636,
      "grad_norm": 1.4805729889982453,
      "learning_rate": 2.7937211830783128e-06,
      "log_odds_chosen": 0.931445300579071,
      "log_odds_ratio": -0.5008789300918579,
      "logits/chosen": -1.0496094226837158,
      "logits/rejected": -0.9468749761581421,
      "logps/chosen": -0.7357422113418579,
      "logps/rejected": -1.3576171398162842,
      "loss": 1.1064,
      "nll_loss": 1.0029296875,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0736083984375,
      "rewards/margins": 0.06226501613855362,
      "rewards/rejected": -0.13579101860523224,
      "step": 820
    },
    {
      "epoch": 0.06043615975534278,
      "grad_norm": 1.4434030394502348,
      "learning_rate": 2.776840538002493e-06,
      "log_odds_chosen": 0.8809570074081421,
      "log_odds_ratio": -0.4793945252895355,
      "logits/chosen": -1.033789038658142,
      "logits/rejected": -0.9521484375,
      "logps/chosen": -0.69775390625,
      "logps/rejected": -1.298437476158142,
      "loss": 1.1303,
      "nll_loss": 1.061914086341858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06975097954273224,
      "rewards/margins": 0.06011962890625,
      "rewards/rejected": -0.12985840439796448,
      "step": 830
    },
    {
      "epoch": 0.0611643062584192,
      "grad_norm": 1.445210902579111,
      "learning_rate": 2.7602622373694163e-06,
      "log_odds_chosen": 1.0266602039337158,
      "log_odds_ratio": -0.48124998807907104,
      "logits/chosen": -1.096093773841858,
      "logits/rejected": -1.048242211341858,
      "logps/chosen": -0.740234375,
      "logps/rejected": -1.460351586341858,
      "loss": 1.1085,
      "nll_loss": 1.003515601158142,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.07198486477136612,
      "rewards/rejected": -0.14599609375,
      "step": 840
    },
    {
      "epoch": 0.06189245276149561,
      "grad_norm": 1.477719898018172,
      "learning_rate": 2.743977362280141e-06,
      "log_odds_chosen": 0.818920910358429,
      "log_odds_ratio": -0.54248046875,
      "logits/chosen": -1.083593726158142,
      "logits/rejected": -0.999804675579071,
      "logps/chosen": -0.7671874761581421,
      "logps/rejected": -1.345117211341858,
      "loss": 1.0925,
      "nll_loss": 1.115820288658142,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07666015625,
      "rewards/margins": 0.05780944973230362,
      "rewards/rejected": -0.134521484375,
      "step": 850
    },
    {
      "epoch": 0.06262059926457203,
      "grad_norm": 1.5664238020213266,
      "learning_rate": 2.7279773578818937e-06,
      "log_odds_chosen": 0.76129150390625,
      "log_odds_ratio": -0.548632800579071,
      "logits/chosen": -1.045312523841858,
      "logits/rejected": -0.977734386920929,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.2587890625,
      "loss": 1.1179,
      "nll_loss": 1.1189453601837158,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07407226413488388,
      "rewards/margins": 0.05186767503619194,
      "rewards/rejected": -0.1259765625,
      "step": 860
    },
    {
      "epoch": 0.06334874576764846,
      "grad_norm": 1.632884354692461,
      "learning_rate": 2.7122540144832417e-06,
      "log_odds_chosen": 0.7477782964706421,
      "log_odds_ratio": -0.574267566204071,
      "logits/chosen": -1.031640648841858,
      "logits/rejected": -0.9537109136581421,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.2312500476837158,
      "loss": 1.1306,
      "nll_loss": 1.1638672351837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0740966796875,
      "rewards/margins": 0.04911651462316513,
      "rewards/rejected": -0.12309570610523224,
      "step": 870
    },
    {
      "epoch": 0.06407689227072487,
      "grad_norm": 1.5482195059977542,
      "learning_rate": 2.696799449852968e-06,
      "log_odds_chosen": 0.804443359375,
      "log_odds_ratio": -0.525634765625,
      "logits/chosen": -1.04296875,
      "logits/rejected": -0.9759765863418579,
      "logps/chosen": -0.711718738079071,
      "logps/rejected": -1.258398413658142,
      "loss": 1.0972,
      "nll_loss": 1.0595703125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0711669921875,
      "rewards/margins": 0.054766081273555756,
      "rewards/rejected": -0.12595215439796448,
      "step": 880
    },
    {
      "epoch": 0.06480503877380128,
      "grad_norm": 3.5555074291227964,
      "learning_rate": 2.6816060926159636e-06,
      "log_odds_chosen": 0.9256347417831421,
      "log_odds_ratio": -0.5201171636581421,
      "logits/chosen": -1.0642578601837158,
      "logits/rejected": -0.98046875,
      "logps/chosen": -0.7430664300918579,
      "logps/rejected": -1.408789038658142,
      "loss": 1.1157,
      "nll_loss": 0.9947265386581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07421875,
      "rewards/margins": 0.066558837890625,
      "rewards/rejected": -0.14091797173023224,
      "step": 890
    },
    {
      "epoch": 0.06553318527687771,
      "grad_norm": 1.4174933600999189,
      "learning_rate": 2.6666666666666664e-06,
      "log_odds_chosen": 0.6136230230331421,
      "log_odds_ratio": -0.5966796875,
      "logits/chosen": -1.002539038658142,
      "logits/rejected": -0.934765636920929,
      "logps/chosen": -0.752148449420929,
      "logps/rejected": -1.1818358898162842,
      "loss": 1.093,
      "nll_loss": 1.0529296398162842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07524414360523224,
      "rewards/margins": 0.04295806959271431,
      "rewards/rejected": -0.11813964694738388,
      "step": 900
    },
    {
      "epoch": 0.06626133177995412,
      "grad_norm": 1.4890553701732194,
      "learning_rate": 2.6519741765271837e-06,
      "log_odds_chosen": 0.9593750238418579,
      "log_odds_ratio": -0.4740234315395355,
      "logits/chosen": -1.014257788658142,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.744921863079071,
      "logps/rejected": -1.415624976158142,
      "loss": 1.1001,
      "nll_loss": 1.103515625,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07451172173023224,
      "rewards/margins": 0.06707458198070526,
      "rewards/rejected": -0.14152832329273224,
      "step": 910
    },
    {
      "epoch": 0.06698947828303055,
      "grad_norm": 1.3975980730325914,
      "learning_rate": 2.637521893583148e-06,
      "log_odds_chosen": 0.848095715045929,
      "log_odds_ratio": -0.515429675579071,
      "logits/chosen": -0.9839843511581421,
      "logits/rejected": -0.9105468988418579,
      "logps/chosen": -0.764843761920929,
      "logps/rejected": -1.356835961341858,
      "loss": 1.1127,
      "nll_loss": 1.0896484851837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07646484673023224,
      "rewards/margins": 0.05929107591509819,
      "rewards/rejected": -0.13579101860523224,
      "step": 920
    },
    {
      "epoch": 0.06771762478610696,
      "grad_norm": 1.435023633429663,
      "learning_rate": 2.6233033431358115e-06,
      "log_odds_chosen": 0.843212902545929,
      "log_odds_ratio": -0.5230468511581421,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.931445300579071,
      "logps/chosen": -0.745898425579071,
      "logps/rejected": -1.337499976158142,
      "loss": 1.0889,
      "nll_loss": 1.1142578125,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07463379204273224,
      "rewards/margins": 0.05906982347369194,
      "rewards/rejected": -0.13371582329273224,
      "step": 930
    },
    {
      "epoch": 0.06844577128918339,
      "grad_norm": 1.4935717264406958,
      "learning_rate": 2.6093122922137685e-06,
      "log_odds_chosen": 0.94183349609375,
      "log_odds_ratio": -0.5414062738418579,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.926953136920929,
      "logps/chosen": -0.775683581829071,
      "logps/rejected": -1.462304711341858,
      "loss": 1.0815,
      "nll_loss": 1.078515648841858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07757568359375,
      "rewards/margins": 0.06874694675207138,
      "rewards/rejected": -0.14619140326976776,
      "step": 940
    },
    {
      "epoch": 0.0691739177922598,
      "grad_norm": 1.510518929242062,
      "learning_rate": 2.5955427380922006e-06,
      "log_odds_chosen": 0.7543700933456421,
      "log_odds_ratio": -0.552539050579071,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.719433605670929,
      "logps/rejected": -1.2312500476837158,
      "loss": 1.1039,
      "nll_loss": 1.088281273841858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07185058295726776,
      "rewards/margins": 0.05126190185546875,
      "rewards/rejected": -0.12324218451976776,
      "step": 950
    },
    {
      "epoch": 0.06990206429533623,
      "grad_norm": 1.4306650324913062,
      "learning_rate": 2.5819888974716113e-06,
      "log_odds_chosen": 0.940722644329071,
      "log_odds_ratio": -0.4866699278354645,
      "logits/chosen": -0.9966796636581421,
      "logits/rejected": -0.9052734375,
      "logps/chosen": -0.7251952886581421,
      "logps/rejected": -1.332421898841858,
      "loss": 1.092,
      "nll_loss": 1.0734374523162842,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07254638522863388,
      "rewards/margins": 0.06078185886144638,
      "rewards/rejected": -0.13332518935203552,
      "step": 960
    },
    {
      "epoch": 0.07063021079841264,
      "grad_norm": 2.329925696452697,
      "learning_rate": 2.5686451962717425e-06,
      "log_odds_chosen": 1.128076195716858,
      "log_odds_ratio": -0.46479493379592896,
      "logits/chosen": -1.0232422351837158,
      "logits/rejected": -0.9603515863418579,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.5144531726837158,
      "loss": 1.0808,
      "nll_loss": 1.017578125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07120361179113388,
      "rewards/margins": 0.08016662299633026,
      "rewards/rejected": -0.1514892578125,
      "step": 970
    },
    {
      "epoch": 0.07135835730148905,
      "grad_norm": 1.6432086166632125,
      "learning_rate": 2.5555062599997596e-06,
      "log_odds_chosen": 0.9437500238418579,
      "log_odds_ratio": -0.5213867425918579,
      "logits/chosen": -1.048437476158142,
      "logits/rejected": -0.9541015625,
      "logps/chosen": -0.7256835699081421,
      "logps/rejected": -1.3894531726837158,
      "loss": 1.1013,
      "nll_loss": 1.0392577648162842,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07248535007238388,
      "rewards/margins": 0.0663604736328125,
      "rewards/rejected": -0.13894042372703552,
      "step": 980
    },
    {
      "epoch": 0.07208650380456548,
      "grad_norm": 1.5872670734097316,
      "learning_rate": 2.5425669046549126e-06,
      "log_odds_chosen": 0.921679675579071,
      "log_odds_ratio": -0.47236329317092896,
      "logits/chosen": -0.975781261920929,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.7533203363418579,
      "logps/rejected": -1.401757836341858,
      "loss": 1.1028,
      "nll_loss": 1.0740234851837158,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07534179836511612,
      "rewards/margins": 0.06479492038488388,
      "rewards/rejected": -0.1402587890625,
      "step": 990
    },
    {
      "epoch": 0.07281465030764189,
      "grad_norm": 1.4221976989012124,
      "learning_rate": 2.5298221281347034e-06,
      "log_odds_chosen": 1.090576171875,
      "log_odds_ratio": -0.48486328125,
      "logits/chosen": -1.0205078125,
      "logits/rejected": -0.9134765863418579,
      "logps/chosen": -0.702832043170929,
      "logps/rejected": -1.4716796875,
      "loss": 1.0746,
      "nll_loss": 0.968945324420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07027588039636612,
      "rewards/margins": 0.076873779296875,
      "rewards/rejected": -0.147216796875,
      "step": 1000
    },
    {
      "epoch": 0.07354279681071832,
      "grad_norm": 1.383077192382236,
      "learning_rate": 2.5172671021102103e-06,
      "log_odds_chosen": 0.972949206829071,
      "log_odds_ratio": -0.515820324420929,
      "logits/chosen": -1.012109398841858,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.741406261920929,
      "logps/rejected": -1.4412109851837158,
      "loss": 1.0932,
      "nll_loss": 0.9925781488418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07416991889476776,
      "rewards/margins": 0.07006530463695526,
      "rewards/rejected": -0.14414063096046448,
      "step": 1010
    },
    {
      "epoch": 0.07427094331379473,
      "grad_norm": 2.223136668653118,
      "learning_rate": 2.504897164340598e-06,
      "log_odds_chosen": 0.84228515625,
      "log_odds_ratio": -0.5210937261581421,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.8960937261581421,
      "logps/chosen": -0.723437488079071,
      "logps/rejected": -1.326171875,
      "loss": 1.0865,
      "nll_loss": 0.9447265863418579,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07232666015625,
      "rewards/margins": 0.06026763841509819,
      "rewards/rejected": -0.13264159858226776,
      "step": 1020
    },
    {
      "epoch": 0.07499908981687116,
      "grad_norm": 1.529298401359949,
      "learning_rate": 2.492707811399023e-06,
      "log_odds_chosen": 0.994677722454071,
      "log_odds_ratio": -0.47016602754592896,
      "logits/chosen": -0.970898449420929,
      "logits/rejected": -0.8958984613418579,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.388085961341858,
      "loss": 1.104,
      "nll_loss": 1.0750000476837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07166747748851776,
      "rewards/margins": 0.06716613471508026,
      "rewards/rejected": -0.13876953721046448,
      "step": 1030
    },
    {
      "epoch": 0.07572723631994757,
      "grad_norm": 1.9274793719149392,
      "learning_rate": 2.480694691784169e-06,
      "log_odds_chosen": 1.168554663658142,
      "log_odds_ratio": -0.451904296875,
      "logits/chosen": -0.960156261920929,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.660351574420929,
      "logps/rejected": -1.4646484851837158,
      "loss": 1.0873,
      "nll_loss": 0.9681640863418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06596679985523224,
      "rewards/margins": 0.0804443359375,
      "rewards/rejected": -0.14638671278953552,
      "step": 1040
    },
    {
      "epoch": 0.076455382823024,
      "grad_norm": 1.4681245416457267,
      "learning_rate": 2.4688535993934706e-06,
      "log_odds_chosen": 0.8275512456893921,
      "log_odds_ratio": -0.522216796875,
      "logits/chosen": -0.9496093988418579,
      "logits/rejected": -0.8873046636581421,
      "logps/chosen": -0.767578125,
      "logps/rejected": -1.328710913658142,
      "loss": 1.0935,
      "nll_loss": 1.037109375,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07677002251148224,
      "rewards/margins": 0.05603943020105362,
      "rewards/rejected": -0.13288573920726776,
      "step": 1050
    },
    {
      "epoch": 0.07718352932610041,
      "grad_norm": 1.368834243715235,
      "learning_rate": 2.457180467335805e-06,
      "log_odds_chosen": 1.0066406726837158,
      "log_odds_ratio": -0.4501953125,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.9017578363418579,
      "logps/chosen": -0.73828125,
      "logps/rejected": -1.4298827648162842,
      "loss": 1.1135,
      "nll_loss": 1.015625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07381591945886612,
      "rewards/margins": 0.06925048679113388,
      "rewards/rejected": -0.1431884765625,
      "step": 1060
    },
    {
      "epoch": 0.07791167582917682,
      "grad_norm": 1.4152081073930696,
      "learning_rate": 2.4456713620629725e-06,
      "log_odds_chosen": 0.772229015827179,
      "log_odds_ratio": -0.541455090045929,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.925585925579071,
      "logps/chosen": -0.760546863079071,
      "logps/rejected": -1.306054711341858,
      "loss": 1.0868,
      "nll_loss": 1.053320288658142,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07602538913488388,
      "rewards/margins": 0.05467681959271431,
      "rewards/rejected": -0.1307373046875,
      "step": 1070
    },
    {
      "epoch": 0.07863982233225325,
      "grad_norm": 1.9822301530309727,
      "learning_rate": 2.4343224778007378e-06,
      "log_odds_chosen": 0.9384521245956421,
      "log_odds_ratio": -0.4916015565395355,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.934765636920929,
      "logps/chosen": -0.746386706829071,
      "logps/rejected": -1.4005858898162842,
      "loss": 1.0838,
      "nll_loss": 1.064062476158142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0745849609375,
      "rewards/margins": 0.06540985405445099,
      "rewards/rejected": -0.13999024033546448,
      "step": 1080
    },
    {
      "epoch": 0.07936796883532966,
      "grad_norm": 2.202526241983929,
      "learning_rate": 2.4231301312615306e-06,
      "log_odds_chosen": 1.0329101085662842,
      "log_odds_ratio": -0.49482423067092896,
      "logits/chosen": -0.975781261920929,
      "logits/rejected": -0.9292968511581421,
      "logps/chosen": -0.716992199420929,
      "logps/rejected": -1.4386718273162842,
      "loss": 1.0802,
      "nll_loss": 1.048437476158142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07170410454273224,
      "rewards/margins": 0.07221069186925888,
      "rewards/rejected": -0.14394530653953552,
      "step": 1090
    },
    {
      "epoch": 0.08009611533840609,
      "grad_norm": 1.6027988332078156,
      "learning_rate": 2.412090756622109e-06,
      "log_odds_chosen": 1.067169189453125,
      "log_odds_ratio": -0.48115235567092896,
      "logits/chosen": -1.029687523841858,
      "logits/rejected": -0.953906238079071,
      "logps/chosen": -0.7548828125,
      "logps/rejected": -1.5302734375,
      "loss": 1.0948,
      "nll_loss": 1.0304687023162842,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07551269233226776,
      "rewards/margins": 0.07766570895910263,
      "rewards/rejected": -0.15302734076976776,
      "step": 1100
    },
    {
      "epoch": 0.0808242618414825,
      "grad_norm": 1.643132733255971,
      "learning_rate": 2.401200900750657e-06,
      "log_odds_chosen": 0.902056872844696,
      "log_odds_ratio": -0.533007800579071,
      "logits/chosen": -1.0412108898162842,
      "logits/rejected": -0.946093738079071,
      "logps/chosen": -0.7572265863418579,
      "logps/rejected": -1.4031250476837158,
      "loss": 1.0726,
      "nll_loss": 1.0583984851837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07576904445886612,
      "rewards/margins": 0.06465759128332138,
      "rewards/rejected": -0.14042969048023224,
      "step": 1110
    },
    {
      "epoch": 0.08155240834455893,
      "grad_norm": 1.9226517362029898,
      "learning_rate": 2.390457218668787e-06,
      "log_odds_chosen": 1.058691382408142,
      "log_odds_ratio": -0.4754882752895355,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.941601574420929,
      "logps/chosen": -0.7318359613418579,
      "logps/rejected": -1.4845702648162842,
      "loss": 1.0736,
      "nll_loss": 1.0519530773162842,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07321777194738388,
      "rewards/margins": 0.07510223239660263,
      "rewards/rejected": -0.1484375,
      "step": 1120
    },
    {
      "epoch": 0.08228055484763534,
      "grad_norm": 1.5707523722744505,
      "learning_rate": 2.379856469234918e-06,
      "log_odds_chosen": 0.9898315668106079,
      "log_odds_ratio": -0.515429675579071,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.9574218988418579,
      "logps/chosen": -0.7496093511581421,
      "logps/rejected": -1.4255859851837158,
      "loss": 1.0878,
      "nll_loss": 1.101171851158142,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07503662258386612,
      "rewards/margins": 0.06759033352136612,
      "rewards/rejected": -0.1427001953125,
      "step": 1130
    },
    {
      "epoch": 0.08300870135071177,
      "grad_norm": 1.5151634227882143,
      "learning_rate": 2.369395511036369e-06,
      "log_odds_chosen": 0.921337902545929,
      "log_odds_ratio": -0.5189453363418579,
      "logits/chosen": -1.0085937976837158,
      "logits/rejected": -0.949414074420929,
      "logps/chosen": -0.712695300579071,
      "logps/rejected": -1.3650391101837158,
      "loss": 1.0905,
      "nll_loss": 0.979296863079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.06532363593578339,
      "rewards/rejected": -0.1365966796875,
      "step": 1140
    },
    {
      "epoch": 0.08373684785378818,
      "grad_norm": 1.6354502026488225,
      "learning_rate": 2.359071298478354e-06,
      "log_odds_chosen": 1.1275146007537842,
      "log_odds_ratio": -0.452392578125,
      "logits/chosen": -1.0578124523162842,
      "logits/rejected": -0.9759765863418579,
      "logps/chosen": -0.7294921875,
      "logps/rejected": -1.506445288658142,
      "loss": 1.0588,
      "nll_loss": 1.0207030773162842,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07301025092601776,
      "rewards/margins": 0.07759933173656464,
      "rewards/rejected": -0.15061035752296448,
      "step": 1150
    },
    {
      "epoch": 0.0844649943568646,
      "grad_norm": 1.6252368308796055,
      "learning_rate": 2.3488808780588137e-06,
      "log_odds_chosen": 0.969042956829071,
      "log_odds_ratio": -0.4974609315395355,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.9468749761581421,
      "logps/chosen": -0.702929675579071,
      "logps/rejected": -1.384765625,
      "loss": 1.0991,
      "nll_loss": 0.9876953363418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07027588039636612,
      "rewards/margins": 0.06812133640050888,
      "rewards/rejected": -0.13837890326976776,
      "step": 1160
    },
    {
      "epoch": 0.08519314085994102,
      "grad_norm": 1.5239459289762518,
      "learning_rate": 2.3388213848187446e-06,
      "log_odds_chosen": 0.9083496332168579,
      "log_odds_ratio": -0.519726574420929,
      "logits/chosen": -1.035742163658142,
      "logits/rejected": -0.9693359136581421,
      "logps/chosen": -0.776562511920929,
      "logps/rejected": -1.4111328125,
      "loss": 1.0821,
      "nll_loss": 1.0128905773162842,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.07764892280101776,
      "rewards/margins": 0.06345214694738388,
      "rewards/rejected": -0.14116211235523224,
      "step": 1170
    },
    {
      "epoch": 0.08592128736301743,
      "grad_norm": 1.565448796675523,
      "learning_rate": 2.328890038958328e-06,
      "log_odds_chosen": 1.08013916015625,
      "log_odds_ratio": -0.508056640625,
      "logits/chosen": -1.022851586341858,
      "logits/rejected": -0.943164050579071,
      "logps/chosen": -0.7431640625,
      "logps/rejected": -1.554101586341858,
      "loss": 1.0871,
      "nll_loss": 0.978515625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07426758110523224,
      "rewards/margins": 0.08112182468175888,
      "rewards/rejected": -0.15544433891773224,
      "step": 1180
    },
    {
      "epoch": 0.08664943386609386,
      "grad_norm": 1.7610646634625353,
      "learning_rate": 2.3190841426097937e-06,
      "log_odds_chosen": 0.895751953125,
      "log_odds_ratio": -0.509570300579071,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.921679675579071,
      "logps/chosen": -0.731640636920929,
      "logps/rejected": -1.3388671875,
      "loss": 1.0859,
      "nll_loss": 1.0378906726837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07314453274011612,
      "rewards/margins": 0.06079406663775444,
      "rewards/rejected": -0.13400879502296448,
      "step": 1190
    },
    {
      "epoch": 0.08737758036917027,
      "grad_norm": 1.4186266041359494,
      "learning_rate": 2.309401076758503e-06,
      "log_odds_chosen": 0.97607421875,
      "log_odds_ratio": -0.501660168170929,
      "logits/chosen": -1.032617211341858,
      "logits/rejected": -0.9296875,
      "logps/chosen": -0.736523449420929,
      "logps/rejected": -1.4294922351837158,
      "loss": 1.0894,
      "nll_loss": 1.083398461341858,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07362060248851776,
      "rewards/margins": 0.06939087063074112,
      "rewards/rejected": -0.14301757514476776,
      "step": 1200
    },
    {
      "epoch": 0.0881057268722467,
      "grad_norm": 1.562127406514597,
      "learning_rate": 2.299838298304276e-06,
      "log_odds_chosen": 0.9479004144668579,
      "log_odds_ratio": -0.4913085997104645,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.9271484613418579,
      "logps/chosen": -0.751171886920929,
      "logps/rejected": -1.411718726158142,
      "loss": 1.055,
      "nll_loss": 1.0447266101837158,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0750732421875,
      "rewards/margins": 0.06611251831054688,
      "rewards/rejected": -0.1412353515625,
      "step": 1210
    },
    {
      "epoch": 0.08883387337532311,
      "grad_norm": 1.5505309130527312,
      "learning_rate": 2.2903933372554728e-06,
      "log_odds_chosen": 1.0265624523162842,
      "log_odds_ratio": -0.45332032442092896,
      "logits/chosen": -1.0087890625,
      "logits/rejected": -0.912304699420929,
      "logps/chosen": -0.716992199420929,
      "logps/rejected": -1.415429711341858,
      "loss": 1.0463,
      "nll_loss": 0.968554675579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07176513969898224,
      "rewards/margins": 0.06972046196460724,
      "rewards/rejected": -0.14157715439796448,
      "step": 1220
    },
    {
      "epoch": 0.08956201987839954,
      "grad_norm": 1.5296659589041084,
      "learning_rate": 2.281063794048804e-06,
      "log_odds_chosen": 1.3583984375,
      "log_odds_ratio": -0.43110352754592896,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.926953136920929,
      "logps/chosen": -0.730664074420929,
      "logps/rejected": -1.696874976158142,
      "loss": 1.0678,
      "nll_loss": 1.011328101158142,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07313232123851776,
      "rewards/margins": 0.09641113132238388,
      "rewards/rejected": -0.16960449516773224,
      "step": 1230
    },
    {
      "epoch": 0.09029016638147595,
      "grad_norm": 1.4841860037908619,
      "learning_rate": 2.271847336988259e-06,
      "log_odds_chosen": 1.1979491710662842,
      "log_odds_ratio": -0.4358886778354645,
      "logits/chosen": -1.092187523841858,
      "logits/rejected": -0.991992175579071,
      "logps/chosen": -0.695507824420929,
      "logps/rejected": -1.5234375,
      "loss": 1.0795,
      "nll_loss": 0.979687511920929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06956787407398224,
      "rewards/margins": 0.08283539116382599,
      "rewards/rejected": -0.15234375,
      "step": 1240
    },
    {
      "epoch": 0.09101831288455237,
      "grad_norm": 1.6108566103762665,
      "learning_rate": 2.262741699796952e-06,
      "log_odds_chosen": 1.051782250404358,
      "log_odds_ratio": -0.49951171875,
      "logits/chosen": -1.0070312023162842,
      "logits/rejected": -0.9400390386581421,
      "logps/chosen": -0.782031238079071,
      "logps/rejected": -1.535546898841858,
      "loss": 1.0888,
      "nll_loss": 1.052734375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07827148586511612,
      "rewards/margins": 0.07554473727941513,
      "rewards/rejected": -0.15375976264476776,
      "step": 1250
    },
    {
      "epoch": 0.09174645938762879,
      "grad_norm": 1.4972463881173497,
      "learning_rate": 2.253744679276044e-06,
      "log_odds_chosen": 1.225622534751892,
      "log_odds_ratio": -0.4591308534145355,
      "logits/chosen": -1.000585913658142,
      "logits/rejected": -0.938671886920929,
      "logps/chosen": -0.698046863079071,
      "logps/rejected": -1.5841796398162842,
      "loss": 1.0734,
      "nll_loss": 0.998046875,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06973876804113388,
      "rewards/margins": 0.08869476616382599,
      "rewards/rejected": -0.158447265625,
      "step": 1260
    },
    {
      "epoch": 0.0924746058907052,
      "grad_norm": 1.8348572484361985,
      "learning_rate": 2.244854133065255e-06,
      "log_odds_chosen": 1.0773437023162842,
      "log_odds_ratio": -0.5150390863418579,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.9466797113418579,
      "logps/chosen": -0.741406261920929,
      "logps/rejected": -1.539648413658142,
      "loss": 1.0932,
      "nll_loss": 1.0251953601837158,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07416991889476776,
      "rewards/margins": 0.07987823337316513,
      "rewards/rejected": -0.15402832627296448,
      "step": 1270
    },
    {
      "epoch": 0.09320275239378163,
      "grad_norm": 1.6133142568483645,
      "learning_rate": 2.2360679774997895e-06,
      "log_odds_chosen": 0.969921886920929,
      "log_odds_ratio": -0.4927734434604645,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.7376953363418579,
      "logps/rejected": -1.412500023841858,
      "loss": 1.0842,
      "nll_loss": 1.0671875476837158,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07375488430261612,
      "rewards/margins": 0.06759033352136612,
      "rewards/rejected": -0.14133301377296448,
      "step": 1280
    },
    {
      "epoch": 0.09393089889685804,
      "grad_norm": 1.6775542759003017,
      "learning_rate": 2.2273841855588183e-06,
      "log_odds_chosen": 1.2883789539337158,
      "log_odds_ratio": -0.42138671875,
      "logits/chosen": -1.088281273841858,
      "logits/rejected": -0.993945300579071,
      "logps/chosen": -0.7162109613418579,
      "logps/rejected": -1.618749976158142,
      "loss": 1.0621,
      "nll_loss": 1.0001952648162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07164306938648224,
      "rewards/margins": 0.09030761569738388,
      "rewards/rejected": -0.16184082627296448,
      "step": 1290
    },
    {
      "epoch": 0.09465904539993447,
      "grad_norm": 1.507207195221332,
      "learning_rate": 2.2188007849009167e-06,
      "log_odds_chosen": 1.1524779796600342,
      "log_odds_ratio": -0.46198731660842896,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.9205077886581421,
      "logps/chosen": -0.708203136920929,
      "logps/rejected": -1.552148461341858,
      "loss": 1.0799,
      "nll_loss": 1.0417969226837158,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07082519680261612,
      "rewards/margins": 0.0844573974609375,
      "rewards/rejected": -0.15507812798023224,
      "step": 1300
    },
    {
      "epoch": 0.09538719190301088,
      "grad_norm": 1.8051144128159562,
      "learning_rate": 2.2103158559821502e-06,
      "log_odds_chosen": 0.9556640386581421,
      "log_odds_ratio": -0.5223633050918579,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.7701171636581421,
      "logps/rejected": -1.4884765148162842,
      "loss": 1.0495,
      "nll_loss": 1.0460937023162842,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07700195163488388,
      "rewards/margins": 0.07175903022289276,
      "rewards/rejected": -0.14890137314796448,
      "step": 1310
    },
    {
      "epoch": 0.09611533840608731,
      "grad_norm": 1.616030602353496,
      "learning_rate": 2.2019275302527213e-06,
      "log_odds_chosen": 1.0687987804412842,
      "log_odds_ratio": -0.46826171875,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.9302734136581421,
      "logps/chosen": -0.7474609613418579,
      "logps/rejected": -1.5080077648162842,
      "loss": 1.057,
      "nll_loss": 1.0576171875,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07470703125,
      "rewards/margins": 0.076141357421875,
      "rewards/rejected": -0.15083007514476776,
      "step": 1320
    },
    {
      "epoch": 0.09684348490916372,
      "grad_norm": 1.5231051481214068,
      "learning_rate": 2.193633988428327e-06,
      "log_odds_chosen": 1.095849633216858,
      "log_odds_ratio": -0.46333009004592896,
      "logits/chosen": -1.0291016101837158,
      "logits/rejected": -0.9437500238418579,
      "logps/chosen": -0.680859386920929,
      "logps/rejected": -1.4445312023162842,
      "loss": 1.0492,
      "nll_loss": 1.014062523841858,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06807861477136612,
      "rewards/margins": 0.07635498046875,
      "rewards/rejected": -0.14453125,
      "step": 1330
    },
    {
      "epoch": 0.09757163141224014,
      "grad_norm": 1.531604982147399,
      "learning_rate": 2.185433458832612e-06,
      "log_odds_chosen": 1.081030249595642,
      "log_odds_ratio": -0.4999023377895355,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -0.921679675579071,
      "logps/chosen": -0.762890636920929,
      "logps/rejected": -1.5333983898162842,
      "loss": 1.0893,
      "nll_loss": 1.128515601158142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0762939453125,
      "rewards/margins": 0.07698974758386612,
      "rewards/rejected": -0.15327148139476776,
      "step": 1340
    },
    {
      "epoch": 0.09829977791531656,
      "grad_norm": 1.7209422087310642,
      "learning_rate": 2.177324215807269e-06,
      "log_odds_chosen": 0.7672363519668579,
      "log_odds_ratio": -0.552441418170929,
      "logits/chosen": -1.064843773841858,
      "logits/rejected": -0.991406261920929,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.2277343273162842,
      "loss": 1.0525,
      "nll_loss": 0.936328113079071,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07163085788488388,
      "rewards/margins": 0.05113830417394638,
      "rewards/rejected": -0.12287597358226776,
      "step": 1350
    },
    {
      "epoch": 0.09902792441839298,
      "grad_norm": 1.4920770020081804,
      "learning_rate": 2.1693045781865616e-06,
      "log_odds_chosen": 1.0016601085662842,
      "log_odds_ratio": -0.48388671875,
      "logits/chosen": -1.0632812976837158,
      "logits/rejected": -0.975390613079071,
      "logps/chosen": -0.706835925579071,
      "logps/rejected": -1.3830077648162842,
      "loss": 1.0659,
      "nll_loss": 1.058007836341858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07062987983226776,
      "rewards/margins": 0.06766357272863388,
      "rewards/rejected": -0.1383056640625,
      "step": 1360
    },
    {
      "epoch": 0.0997560709214694,
      "grad_norm": 1.8918684639580574,
      "learning_rate": 2.1613729078331965e-06,
      "log_odds_chosen": 1.3237793445587158,
      "log_odds_ratio": -0.4039062559604645,
      "logits/chosen": -1.0458984375,
      "logits/rejected": -0.994335949420929,
      "logps/chosen": -0.695507824420929,
      "logps/rejected": -1.6281249523162842,
      "loss": 1.0803,
      "nll_loss": 1.0031249523162842,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.09326782077550888,
      "rewards/rejected": -0.16286620497703552,
      "step": 1370
    },
    {
      "epoch": 0.10048421742454582,
      "grad_norm": 1.5676314373127447,
      "learning_rate": 2.1535276082326617e-06,
      "log_odds_chosen": 1.0940430164337158,
      "log_odds_ratio": -0.49433594942092896,
      "logits/chosen": -1.052734375,
      "logits/rejected": -0.975781261920929,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -1.4083983898162842,
      "loss": 1.0669,
      "nll_loss": 0.9701172113418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06704101711511612,
      "rewards/margins": 0.07375182956457138,
      "rewards/rejected": -0.1407470703125,
      "step": 1380
    },
    {
      "epoch": 0.10121236392762224,
      "grad_norm": 1.6226410989304125,
      "learning_rate": 2.14576712314328e-06,
      "log_odds_chosen": 1.211523413658142,
      "log_odds_ratio": -0.46953123807907104,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.9013671875,
      "logps/chosen": -0.6998046636581421,
      "logps/rejected": -1.5490233898162842,
      "loss": 1.0453,
      "nll_loss": 1.0148437023162842,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06998290866613388,
      "rewards/margins": 0.08496703952550888,
      "rewards/rejected": -0.15493163466453552,
      "step": 1390
    },
    {
      "epoch": 0.10194051043069866,
      "grad_norm": 1.5153257120800114,
      "learning_rate": 2.138089935299395e-06,
      "log_odds_chosen": 1.2509765625,
      "log_odds_ratio": -0.4617675840854645,
      "logits/chosen": -1.0714843273162842,
      "logits/rejected": -0.9873046875,
      "logps/chosen": -0.7191406488418579,
      "logps/rejected": -1.635156273841858,
      "loss": 1.0624,
      "nll_loss": 0.972851574420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07192382961511612,
      "rewards/margins": 0.09164123237133026,
      "rewards/rejected": -0.16357421875,
      "step": 1400
    },
    {
      "epoch": 0.10266865693377508,
      "grad_norm": 1.547222073839455,
      "learning_rate": 2.1304945651652297e-06,
      "log_odds_chosen": 1.2150390148162842,
      "log_odds_ratio": -0.45502930879592896,
      "logits/chosen": -1.034765601158142,
      "logits/rejected": -0.954296886920929,
      "logps/chosen": -0.689746081829071,
      "logps/rejected": -1.5789062976837158,
      "loss": 1.0458,
      "nll_loss": 1.0642578601837158,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06899414211511612,
      "rewards/margins": 0.08889465034008026,
      "rewards/rejected": -0.157958984375,
      "step": 1410
    },
    {
      "epoch": 0.1033968034368515,
      "grad_norm": 1.5809340493267345,
      "learning_rate": 2.122979569737101e-06,
      "log_odds_chosen": 1.0255858898162842,
      "log_odds_ratio": -0.46245115995407104,
      "logits/chosen": -1.041601538658142,
      "logits/rejected": -0.9517577886581421,
      "logps/chosen": -0.7085937261581421,
      "logps/rejected": -1.4210937023162842,
      "loss": 1.0833,
      "nll_loss": 1.022851586341858,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07087402045726776,
      "rewards/margins": 0.07132568210363388,
      "rewards/rejected": -0.14211425185203552,
      "step": 1420
    },
    {
      "epoch": 0.10412494993992791,
      "grad_norm": 1.5964919924770689,
      "learning_rate": 2.11554354139178e-06,
      "log_odds_chosen": 0.9452148675918579,
      "log_odds_ratio": -0.533496081829071,
      "logits/chosen": -1.1056640148162842,
      "logits/rejected": -1.008203148841858,
      "logps/chosen": -0.765625,
      "logps/rejected": -1.4462890625,
      "loss": 1.0598,
      "nll_loss": 1.0818359851837158,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07664795219898224,
      "rewards/margins": 0.06786040961742401,
      "rewards/rejected": -0.14448241889476776,
      "step": 1430
    },
    {
      "epoch": 0.10485309644300433,
      "grad_norm": 1.5760409793442225,
      "learning_rate": 2.1081851067789196e-06,
      "log_odds_chosen": 1.1474609375,
      "log_odds_ratio": -0.48115235567092896,
      "logits/chosen": -1.045507788658142,
      "logits/rejected": -0.950390636920929,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.537500023841858,
      "loss": 1.0492,
      "nll_loss": 0.9341796636581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07148437201976776,
      "rewards/margins": 0.08223877102136612,
      "rewards/rejected": -0.15383300185203552,
      "step": 1440
    },
    {
      "epoch": 0.10558124294608075,
      "grad_norm": 1.551482511303002,
      "learning_rate": 2.1009029257555606e-06,
      "log_odds_chosen": 1.193457007408142,
      "log_odds_ratio": -0.4422851502895355,
      "logits/chosen": -1.006250023841858,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.697070300579071,
      "logps/rejected": -1.5128905773162842,
      "loss": 1.0492,
      "nll_loss": 0.9736328125,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06971435248851776,
      "rewards/margins": 0.08153076469898224,
      "rewards/rejected": -0.1512451171875,
      "step": 1450
    },
    {
      "epoch": 0.10630938944915717,
      "grad_norm": 1.6128891531028628,
      "learning_rate": 2.0936956903608545e-06,
      "log_odds_chosen": 1.230859398841858,
      "log_odds_ratio": -0.455078125,
      "logits/chosen": -1.019921898841858,
      "logits/rejected": -0.9375,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -1.599609375,
      "loss": 1.0249,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07009277492761612,
      "rewards/margins": 0.089935302734375,
      "rewards/rejected": -0.15998534858226776,
      "step": 1460
    },
    {
      "epoch": 0.10703753595223359,
      "grad_norm": 1.8501092889250195,
      "learning_rate": 2.0865621238292046e-06,
      "log_odds_chosen": 0.9693359136581421,
      "log_odds_ratio": -0.5165039300918579,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.751953125,
      "logps/rejected": -1.457617163658142,
      "loss": 1.0762,
      "nll_loss": 1.015234351158142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07513427734375,
      "rewards/margins": 0.070587158203125,
      "rewards/rejected": -0.14589843153953552,
      "step": 1470
    },
    {
      "epoch": 0.10776568245531001,
      "grad_norm": 1.535500647707011,
      "learning_rate": 2.079500979640145e-06,
      "log_odds_chosen": 1.0024902820587158,
      "log_odds_ratio": -0.4825195372104645,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -0.9234374761581421,
      "logps/chosen": -0.7740234136581421,
      "logps/rejected": -1.4939453601837158,
      "loss": 1.0627,
      "nll_loss": 0.988476574420929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07738037407398224,
      "rewards/margins": 0.07195739448070526,
      "rewards/rejected": -0.14926758408546448,
      "step": 1480
    },
    {
      "epoch": 0.10849382895838643,
      "grad_norm": 1.5811838534620308,
      "learning_rate": 2.072511040603359e-06,
      "log_odds_chosen": 1.189453125,
      "log_odds_ratio": -0.4715820252895355,
      "logits/chosen": -1.0185546875,
      "logits/rejected": -0.9228515625,
      "logps/chosen": -0.730273425579071,
      "logps/rejected": -1.60546875,
      "loss": 1.0477,
      "nll_loss": 0.983203113079071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07296142727136612,
      "rewards/margins": 0.08760376274585724,
      "rewards/rejected": -0.16049805283546448,
      "step": 1490
    },
    {
      "epoch": 0.10922197546146285,
      "grad_norm": 1.6359205790422995,
      "learning_rate": 2.065591117977289e-06,
      "log_odds_chosen": 1.2340209484100342,
      "log_odds_ratio": -0.4440673887729645,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.898242175579071,
      "logps/chosen": -0.695996105670929,
      "logps/rejected": -1.567968726158142,
      "loss": 1.0545,
      "nll_loss": 1.014257788658142,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.08713836967945099,
      "rewards/rejected": -0.15676268935203552,
      "step": 1500
    },
    {
      "epoch": 0.10995012196453927,
      "grad_norm": 1.6210576908156278,
      "learning_rate": 2.058740050619915e-06,
      "log_odds_chosen": 1.2014648914337158,
      "log_odds_ratio": -0.40815430879592896,
      "logits/chosen": -1.038476586341858,
      "logits/rejected": -0.9828125238418579,
      "logps/chosen": -0.640625,
      "logps/rejected": -1.4357421398162842,
      "loss": 1.045,
      "nll_loss": 0.970507800579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06407471001148224,
      "rewards/margins": 0.07945556938648224,
      "rewards/rejected": -0.1435546875,
      "step": 1510
    },
    {
      "epoch": 0.11067826846761568,
      "grad_norm": 1.5094454583682124,
      "learning_rate": 2.0519567041703083e-06,
      "log_odds_chosen": 0.9427490234375,
      "log_odds_ratio": -0.49150389432907104,
      "logits/chosen": -1.041406273841858,
      "logits/rejected": -0.9345703125,
      "logps/chosen": -0.7076171636581421,
      "logps/rejected": -1.310937523841858,
      "loss": 1.0642,
      "nll_loss": 0.9994140863418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.060192108154296875,
      "rewards/rejected": -0.1309814453125,
      "step": 1520
    },
    {
      "epoch": 0.1114064149706921,
      "grad_norm": 1.5746253179960445,
      "learning_rate": 2.0452399702596544e-06,
      "log_odds_chosen": 1.3107421398162842,
      "log_odds_ratio": -0.45263671875,
      "logits/chosen": -1.0205078125,
      "logits/rejected": -0.9203125238418579,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.6863281726837158,
      "loss": 1.0531,
      "nll_loss": 1.031640648841858,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07138671725988388,
      "rewards/margins": 0.09716796875,
      "rewards/rejected": -0.16853027045726776,
      "step": 1530
    },
    {
      "epoch": 0.11213456147376852,
      "grad_norm": 1.6428800506670196,
      "learning_rate": 2.0385887657505017e-06,
      "log_odds_chosen": 0.9951171875,
      "log_odds_ratio": -0.5116211175918579,
      "logits/chosen": -0.987109363079071,
      "logits/rejected": -0.91015625,
      "logps/chosen": -0.701367199420929,
      "logps/rejected": -1.375,
      "loss": 1.049,
      "nll_loss": 0.955078125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07011719048023224,
      "rewards/margins": 0.06721191108226776,
      "rewards/rejected": -0.13728027045726776,
      "step": 1540
    },
    {
      "epoch": 0.11286270797684494,
      "grad_norm": 1.7743280649885278,
      "learning_rate": 2.032002032003048e-06,
      "log_odds_chosen": 1.0622069835662842,
      "log_odds_ratio": -0.4854980409145355,
      "logits/chosen": -1.019140601158142,
      "logits/rejected": -0.9365234375,
      "logps/chosen": -0.7261718511581421,
      "logps/rejected": -1.482031226158142,
      "loss": 1.0583,
      "nll_loss": 1.0095703601837158,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0726318359375,
      "rewards/margins": 0.075592041015625,
      "rewards/rejected": -0.14816895127296448,
      "step": 1550
    },
    {
      "epoch": 0.11359085447992136,
      "grad_norm": 1.6231324220953163,
      "learning_rate": 2.025478734167333e-06,
      "log_odds_chosen": 1.053613305091858,
      "log_odds_ratio": -0.4950195252895355,
      "logits/chosen": -0.993359386920929,
      "logits/rejected": -0.906054675579071,
      "logps/chosen": -0.745898425579071,
      "logps/rejected": -1.5128905773162842,
      "loss": 1.0373,
      "nll_loss": 1.0216796398162842,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07468261569738388,
      "rewards/margins": 0.07673492282629013,
      "rewards/rejected": -0.15141601860523224,
      "step": 1560
    },
    {
      "epoch": 0.11431900098299778,
      "grad_norm": 1.448829911711827,
      "learning_rate": 2.0190178605002747e-06,
      "log_odds_chosen": 1.182714819908142,
      "log_odds_ratio": -0.4615234434604645,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.9166015386581421,
      "logps/chosen": -0.7081054449081421,
      "logps/rejected": -1.539648413658142,
      "loss": 1.0537,
      "nll_loss": 0.97265625,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07088623195886612,
      "rewards/margins": 0.08308105170726776,
      "rewards/rejected": -0.15396729111671448,
      "step": 1570
    },
    {
      "epoch": 0.1150471474860742,
      "grad_norm": 1.8672995427915349,
      "learning_rate": 2.0126184217065104e-06,
      "log_odds_chosen": 1.3938477039337158,
      "log_odds_ratio": -0.428466796875,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -0.706250011920929,
      "logps/rejected": -1.7068359851837158,
      "loss": 1.0567,
      "nll_loss": 0.9996093511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07064209133386612,
      "rewards/margins": 0.09992065280675888,
      "rewards/rejected": -0.17062988877296448,
      "step": 1580
    },
    {
      "epoch": 0.11577529398915062,
      "grad_norm": 2.0174321197396736,
      "learning_rate": 2.0062794503020765e-06,
      "log_odds_chosen": 1.16357421875,
      "log_odds_ratio": -0.4569335877895355,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -0.689453125,
      "logps/rejected": -1.5466797351837158,
      "loss": 1.0499,
      "nll_loss": 1.036523461341858,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06893310695886612,
      "rewards/margins": 0.08577270805835724,
      "rewards/rejected": -0.1546630859375,
      "step": 1590
    },
    {
      "epoch": 0.11650344049222704,
      "grad_norm": 1.5577324609982013,
      "learning_rate": 2e-06,
      "log_odds_chosen": 1.169335961341858,
      "log_odds_ratio": -0.4536376893520355,
      "logits/chosen": -1.0144531726837158,
      "logits/rejected": -0.9261718988418579,
      "logps/chosen": -0.7152343988418579,
      "logps/rejected": -1.5710937976837158,
      "loss": 1.0785,
      "nll_loss": 1.0417969226837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07155761867761612,
      "rewards/margins": 0.0855712890625,
      "rewards/rejected": -0.15708008408546448,
      "step": 1600
    },
    {
      "epoch": 0.11723158699530345,
      "grad_norm": 1.6379848934154986,
      "learning_rate": 1.993779145116907e-06,
      "log_odds_chosen": 1.309179663658142,
      "log_odds_ratio": -0.42583006620407104,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.7095702886581421,
      "logps/rejected": -1.640625,
      "loss": 1.0324,
      "nll_loss": 0.9931640625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07093505561351776,
      "rewards/margins": 0.09320678561925888,
      "rewards/rejected": -0.1641845703125,
      "step": 1610
    },
    {
      "epoch": 0.11795973349837988,
      "grad_norm": 1.6026185532394994,
      "learning_rate": 1.987615979999813e-06,
      "log_odds_chosen": 1.145898461341858,
      "log_odds_ratio": -0.45966798067092896,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.7115234136581421,
      "logps/rejected": -1.527929663658142,
      "loss": 1.0287,
      "nll_loss": 0.9453125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07115478813648224,
      "rewards/margins": 0.08154602348804474,
      "rewards/rejected": -0.15263672173023224,
      "step": 1620
    },
    {
      "epoch": 0.11868788000145629,
      "grad_norm": 1.7969036712533162,
      "learning_rate": 1.9815096184722797e-06,
      "log_odds_chosen": 1.2506835460662842,
      "log_odds_ratio": -0.4867187440395355,
      "logits/chosen": -0.955078125,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.7259765863418579,
      "logps/rejected": -1.6574218273162842,
      "loss": 1.0561,
      "nll_loss": 0.9789062738418579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07258300483226776,
      "rewards/margins": 0.09311523288488388,
      "rewards/rejected": -0.16574707627296448,
      "step": 1630
    },
    {
      "epoch": 0.11941602650453272,
      "grad_norm": 1.5222598488761336,
      "learning_rate": 1.9754591932991793e-06,
      "log_odds_chosen": 1.148339867591858,
      "log_odds_ratio": -0.455322265625,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.8656250238418579,
      "logps/chosen": -0.7069336175918579,
      "logps/rejected": -1.485937476158142,
      "loss": 1.019,
      "nll_loss": 0.9462890625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07071533054113388,
      "rewards/margins": 0.077911376953125,
      "rewards/rejected": -0.14858397841453552,
      "step": 1640
    },
    {
      "epoch": 0.12014417300760913,
      "grad_norm": 1.909481026394746,
      "learning_rate": 1.9694638556693235e-06,
      "log_odds_chosen": 1.142968773841858,
      "log_odds_ratio": -0.4237304627895355,
      "logits/chosen": -0.983203113079071,
      "logits/rejected": -0.9037109613418579,
      "logps/chosen": -0.636914074420929,
      "logps/rejected": -1.407812476158142,
      "loss": 1.043,
      "nll_loss": 0.990429699420929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06368408352136612,
      "rewards/margins": 0.07710571587085724,
      "rewards/rejected": -0.14082030951976776,
      "step": 1650
    },
    {
      "epoch": 0.12087231951068556,
      "grad_norm": 1.544957152191981,
      "learning_rate": 1.963522774695264e-06,
      "log_odds_chosen": 1.455664038658142,
      "log_odds_ratio": -0.4110351502895355,
      "logits/chosen": -1.022070288658142,
      "logits/rejected": -0.94140625,
      "logps/chosen": -0.67919921875,
      "logps/rejected": -1.7333984375,
      "loss": 1.0146,
      "nll_loss": 0.9146484136581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.10528564453125,
      "rewards/rejected": -0.17329101264476776,
      "step": 1660
    },
    {
      "epoch": 0.12160046601376197,
      "grad_norm": 1.691361663087259,
      "learning_rate": 1.9576351369295853e-06,
      "log_odds_chosen": 1.160546898841858,
      "log_odds_ratio": -0.43828123807907104,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.931835949420929,
      "logps/chosen": -0.6924804449081421,
      "logps/rejected": -1.48046875,
      "loss": 1.0242,
      "nll_loss": 1.005859375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06927490234375,
      "rewards/margins": 0.0787353515625,
      "rewards/rejected": -0.1480712890625,
      "step": 1670
    },
    {
      "epoch": 0.1223286125168384,
      "grad_norm": 1.4603663949519994,
      "learning_rate": 1.951800145897066e-06,
      "log_odds_chosen": 1.5099608898162842,
      "log_odds_ratio": -0.412841796875,
      "logits/chosen": -1.0283203125,
      "logits/rejected": -0.9009765386581421,
      "logps/chosen": -0.7222656011581421,
      "logps/rejected": -1.8156249523162842,
      "loss": 1.0419,
      "nll_loss": 0.966601550579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07222900539636612,
      "rewards/margins": 0.109375,
      "rewards/rejected": -0.18154296278953552,
      "step": 1680
    },
    {
      "epoch": 0.12305675901991481,
      "grad_norm": 1.683093672443576,
      "learning_rate": 1.9460170216420797e-06,
      "log_odds_chosen": 1.371826171875,
      "log_odds_ratio": -0.4059081971645355,
      "logits/chosen": -1.0158202648162842,
      "logits/rejected": -0.9189453125,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.6408202648162842,
      "loss": 1.0129,
      "nll_loss": 0.955273449420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06650390475988388,
      "rewards/margins": 0.09738769382238388,
      "rewards/rejected": -0.16386719048023224,
      "step": 1690
    },
    {
      "epoch": 0.12378490552299122,
      "grad_norm": 1.61091241435004,
      "learning_rate": 1.9402850002906637e-06,
      "log_odds_chosen": 1.0524780750274658,
      "log_odds_ratio": -0.5194336175918579,
      "logits/chosen": -0.971875011920929,
      "logits/rejected": -0.901562511920929,
      "logps/chosen": -0.6923828125,
      "logps/rejected": -1.435156226158142,
      "loss": 1.0037,
      "nll_loss": 0.9681640863418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06923828274011612,
      "rewards/margins": 0.07434539496898651,
      "rewards/rejected": -0.14357909560203552,
      "step": 1700
    },
    {
      "epoch": 0.12451305202606765,
      "grad_norm": 1.7458372462703642,
      "learning_rate": 1.9346033336266974e-06,
      "log_odds_chosen": 0.9312499761581421,
      "log_odds_ratio": -0.4717773497104645,
      "logits/chosen": -1.0,
      "logits/rejected": -0.903124988079071,
      "logps/chosen": -0.678027331829071,
      "logps/rejected": -1.297265648841858,
      "loss": 1.0423,
      "nll_loss": 0.9349609613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06783447414636612,
      "rewards/margins": 0.06181640550494194,
      "rewards/rejected": -0.12971191108226776,
      "step": 1710
    },
    {
      "epoch": 0.12524119852914406,
      "grad_norm": 1.5806300632378767,
      "learning_rate": 1.9289712886816486e-06,
      "log_odds_chosen": 1.1687500476837158,
      "log_odds_ratio": -0.4324707090854645,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.908007800579071,
      "logps/chosen": -0.736523449420929,
      "logps/rejected": -1.581640601158142,
      "loss": 1.0356,
      "nll_loss": 1.010156273841858,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07365722954273224,
      "rewards/margins": 0.08444824069738388,
      "rewards/rejected": -0.15815429389476776,
      "step": 1720
    },
    {
      "epoch": 0.1259693450322205,
      "grad_norm": 1.7187327408762862,
      "learning_rate": 1.92338814733738e-06,
      "log_odds_chosen": 1.0644042491912842,
      "log_odds_ratio": -0.46147459745407104,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.921679675579071,
      "logps/chosen": -0.7040039300918579,
      "logps/rejected": -1.406835913658142,
      "loss": 1.0049,
      "nll_loss": 0.957226574420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07036133110523224,
      "rewards/margins": 0.0702667236328125,
      "rewards/rejected": -0.14067383110523224,
      "step": 1730
    },
    {
      "epoch": 0.1266974915352969,
      "grad_norm": 1.3964795279251796,
      "learning_rate": 1.9178532059415367e-06,
      "log_odds_chosen": 0.9427734613418579,
      "log_odds_ratio": -0.508349597454071,
      "logits/chosen": -1.0361328125,
      "logits/rejected": -0.932812511920929,
      "logps/chosen": -0.7269531488418579,
      "logps/rejected": -1.367773413658142,
      "loss": 1.0304,
      "nll_loss": 0.917773425579071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07272949069738388,
      "rewards/margins": 0.06398315727710724,
      "rewards/rejected": -0.13666991889476776,
      "step": 1740
    },
    {
      "epoch": 0.1274256380383733,
      "grad_norm": 1.5914585831593873,
      "learning_rate": 1.9123657749350298e-06,
      "log_odds_chosen": 1.140905737876892,
      "log_odds_ratio": -0.4703125059604645,
      "logits/chosen": -0.998828113079071,
      "logits/rejected": -0.9322265386581421,
      "logps/chosen": -0.7279297113418579,
      "logps/rejected": -1.536523461341858,
      "loss": 1.029,
      "nll_loss": 1.011132836341858,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07285156100988388,
      "rewards/margins": 0.08081360161304474,
      "rewards/rejected": -0.15358886122703552,
      "step": 1750
    },
    {
      "epoch": 0.12815378454144974,
      "grad_norm": 1.4478596476264443,
      "learning_rate": 1.9069251784911844e-06,
      "log_odds_chosen": 1.255761742591858,
      "log_odds_ratio": -0.43095701932907104,
      "logits/chosen": -0.9482421875,
      "logits/rejected": -0.830273449420929,
      "logps/chosen": -0.68359375,
      "logps/rejected": -1.570703148841858,
      "loss": 1.0189,
      "nll_loss": 1.0222656726837158,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.08867187798023224,
      "rewards/rejected": -0.15700682997703552,
      "step": 1760
    },
    {
      "epoch": 0.12888193104452617,
      "grad_norm": 1.9255632739901059,
      "learning_rate": 1.9015307541661132e-06,
      "log_odds_chosen": 1.0532715320587158,
      "log_odds_ratio": -0.5038086175918579,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.918749988079071,
      "logps/chosen": -0.7562500238418579,
      "logps/rejected": -1.5353515148162842,
      "loss": 1.0406,
      "nll_loss": 0.999804675579071,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07567138969898224,
      "rewards/margins": 0.07784118503332138,
      "rewards/rejected": -0.15336914360523224,
      "step": 1770
    },
    {
      "epoch": 0.12961007754760256,
      "grad_norm": 1.629459324851094,
      "learning_rate": 1.8961818525599089e-06,
      "log_odds_chosen": 1.028100609779358,
      "log_odds_ratio": -0.486328125,
      "logits/chosen": -0.981640636920929,
      "logits/rejected": -0.9017578363418579,
      "logps/chosen": -0.726757824420929,
      "logps/rejected": -1.451171875,
      "loss": 1.0198,
      "nll_loss": 0.9769531488418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07271728664636612,
      "rewards/margins": 0.07234954833984375,
      "rewards/rejected": -0.1451416015625,
      "step": 1780
    },
    {
      "epoch": 0.130338224050679,
      "grad_norm": 1.6428303502174528,
      "learning_rate": 1.890877836988262e-06,
      "log_odds_chosen": 1.18310546875,
      "log_odds_ratio": -0.4529785215854645,
      "logits/chosen": -1.0041015148162842,
      "logits/rejected": -0.88671875,
      "logps/chosen": -0.7494140863418579,
      "logps/rejected": -1.598046898841858,
      "loss": 1.0324,
      "nll_loss": 1.023828148841858,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07490234076976776,
      "rewards/margins": 0.08489990234375,
      "rewards/rejected": -0.15993651747703552,
      "step": 1790
    },
    {
      "epoch": 0.13106637055375542,
      "grad_norm": 1.6414899541327086,
      "learning_rate": 1.8856180831641269e-06,
      "log_odds_chosen": 1.225976586341858,
      "log_odds_ratio": -0.4285644590854645,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -0.70166015625,
      "logps/rejected": -1.544531226158142,
      "loss": 1.0124,
      "nll_loss": 0.995898425579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07015381008386612,
      "rewards/margins": 0.08428344875574112,
      "rewards/rejected": -0.1544189453125,
      "step": 1800
    },
    {
      "epoch": 0.13179451705683184,
      "grad_norm": 2.1200877644282374,
      "learning_rate": 1.8804019788890737e-06,
      "log_odds_chosen": 1.094824194908142,
      "log_odds_ratio": -0.48066407442092896,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.735058605670929,
      "logps/rejected": -1.474218726158142,
      "loss": 1.0347,
      "nll_loss": 0.993945300579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07359619438648224,
      "rewards/margins": 0.07393188774585724,
      "rewards/rejected": -0.14743652939796448,
      "step": 1810
    },
    {
      "epoch": 0.13252266355990824,
      "grad_norm": 1.5932154131897984,
      "learning_rate": 1.8752289237539816e-06,
      "log_odds_chosen": 1.0927734375,
      "log_odds_ratio": -0.4952148497104645,
      "logits/chosen": -0.951953113079071,
      "logits/rejected": -0.9048827886581421,
      "logps/chosen": -0.7237304449081421,
      "logps/rejected": -1.471093773841858,
      "loss": 1.0375,
      "nll_loss": 1.0300781726837158,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.07235107570886612,
      "rewards/margins": 0.07469825446605682,
      "rewards/rejected": -0.14699706435203552,
      "step": 1820
    },
    {
      "epoch": 0.13325081006298467,
      "grad_norm": 1.6108589071788282,
      "learning_rate": 1.8700983288487376e-06,
      "log_odds_chosen": 0.8958984613418579,
      "log_odds_ratio": -0.540234386920929,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.7074218988418579,
      "logps/rejected": -1.3308594226837158,
      "loss": 1.0417,
      "nll_loss": 0.991992175579071,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07075195014476776,
      "rewards/margins": 0.06231689453125,
      "rewards/rejected": -0.13303223252296448,
      "step": 1830
    },
    {
      "epoch": 0.1339789565660611,
      "grad_norm": 1.7213276876637547,
      "learning_rate": 1.8650096164806275e-06,
      "log_odds_chosen": 1.28369140625,
      "log_odds_ratio": -0.42241209745407104,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -0.673144519329071,
      "logps/rejected": -1.552734375,
      "loss": 1.0317,
      "nll_loss": 0.96484375,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06730957329273224,
      "rewards/margins": 0.08797607570886612,
      "rewards/rejected": -0.15532226860523224,
      "step": 1840
    },
    {
      "epoch": 0.13470710306913752,
      "grad_norm": 1.6130667842465651,
      "learning_rate": 1.8599622199011084e-06,
      "log_odds_chosen": 0.9668945074081421,
      "log_odds_ratio": -0.4765625,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.8548828363418579,
      "logps/chosen": -0.694140613079071,
      "logps/rejected": -1.3259766101837158,
      "loss": 1.0477,
      "nll_loss": 0.984375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.06313476711511612,
      "rewards/rejected": -0.1324462890625,
      "step": 1850
    },
    {
      "epoch": 0.13543524957221392,
      "grad_norm": 1.6609879194448833,
      "learning_rate": 1.854955583040673e-06,
      "log_odds_chosen": 1.163476586341858,
      "log_odds_ratio": -0.46025389432907104,
      "logits/chosen": -0.952343761920929,
      "logits/rejected": -0.837695300579071,
      "logps/chosen": -0.6958984136581421,
      "logps/rejected": -1.5177733898162842,
      "loss": 1.0413,
      "nll_loss": 0.9771484136581421,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.08206786960363388,
      "rewards/rejected": -0.1517333984375,
      "step": 1860
    },
    {
      "epoch": 0.13616339607529035,
      "grad_norm": 1.7462902977892767,
      "learning_rate": 1.849989160251521e-06,
      "log_odds_chosen": 1.3381836414337158,
      "log_odds_ratio": -0.429443359375,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.8511718511581421,
      "logps/chosen": -0.693652331829071,
      "logps/rejected": -1.626953125,
      "loss": 1.0398,
      "nll_loss": 0.9765625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.09348144382238388,
      "rewards/rejected": -0.16276855766773224,
      "step": 1870
    },
    {
      "epoch": 0.13689154257836678,
      "grad_norm": 1.8769820126274868,
      "learning_rate": 1.8450624160577701e-06,
      "log_odds_chosen": 1.0939452648162842,
      "log_odds_ratio": -0.47309571504592896,
      "logits/chosen": -0.9486328363418579,
      "logits/rejected": -0.8666015863418579,
      "logps/chosen": -0.7012695074081421,
      "logps/rejected": -1.4523437023162842,
      "loss": 1.0159,
      "nll_loss": 0.974609375,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07015381008386612,
      "rewards/margins": 0.075103759765625,
      "rewards/rejected": -0.14521484076976776,
      "step": 1880
    },
    {
      "epoch": 0.13761968908144318,
      "grad_norm": 1.9544043745370518,
      "learning_rate": 1.8401748249129445e-06,
      "log_odds_chosen": 1.1393554210662842,
      "log_odds_ratio": -0.4766601622104645,
      "logits/chosen": -1.0300781726837158,
      "logits/rejected": -0.9410156011581421,
      "logps/chosen": -0.6929687261581421,
      "logps/rejected": -1.5060546398162842,
      "loss": 1.0281,
      "nll_loss": 0.872265636920929,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06923828274011612,
      "rewards/margins": 0.081298828125,
      "rewards/rejected": -0.15065917372703552,
      "step": 1890
    },
    {
      "epoch": 0.1383478355845196,
      "grad_norm": 2.4515750208575118,
      "learning_rate": 1.8353258709644938e-06,
      "log_odds_chosen": 1.2607421875,
      "log_odds_ratio": -0.4656738340854645,
      "logits/chosen": -0.977734386920929,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.734570324420929,
      "logps/rejected": -1.689843773841858,
      "loss": 1.0297,
      "nll_loss": 0.941601574420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07338867336511612,
      "rewards/margins": 0.09562377631664276,
      "rewards/rejected": -0.16896972060203552,
      "step": 1900
    },
    {
      "epoch": 0.13907598208759603,
      "grad_norm": 1.6034938189999806,
      "learning_rate": 1.830515047825102e-06,
      "log_odds_chosen": 1.1419188976287842,
      "log_odds_ratio": -0.41582030057907104,
      "logits/chosen": -0.9720703363418579,
      "logits/rejected": -0.904101550579071,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.480078101158142,
      "loss": 1.0239,
      "nll_loss": 0.9410156011581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07072754204273224,
      "rewards/margins": 0.07727813720703125,
      "rewards/rejected": -0.14799804985523224,
      "step": 1910
    },
    {
      "epoch": 0.13980412859067246,
      "grad_norm": 2.8715977067876706,
      "learning_rate": 1.8257418583505536e-06,
      "log_odds_chosen": 1.2405273914337158,
      "log_odds_ratio": -0.431640625,
      "logits/chosen": -0.972851574420929,
      "logits/rejected": -0.873242199420929,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.533593773841858,
      "loss": 1.0203,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.08740539848804474,
      "rewards/rejected": -0.15327148139476776,
      "step": 1920
    },
    {
      "epoch": 0.14053227509374885,
      "grad_norm": 1.6102585892198653,
      "learning_rate": 1.8210058144239416e-06,
      "log_odds_chosen": 1.2312500476837158,
      "log_odds_ratio": -0.4256347715854645,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.9007812738418579,
      "logps/chosen": -0.7142578363418579,
      "logps/rejected": -1.5652344226837158,
      "loss": 1.0311,
      "nll_loss": 1.0158202648162842,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07138671725988388,
      "rewards/margins": 0.08516235649585724,
      "rewards/rejected": -0.15656737983226776,
      "step": 1930
    },
    {
      "epoch": 0.14126042159682528,
      "grad_norm": 2.4298585873916694,
      "learning_rate": 1.816306436745999e-06,
      "log_odds_chosen": 1.2435791492462158,
      "log_odds_ratio": -0.43940430879592896,
      "logits/chosen": -1.019921898841858,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -1.5244140625,
      "loss": 1.0506,
      "nll_loss": 0.9794921875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06746826320886612,
      "rewards/margins": 0.08503799140453339,
      "rewards/rejected": -0.15249022841453552,
      "step": 1940
    },
    {
      "epoch": 0.1419885680999017,
      "grad_norm": 1.9332931824159094,
      "learning_rate": 1.8116432546313529e-06,
      "log_odds_chosen": 1.0442383289337158,
      "log_odds_ratio": -0.5205078125,
      "logits/chosen": -0.969921886920929,
      "logits/rejected": -0.8919922113418579,
      "logps/chosen": -0.704785168170929,
      "logps/rejected": -1.465234398841858,
      "loss": 1.0173,
      "nll_loss": 0.973437488079071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.07054443657398224,
      "rewards/margins": 0.07599182426929474,
      "rewards/rejected": -0.14654541015625,
      "step": 1950
    },
    {
      "epoch": 0.1427167146029781,
      "grad_norm": 1.6274816620966024,
      "learning_rate": 1.8070158058105026e-06,
      "log_odds_chosen": 1.350683569908142,
      "log_odds_ratio": -0.4427246153354645,
      "logits/chosen": -1.038671851158142,
      "logits/rejected": -0.9292968511581421,
      "logps/chosen": -0.7430664300918579,
      "logps/rejected": -1.698828101158142,
      "loss": 1.0312,
      "nll_loss": 1.0558593273162842,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07437743991613388,
      "rewards/margins": 0.09544067084789276,
      "rewards/rejected": -0.16987304389476776,
      "step": 1960
    },
    {
      "epoch": 0.14344486110605453,
      "grad_norm": 1.7299797054804598,
      "learning_rate": 1.8024236362373315e-06,
      "log_odds_chosen": 1.3367187976837158,
      "log_odds_ratio": -0.441650390625,
      "logits/chosen": -1.071874976158142,
      "logits/rejected": -0.971484363079071,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.6267578601837158,
      "loss": 0.9921,
      "nll_loss": 0.890429675579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07069091498851776,
      "rewards/margins": 0.09212646633386612,
      "rewards/rejected": -0.16262206435203552,
      "step": 1970
    },
    {
      "epoch": 0.14417300760913096,
      "grad_norm": 1.69885124566666,
      "learning_rate": 1.7978662999019787e-06,
      "log_odds_chosen": 1.156640648841858,
      "log_odds_ratio": -0.42802733182907104,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.8677734136581421,
      "logps/chosen": -0.68310546875,
      "logps/rejected": -1.4865233898162842,
      "loss": 1.0165,
      "nll_loss": 0.932421863079071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06826172024011612,
      "rewards/margins": 0.08034668117761612,
      "rewards/rejected": -0.14858397841453552,
      "step": 1980
    },
    {
      "epoch": 0.1449011541122074,
      "grad_norm": 1.800291145787404,
      "learning_rate": 1.793343358648881e-06,
      "log_odds_chosen": 1.2516601085662842,
      "log_odds_ratio": -0.4446777403354645,
      "logits/chosen": -0.9820312261581421,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.716601550579071,
      "logps/rejected": -1.611328125,
      "loss": 1.0027,
      "nll_loss": 1.0251953601837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07163085788488388,
      "rewards/margins": 0.089508056640625,
      "rewards/rejected": -0.16120605170726776,
      "step": 1990
    },
    {
      "epoch": 0.14562930061528379,
      "grad_norm": 1.550129809567034,
      "learning_rate": 1.7888543819998317e-06,
      "log_odds_chosen": 1.2411620616912842,
      "log_odds_ratio": -0.45341795682907104,
      "logits/chosen": -1.0099608898162842,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -1.540429711341858,
      "loss": 1.0294,
      "nll_loss": 0.9212890863418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.08555908501148224,
      "rewards/rejected": -0.15400390326976776,
      "step": 2000
    },
    {
      "epoch": 0.1463574471183602,
      "grad_norm": 1.6381828358597794,
      "learning_rate": 1.7843989469818819e-06,
      "log_odds_chosen": 1.3162109851837158,
      "log_odds_ratio": -0.3995117247104645,
      "logits/chosen": -0.950390636920929,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -0.6778320074081421,
      "logps/rejected": -1.581640601158142,
      "loss": 1.0136,
      "nll_loss": 0.9765625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.09033203125,
      "rewards/rejected": -0.15805664658546448,
      "step": 2010
    },
    {
      "epoch": 0.14708559362143664,
      "grad_norm": 1.746531136512054,
      "learning_rate": 1.779976637959939e-06,
      "log_odds_chosen": 1.3447265625,
      "log_odds_ratio": -0.384033203125,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -0.675585925579071,
      "logps/rejected": -1.6085937023162842,
      "loss": 1.0072,
      "nll_loss": 0.931640625,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06757812201976776,
      "rewards/margins": 0.09316406399011612,
      "rewards/rejected": -0.16081543266773224,
      "step": 2020
    },
    {
      "epoch": 0.14781374012451307,
      "grad_norm": 1.8047805973762499,
      "learning_rate": 1.7755870464739012e-06,
      "log_odds_chosen": 0.8568359613418579,
      "log_odds_ratio": -0.551464855670929,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.7124999761581421,
      "logps/rejected": -1.273828148841858,
      "loss": 1.0163,
      "nll_loss": 1.0380859375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07120361179113388,
      "rewards/margins": 0.05621948093175888,
      "rewards/rejected": -0.12734374403953552,
      "step": 2030
    },
    {
      "epoch": 0.14854188662758946,
      "grad_norm": 1.749005326222433,
      "learning_rate": 1.7712297710801907e-06,
      "log_odds_chosen": 1.0390746593475342,
      "log_odds_ratio": -0.4970703125,
      "logits/chosen": -1.0095703601837158,
      "logits/rejected": -0.9261718988418579,
      "logps/chosen": -0.7406250238418579,
      "logps/rejected": -1.4939453601837158,
      "loss": 1.0249,
      "nll_loss": 1.000585913658142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07404784858226776,
      "rewards/margins": 0.075225830078125,
      "rewards/rejected": -0.14921875298023224,
      "step": 2040
    },
    {
      "epoch": 0.1492700331306659,
      "grad_norm": 1.7189210971520612,
      "learning_rate": 1.7669044171975444e-06,
      "log_odds_chosen": 1.1669921875,
      "log_odds_ratio": -0.459716796875,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.9287109375,
      "logps/chosen": -0.713085949420929,
      "logps/rejected": -1.533593773841858,
      "loss": 1.0038,
      "nll_loss": 0.9828125238418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0712890625,
      "rewards/margins": 0.08214111626148224,
      "rewards/rejected": -0.15346679091453552,
      "step": 2050
    },
    {
      "epoch": 0.14999817963374232,
      "grad_norm": 1.71758005597113,
      "learning_rate": 1.7626105969569268e-06,
      "log_odds_chosen": 1.257568359375,
      "log_odds_ratio": -0.41083985567092896,
      "logits/chosen": -1.020898461341858,
      "logits/rejected": -0.8990234136581421,
      "logps/chosen": -0.661328136920929,
      "logps/rejected": -1.5107421875,
      "loss": 1.0069,
      "nll_loss": 0.979296863079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.08494873344898224,
      "rewards/rejected": -0.1510009765625,
      "step": 2060
    },
    {
      "epoch": 0.15072632613681872,
      "grad_norm": 1.7130965267383629,
      "learning_rate": 1.758347929055432e-06,
      "log_odds_chosen": 1.084570288658142,
      "log_odds_ratio": -0.48554688692092896,
      "logits/chosen": -1.0646483898162842,
      "logits/rejected": -0.9613281488418579,
      "logps/chosen": -0.718945324420929,
      "logps/rejected": -1.4775390625,
      "loss": 1.0217,
      "nll_loss": 0.9417968988418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07188721001148224,
      "rewards/margins": 0.07598266750574112,
      "rewards/rejected": -0.14785155653953552,
      "step": 2070
    },
    {
      "epoch": 0.15145447263989514,
      "grad_norm": 1.8344799461563874,
      "learning_rate": 1.7541160386140582e-06,
      "log_odds_chosen": 1.0607421398162842,
      "log_odds_ratio": -0.4691406190395355,
      "logits/chosen": -1.025390625,
      "logits/rejected": -0.9564453363418579,
      "logps/chosen": -0.7255859375,
      "logps/rejected": -1.472265601158142,
      "loss": 1.0015,
      "nll_loss": 0.9560546875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07259521633386612,
      "rewards/margins": 0.07469482719898224,
      "rewards/rejected": -0.14731445908546448,
      "step": 2080
    },
    {
      "epoch": 0.15218261914297157,
      "grad_norm": 1.539438641009839,
      "learning_rate": 1.7499145570392284e-06,
      "log_odds_chosen": 1.2386963367462158,
      "log_odds_ratio": -0.44999998807907104,
      "logits/chosen": -1.041406273841858,
      "logits/rejected": -0.9437500238418579,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.609765648841858,
      "loss": 1.0223,
      "nll_loss": 0.9937499761581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.072265625,
      "rewards/margins": 0.08864593505859375,
      "rewards/rejected": -0.16083984076976776,
      "step": 2090
    },
    {
      "epoch": 0.152910765646048,
      "grad_norm": 2.0451787667388235,
      "learning_rate": 1.745743121887939e-06,
      "log_odds_chosen": 1.3621094226837158,
      "log_odds_ratio": -0.43037110567092896,
      "logits/chosen": -1.0498046875,
      "logits/rejected": -0.9457031488418579,
      "logps/chosen": -0.6919921636581421,
      "logps/rejected": -1.65625,
      "loss": 1.0299,
      "nll_loss": 0.997265636920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06912841647863388,
      "rewards/margins": 0.09648437798023224,
      "rewards/rejected": -0.16562500596046448,
      "step": 2100
    },
    {
      "epoch": 0.1536389121491244,
      "grad_norm": 1.5874290564085507,
      "learning_rate": 1.7416013767364324e-06,
      "log_odds_chosen": 1.393457055091858,
      "log_odds_ratio": -0.41093748807907104,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.921093761920929,
      "logps/chosen": -0.6705077886581421,
      "logps/rejected": -1.6492187976837158,
      "loss": 1.0175,
      "nll_loss": 0.943164050579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06704101711511612,
      "rewards/margins": 0.097869873046875,
      "rewards/rejected": -0.16496582329273224,
      "step": 2110
    },
    {
      "epoch": 0.15436705865220082,
      "grad_norm": 1.6631761854334697,
      "learning_rate": 1.7374889710522776e-06,
      "log_odds_chosen": 0.9184204339981079,
      "log_odds_ratio": -0.533984363079071,
      "logits/chosen": -0.9873046875,
      "logits/rejected": -0.900585949420929,
      "logps/chosen": -0.724804699420929,
      "logps/rejected": -1.4033203125,
      "loss": 0.9923,
      "nll_loss": 1.0046875476837158,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.072509765625,
      "rewards/margins": 0.06780395656824112,
      "rewards/rejected": -0.140380859375,
      "step": 2120
    },
    {
      "epoch": 0.15509520515527725,
      "grad_norm": 1.6789811101008383,
      "learning_rate": 1.7334055600697579e-06,
      "log_odds_chosen": 1.1096680164337158,
      "log_odds_ratio": -0.47880858182907104,
      "logits/chosen": -0.9761718511581421,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -0.715039074420929,
      "logps/rejected": -1.5193359851837158,
      "loss": 0.9945,
      "nll_loss": 0.962695300579071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07147216796875,
      "rewards/margins": 0.08041992038488388,
      "rewards/rejected": -0.15190429985523224,
      "step": 2130
    },
    {
      "epoch": 0.15582335165835365,
      "grad_norm": 1.6094773818234376,
      "learning_rate": 1.7293508046684678e-06,
      "log_odds_chosen": 1.352929711341858,
      "log_odds_ratio": -0.443115234375,
      "logits/chosen": -1.0148437023162842,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.72265625,
      "logps/rejected": -1.721289038658142,
      "loss": 1.0064,
      "nll_loss": 0.9701172113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07231445610523224,
      "rewards/margins": 0.09984131157398224,
      "rewards/rejected": -0.17219237983226776,
      "step": 2140
    },
    {
      "epoch": 0.15655149816143007,
      "grad_norm": 1.5949788466112023,
      "learning_rate": 1.7253243712550145e-06,
      "log_odds_chosen": 1.190820336341858,
      "log_odds_ratio": -0.44501954317092896,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.929492175579071,
      "logps/chosen": -0.670703113079071,
      "logps/rejected": -1.494531273841858,
      "loss": 0.9975,
      "nll_loss": 0.910937488079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06711425632238388,
      "rewards/margins": 0.08246765285730362,
      "rewards/rejected": -0.1495361328125,
      "step": 2150
    },
    {
      "epoch": 0.1572796446645065,
      "grad_norm": 1.6678535049326102,
      "learning_rate": 1.7213259316477406e-06,
      "log_odds_chosen": 1.051123023033142,
      "log_odds_ratio": -0.45195311307907104,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.9521484375,
      "logps/chosen": -0.668652355670929,
      "logps/rejected": -1.368554711341858,
      "loss": 0.9923,
      "nll_loss": 0.932812511920929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06688232719898224,
      "rewards/margins": 0.069915771484375,
      "rewards/rejected": -0.13691405951976776,
      "step": 2160
    },
    {
      "epoch": 0.15800779116758293,
      "grad_norm": 1.9545158765252604,
      "learning_rate": 1.7173551629643674e-06,
      "log_odds_chosen": 1.1601073741912842,
      "log_odds_ratio": -0.44755858182907104,
      "logits/chosen": -1.0255858898162842,
      "logits/rejected": -0.9501953125,
      "logps/chosen": -0.719531238079071,
      "logps/rejected": -1.5519530773162842,
      "loss": 1.0346,
      "nll_loss": 1.0048828125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.07194824516773224,
      "rewards/margins": 0.0833740234375,
      "rewards/rejected": -0.15529784560203552,
      "step": 2170
    },
    {
      "epoch": 0.15873593767065933,
      "grad_norm": 1.979180439485981,
      "learning_rate": 1.713411747512477e-06,
      "log_odds_chosen": 1.4584228992462158,
      "log_odds_ratio": -0.40791016817092896,
      "logits/chosen": -0.9716796875,
      "logits/rejected": -0.8960937261581421,
      "logps/chosen": -0.7134765386581421,
      "logps/rejected": -1.748437523841858,
      "loss": 0.9882,
      "nll_loss": 0.94482421875,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0714111328125,
      "rewards/margins": 0.10352478176355362,
      "rewards/rejected": -0.17496338486671448,
      "step": 2180
    },
    {
      "epoch": 0.15946408417373575,
      "grad_norm": 1.5273491940559047,
      "learning_rate": 1.709495372682753e-06,
      "log_odds_chosen": 1.4318358898162842,
      "log_odds_ratio": -0.38471680879592896,
      "logits/chosen": -1.0595703125,
      "logits/rejected": -0.946093738079071,
      "logps/chosen": -0.740429699420929,
      "logps/rejected": -1.803125023841858,
      "loss": 1.005,
      "nll_loss": 0.9310547113418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07410888373851776,
      "rewards/margins": 0.10616455227136612,
      "rewards/rejected": -0.18010254204273224,
      "step": 2190
    },
    {
      "epoch": 0.16019223067681218,
      "grad_norm": 1.711683710044129,
      "learning_rate": 1.7056057308448832e-06,
      "log_odds_chosen": 1.303808569908142,
      "log_odds_ratio": -0.42353516817092896,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.911914050579071,
      "logps/chosen": -0.651171863079071,
      "logps/rejected": -1.580078125,
      "loss": 0.9733,
      "nll_loss": 0.8873046636581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.065185546875,
      "rewards/margins": 0.09279175102710724,
      "rewards/rejected": -0.15798339247703552,
      "step": 2200
    },
    {
      "epoch": 0.1609203771798886,
      "grad_norm": 1.9284877838544199,
      "learning_rate": 1.701742519246068e-06,
      "log_odds_chosen": 1.0916016101837158,
      "log_odds_ratio": -0.469482421875,
      "logits/chosen": -1.009179711341858,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -0.714648425579071,
      "logps/rejected": -1.480078101158142,
      "loss": 0.9963,
      "nll_loss": 0.9859374761581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07148437201976776,
      "rewards/margins": 0.07654418796300888,
      "rewards/rejected": -0.14804688096046448,
      "step": 2210
    },
    {
      "epoch": 0.161648523682965,
      "grad_norm": 1.6677132302523026,
      "learning_rate": 1.6979054399120355e-06,
      "log_odds_chosen": 1.203222632408142,
      "log_odds_ratio": -0.412353515625,
      "logits/chosen": -0.998242199420929,
      "logits/rejected": -0.8988281488418579,
      "logps/chosen": -0.6796875,
      "logps/rejected": -1.4855468273162842,
      "loss": 1.0306,
      "nll_loss": 0.9644531011581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.0679931640625,
      "rewards/margins": 0.08062133938074112,
      "rewards/rejected": -0.14865723252296448,
      "step": 2220
    },
    {
      "epoch": 0.16237667018604143,
      "grad_norm": 1.5347502390545833,
      "learning_rate": 1.6940941995505069e-06,
      "log_odds_chosen": 1.1356933116912842,
      "log_odds_ratio": -0.4703125059604645,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.8759765625,
      "logps/chosen": -0.7196289300918579,
      "logps/rejected": -1.5158202648162842,
      "loss": 0.9893,
      "nll_loss": 0.92138671875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07192382961511612,
      "rewards/margins": 0.07964172214269638,
      "rewards/rejected": -0.15166015923023224,
      "step": 2230
    },
    {
      "epoch": 0.16310481668911786,
      "grad_norm": 1.6299746914295434,
      "learning_rate": 1.6903085094570331e-06,
      "log_odds_chosen": 1.1393554210662842,
      "log_odds_ratio": -0.4344726502895355,
      "logits/chosen": -0.966796875,
      "logits/rejected": -0.855273425579071,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -1.4609375,
      "loss": 0.9922,
      "nll_loss": 0.98828125,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06676025688648224,
      "rewards/margins": 0.07943115383386612,
      "rewards/rejected": -0.14603272080421448,
      "step": 2240
    },
    {
      "epoch": 0.16383296319219426,
      "grad_norm": 1.7449231003566283,
      "learning_rate": 1.6865480854231356e-06,
      "log_odds_chosen": 1.25592041015625,
      "log_odds_ratio": -0.4454589784145355,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.861132800579071,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -1.549414038658142,
      "loss": 0.9795,
      "nll_loss": 0.9642578363418579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.08760986477136612,
      "rewards/rejected": -0.15495605766773224,
      "step": 2250
    },
    {
      "epoch": 0.16456110969527069,
      "grad_norm": 1.7002753047029264,
      "learning_rate": 1.682812647646685e-06,
      "log_odds_chosen": 1.313378930091858,
      "log_odds_ratio": -0.4354003965854645,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.8310546875,
      "logps/chosen": -0.67626953125,
      "logps/rejected": -1.6375000476837158,
      "loss": 1.004,
      "nll_loss": 0.909960925579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.067626953125,
      "rewards/margins": 0.09614257514476776,
      "rewards/rejected": -0.16376952826976776,
      "step": 2260
    },
    {
      "epoch": 0.1652892561983471,
      "grad_norm": 2.0232683933890776,
      "learning_rate": 1.6791019206444541e-06,
      "log_odds_chosen": 1.073339819908142,
      "log_odds_ratio": -0.4795898497104645,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.478124976158142,
      "loss": 0.9939,
      "nll_loss": 0.927539050579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07056884467601776,
      "rewards/margins": 0.07724609225988388,
      "rewards/rejected": -0.14790038764476776,
      "step": 2270
    },
    {
      "epoch": 0.16601740270142354,
      "grad_norm": 1.9798155265204258,
      "learning_rate": 1.675415633166782e-06,
      "log_odds_chosen": 1.2253906726837158,
      "log_odds_ratio": -0.45561522245407104,
      "logits/chosen": -0.9566406011581421,
      "logits/rejected": -0.844531238079071,
      "logps/chosen": -0.6962890625,
      "logps/rejected": -1.544335961341858,
      "loss": 1.0055,
      "nll_loss": 0.9769531488418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.08475951850414276,
      "rewards/rejected": -0.15424804389476776,
      "step": 2280
    },
    {
      "epoch": 0.16674554920449994,
      "grad_norm": 1.7011885763722043,
      "learning_rate": 1.6717535181142914e-06,
      "log_odds_chosen": 1.075585961341858,
      "log_odds_ratio": -0.46967774629592896,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.884765625,
      "logps/chosen": -0.696093738079071,
      "logps/rejected": -1.4347655773162842,
      "loss": 0.9983,
      "nll_loss": 0.9501953125,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06960449367761612,
      "rewards/margins": 0.07398681342601776,
      "rewards/rejected": -0.14365234971046448,
      "step": 2290
    },
    {
      "epoch": 0.16747369570757636,
      "grad_norm": 1.7367949626065802,
      "learning_rate": 1.668115312456598e-06,
      "log_odds_chosen": 1.261816382408142,
      "log_odds_ratio": -0.4483398497104645,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.841503918170929,
      "logps/chosen": -0.728222668170929,
      "logps/rejected": -1.638085961341858,
      "loss": 1.0066,
      "nll_loss": 1.013085961341858,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07285156100988388,
      "rewards/margins": 0.09094543755054474,
      "rewards/rejected": -0.163818359375,
      "step": 2300
    },
    {
      "epoch": 0.1682018422106528,
      "grad_norm": 1.7625586947575886,
      "learning_rate": 1.6645007571529578e-06,
      "log_odds_chosen": 1.155371069908142,
      "log_odds_ratio": -0.4588867127895355,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.8167968988418579,
      "logps/chosen": -0.685839831829071,
      "logps/rejected": -1.5128905773162842,
      "loss": 0.9724,
      "nll_loss": 0.880664050579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06859131157398224,
      "rewards/margins": 0.08280029147863388,
      "rewards/rejected": -0.15139159560203552,
      "step": 2310
    },
    {
      "epoch": 0.1689299887137292,
      "grad_norm": 1.8747264036819244,
      "learning_rate": 1.6609095970747992e-06,
      "log_odds_chosen": 1.260351538658142,
      "log_odds_ratio": -0.42768555879592896,
      "logits/chosen": -0.9390624761581421,
      "logits/rejected": -0.8550781011581421,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.5261719226837158,
      "loss": 0.9614,
      "nll_loss": 0.926562488079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06585693359375,
      "rewards/margins": 0.08670654147863388,
      "rewards/rejected": -0.15256348252296448,
      "step": 2320
    },
    {
      "epoch": 0.16965813521680562,
      "grad_norm": 1.7836679926663312,
      "learning_rate": 1.6573415809300833e-06,
      "log_odds_chosen": 1.0992310047149658,
      "log_odds_ratio": -0.4937500059604645,
      "logits/chosen": -0.962109386920929,
      "logits/rejected": -0.906054675579071,
      "logps/chosen": -0.730761706829071,
      "logps/rejected": -1.507421851158142,
      "loss": 0.9864,
      "nll_loss": 0.9111328125,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07304687798023224,
      "rewards/margins": 0.07770691066980362,
      "rewards/rejected": -0.15083007514476776,
      "step": 2330
    },
    {
      "epoch": 0.17038628171988204,
      "grad_norm": 1.6412883442094568,
      "learning_rate": 1.6537964611894462e-06,
      "log_odds_chosen": 1.260644555091858,
      "log_odds_ratio": -0.4375976622104645,
      "logits/chosen": -0.8960937261581421,
      "logits/rejected": -0.819140613079071,
      "logps/chosen": -0.711132824420929,
      "logps/rejected": -1.640625,
      "loss": 0.997,
      "nll_loss": 0.9283202886581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07108154147863388,
      "rewards/margins": 0.09306640923023224,
      "rewards/rejected": -0.16408690810203552,
      "step": 2340
    },
    {
      "epoch": 0.17111442822295847,
      "grad_norm": 1.6539924885076915,
      "learning_rate": 1.6502739940140692e-06,
      "log_odds_chosen": 1.266992211341858,
      "log_odds_ratio": -0.3993164002895355,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.697265625,
      "logps/rejected": -1.6082031726837158,
      "loss": 1.0253,
      "nll_loss": 0.9759765863418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06977538764476776,
      "rewards/margins": 0.09099121391773224,
      "rewards/rejected": -0.16081543266773224,
      "step": 2350
    },
    {
      "epoch": 0.17184257472603487,
      "grad_norm": 1.8805234680157779,
      "learning_rate": 1.6467739391852364e-06,
      "log_odds_chosen": 1.14990234375,
      "log_odds_ratio": -0.4625488221645355,
      "logits/chosen": -0.9556640386581421,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.6856445074081421,
      "logps/rejected": -1.494531273841858,
      "loss": 0.9757,
      "nll_loss": 0.9189453125,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06861571967601776,
      "rewards/margins": 0.08083496242761612,
      "rewards/rejected": -0.14934082329273224,
      "step": 2360
    },
    {
      "epoch": 0.1725707212291113,
      "grad_norm": 1.851740048111942,
      "learning_rate": 1.6432960600355221e-06,
      "log_odds_chosen": 1.459619164466858,
      "log_odds_ratio": -0.42827147245407104,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.907031238079071,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -1.7345702648162842,
      "loss": 0.9805,
      "nll_loss": 0.9203125238418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 0.10480346530675888,
      "rewards/rejected": -0.17348632216453552,
      "step": 2370
    },
    {
      "epoch": 0.17329886773218772,
      "grad_norm": 1.8693265724537342,
      "learning_rate": 1.6398401233815756e-06,
      "log_odds_chosen": 1.18212890625,
      "log_odds_ratio": -0.4681152403354645,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.8603515625,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.5583984851837158,
      "loss": 0.9945,
      "nll_loss": 0.9283202886581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07237549126148224,
      "rewards/margins": 0.08355712890625,
      "rewards/rejected": -0.15590819716453552,
      "step": 2380
    },
    {
      "epoch": 0.17402701423526412,
      "grad_norm": 1.659878915220236,
      "learning_rate": 1.6364058994584524e-06,
      "log_odds_chosen": 1.001953125,
      "log_odds_ratio": -0.4740234315395355,
      "logits/chosen": -0.9496093988418579,
      "logits/rejected": -0.834765613079071,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -1.339257836341858,
      "loss": 0.9863,
      "nll_loss": 0.8802734613418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0667724609375,
      "rewards/margins": 0.06713561713695526,
      "rewards/rejected": -0.13383789360523224,
      "step": 2390
    },
    {
      "epoch": 0.17475516073834055,
      "grad_norm": 2.316586642853329,
      "learning_rate": 1.6329931618554522e-06,
      "log_odds_chosen": 1.373632788658142,
      "log_odds_ratio": -0.377685546875,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.844921886920929,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -1.600976586341858,
      "loss": 0.9967,
      "nll_loss": 0.9170898199081421,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.09375,
      "rewards/rejected": -0.16008301079273224,
      "step": 2400
    },
    {
      "epoch": 0.17548330724141697,
      "grad_norm": 2.4590357818443254,
      "learning_rate": 1.6296016874534209e-06,
      "log_odds_chosen": 1.5207030773162842,
      "log_odds_ratio": -0.39570313692092896,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.8392578363418579,
      "logps/chosen": -0.642285168170929,
      "logps/rejected": -1.730078101158142,
      "loss": 0.9686,
      "nll_loss": 0.908203125,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06423339992761612,
      "rewards/margins": 0.10895995795726776,
      "rewards/rejected": -0.17316894233226776,
      "step": 2410
    },
    {
      "epoch": 0.1762114537444934,
      "grad_norm": 1.9392199805473873,
      "learning_rate": 1.6262312563634835e-06,
      "log_odds_chosen": 1.333740234375,
      "log_odds_ratio": -0.40507811307907104,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.631640613079071,
      "logps/rejected": -1.5558593273162842,
      "loss": 0.9756,
      "nll_loss": 0.9183593988418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06317138671875,
      "rewards/margins": 0.09233398735523224,
      "rewards/rejected": -0.15561524033546448,
      "step": 2420
    },
    {
      "epoch": 0.1769396002475698,
      "grad_norm": 1.7401538261420497,
      "learning_rate": 1.6228816518671587e-06,
      "log_odds_chosen": 1.1311156749725342,
      "log_odds_ratio": -0.458740234375,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.879687488079071,
      "logps/chosen": -0.699999988079071,
      "logps/rejected": -1.5105469226837158,
      "loss": 0.9956,
      "nll_loss": 0.96875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07001952826976776,
      "rewards/margins": 0.08120880275964737,
      "rewards/rejected": -0.15114745497703552,
      "step": 2430
    },
    {
      "epoch": 0.17766774675064623,
      "grad_norm": 1.7387675863635101,
      "learning_rate": 1.619552660357832e-06,
      "log_odds_chosen": 1.4942626953125,
      "log_odds_ratio": -0.3973144590854645,
      "logits/chosen": -1.0046875476837158,
      "logits/rejected": -0.9009765386581421,
      "logps/chosen": -0.67626953125,
      "logps/rejected": -1.777734398841858,
      "loss": 0.9984,
      "nll_loss": 0.9429687261581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06761474907398224,
      "rewards/margins": 0.11007995903491974,
      "rewards/rejected": -0.177734375,
      "step": 2440
    },
    {
      "epoch": 0.17839589325372265,
      "grad_norm": 1.7524662122322663,
      "learning_rate": 1.616244071283537e-06,
      "log_odds_chosen": 1.662500023841858,
      "log_odds_ratio": -0.3309082090854645,
      "logits/chosen": -0.9994140863418579,
      "logits/rejected": -0.8785156011581421,
      "logps/chosen": -0.705859363079071,
      "logps/rejected": -1.9113280773162842,
      "loss": 0.9807,
      "nll_loss": 0.910351574420929,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.07054443657398224,
      "rewards/margins": 0.12045898288488388,
      "rewards/rejected": -0.19106444716453552,
      "step": 2450
    },
    {
      "epoch": 0.17912403975679908,
      "grad_norm": 1.936513413974107,
      "learning_rate": 1.6129556770910235e-06,
      "log_odds_chosen": 1.3793456554412842,
      "log_odds_ratio": -0.40385740995407104,
      "logits/chosen": -0.949999988079071,
      "logits/rejected": -0.8304687738418579,
      "logps/chosen": -0.66650390625,
      "logps/rejected": -1.62890625,
      "loss": 0.9961,
      "nll_loss": 0.9654296636581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.09617309272289276,
      "rewards/rejected": -0.16293945908546448,
      "step": 2460
    },
    {
      "epoch": 0.17985218625987548,
      "grad_norm": 1.8747143255272356,
      "learning_rate": 1.6096872731710673e-06,
      "log_odds_chosen": 1.072167992591858,
      "log_odds_ratio": -0.47880858182907104,
      "logits/chosen": -0.949414074420929,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.700976550579071,
      "logps/rejected": -1.4314453601837158,
      "loss": 0.9992,
      "nll_loss": 0.919140636920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07009277492761612,
      "rewards/margins": 0.07298889011144638,
      "rewards/rejected": -0.14304199814796448,
      "step": 2470
    },
    {
      "epoch": 0.1805803327629519,
      "grad_norm": 1.7652420176242012,
      "learning_rate": 1.6064386578049978e-06,
      "log_odds_chosen": 1.355810523033142,
      "log_odds_ratio": -0.43608397245407104,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.751953125,
      "logps/rejected": -1.7556641101837158,
      "loss": 0.9679,
      "nll_loss": 0.9833984375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07515869289636612,
      "rewards/margins": 0.10038451850414276,
      "rewards/rejected": -0.17556151747703552,
      "step": 2480
    },
    {
      "epoch": 0.18130847926602833,
      "grad_norm": 1.6342061802531707,
      "learning_rate": 1.6032096321124046e-06,
      "log_odds_chosen": 1.2822997570037842,
      "log_odds_ratio": -0.4527343809604645,
      "logits/chosen": -0.9771484136581421,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.759570300579071,
      "logps/rejected": -1.660546898841858,
      "loss": 0.9824,
      "nll_loss": 0.9931640625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.075927734375,
      "rewards/margins": 0.09000472724437714,
      "rewards/rejected": -0.16591796278953552,
      "step": 2490
    },
    {
      "epoch": 0.18203662576910473,
      "grad_norm": 2.2198920472661974,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 1.123779296875,
      "log_odds_ratio": -0.45781248807907104,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -0.903124988079071,
      "logps/chosen": -0.6566406488418579,
      "logps/rejected": -1.4304687976837158,
      "loss": 0.9895,
      "nll_loss": 0.930859386920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06564941257238388,
      "rewards/margins": 0.07747497409582138,
      "rewards/rejected": -0.14309081435203552,
      "step": 2500
    },
    {
      "epoch": 0.18276477227218116,
      "grad_norm": 1.5938048746443403,
      "learning_rate": 1.5968095681115984e-06,
      "log_odds_chosen": 1.220800757408142,
      "log_odds_ratio": -0.4927734434604645,
      "logits/chosen": -0.9662109613418579,
      "logits/rejected": -0.876171886920929,
      "logps/chosen": -0.7642577886581421,
      "logps/rejected": -1.6121094226837158,
      "loss": 0.9914,
      "nll_loss": 0.977734386920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07648925483226776,
      "rewards/margins": 0.08474121242761612,
      "rewards/rejected": -0.16120605170726776,
      "step": 2510
    },
    {
      "epoch": 0.18349291877525759,
      "grad_norm": 1.757444473374284,
      "learning_rate": 1.5936381457791914e-06,
      "log_odds_chosen": 1.196313500404358,
      "log_odds_ratio": -0.4432128965854645,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.687792956829071,
      "logps/rejected": -1.500390648841858,
      "loss": 0.981,
      "nll_loss": 0.935351550579071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06877441704273224,
      "rewards/margins": 0.08116760104894638,
      "rewards/rejected": -0.14992675185203552,
      "step": 2520
    },
    {
      "epoch": 0.184221065278334,
      "grad_norm": 2.1627537795712213,
      "learning_rate": 1.590485544975088e-06,
      "log_odds_chosen": 1.232421875,
      "log_odds_ratio": -0.4478515684604645,
      "logits/chosen": -1.016210913658142,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.7220703363418579,
      "logps/rejected": -1.6316406726837158,
      "loss": 0.9771,
      "nll_loss": 0.918164074420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07222900539636612,
      "rewards/margins": 0.09097900241613388,
      "rewards/rejected": -0.1632080078125,
      "step": 2530
    },
    {
      "epoch": 0.1849492117814104,
      "grad_norm": 1.8098305840340616,
      "learning_rate": 1.5873515802650901e-06,
      "log_odds_chosen": 1.4041016101837158,
      "log_odds_ratio": -0.40693360567092896,
      "logits/chosen": -1.017187476158142,
      "logits/rejected": -0.894726574420929,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -1.710546851158142,
      "loss": 0.9553,
      "nll_loss": 0.916015625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07106933742761612,
      "rewards/margins": 0.10007324069738388,
      "rewards/rejected": -0.17119140923023224,
      "step": 2540
    },
    {
      "epoch": 0.18567735828448684,
      "grad_norm": 1.6365423388715024,
      "learning_rate": 1.584236068762679e-06,
      "log_odds_chosen": 1.2229125499725342,
      "log_odds_ratio": -0.44731444120407104,
      "logits/chosen": -1.0154297351837158,
      "logits/rejected": -0.91796875,
      "logps/chosen": -0.6957031488418579,
      "logps/rejected": -1.574804663658142,
      "loss": 0.9939,
      "nll_loss": 0.923632800579071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.069580078125,
      "rewards/margins": 0.08797454833984375,
      "rewards/rejected": -0.15749511122703552,
      "step": 2550
    },
    {
      "epoch": 0.18640550478756326,
      "grad_norm": 1.844757640743835,
      "learning_rate": 1.5811388300841894e-06,
      "log_odds_chosen": 1.3916015625,
      "log_odds_ratio": -0.4144287109375,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.900585949420929,
      "logps/chosen": -0.641406238079071,
      "logps/rejected": -1.586523413658142,
      "loss": 0.9988,
      "nll_loss": 0.874218761920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06416015326976776,
      "rewards/margins": 0.0944976806640625,
      "rewards/rejected": -0.15854492783546448,
      "step": 2560
    },
    {
      "epoch": 0.18713365129063966,
      "grad_norm": 1.7225735430035751,
      "learning_rate": 1.5780596863049431e-06,
      "log_odds_chosen": 1.188574194908142,
      "log_odds_ratio": -0.44023436307907104,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.893750011920929,
      "logps/chosen": -0.6748046875,
      "logps/rejected": -1.447851538658142,
      "loss": 0.989,
      "nll_loss": 1.0080077648162842,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06746826320886612,
      "rewards/margins": 0.07734374701976776,
      "rewards/rejected": -0.14472655951976776,
      "step": 2570
    },
    {
      "epoch": 0.1878617977937161,
      "grad_norm": 3.8651739541641152,
      "learning_rate": 1.5749984619163156e-06,
      "log_odds_chosen": 1.2480957508087158,
      "log_odds_ratio": -0.4471191465854645,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.6947265863418579,
      "logps/rejected": -1.547265648841858,
      "loss": 0.9871,
      "nll_loss": 0.920703113079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06944580376148224,
      "rewards/margins": 0.08529052883386612,
      "rewards/rejected": -0.15476074814796448,
      "step": 2580
    },
    {
      "epoch": 0.18858994429679252,
      "grad_norm": 2.016288610592031,
      "learning_rate": 1.5719549837837187e-06,
      "log_odds_chosen": 1.294824242591858,
      "log_odds_ratio": -0.45556640625,
      "logits/chosen": -0.9857422113418579,
      "logits/rejected": -0.8753906488418579,
      "logps/chosen": -0.773144543170929,
      "logps/rejected": -1.7058594226837158,
      "loss": 0.9863,
      "nll_loss": 0.968945324420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.077392578125,
      "rewards/margins": 0.09327087551355362,
      "rewards/rejected": -0.1705322265625,
      "step": 2590
    },
    {
      "epoch": 0.18931809079986894,
      "grad_norm": 1.9160316798924124,
      "learning_rate": 1.5689290811054722e-06,
      "log_odds_chosen": 1.395898461341858,
      "log_odds_ratio": -0.37739259004592896,
      "logits/chosen": -0.9791015386581421,
      "logits/rejected": -0.8958984613418579,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.644921898841858,
      "loss": 1.0017,
      "nll_loss": 1.013671875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06876220554113388,
      "rewards/margins": 0.09564208984375,
      "rewards/rejected": -0.16445311903953552,
      "step": 2600
    },
    {
      "epoch": 0.19004623730294534,
      "grad_norm": 3.096157852600544,
      "learning_rate": 1.5659205853725426e-06,
      "log_odds_chosen": 1.1210448741912842,
      "log_odds_ratio": -0.4522460997104645,
      "logits/chosen": -0.959179699420929,
      "logits/rejected": -0.8734375238418579,
      "logps/chosen": -0.6944335699081421,
      "logps/rejected": -1.493749976158142,
      "loss": 0.9654,
      "nll_loss": 0.923828125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06953124701976776,
      "rewards/margins": 0.07993622124195099,
      "rewards/rejected": -0.14951172471046448,
      "step": 2610
    },
    {
      "epoch": 0.19077438380602177,
      "grad_norm": 1.7301517280448162,
      "learning_rate": 1.562929330329127e-06,
      "log_odds_chosen": 0.9090331792831421,
      "log_odds_ratio": -0.5238281488418579,
      "logits/chosen": -0.991992175579071,
      "logits/rejected": -0.908203125,
      "logps/chosen": -0.702929675579071,
      "logps/rejected": -1.3173828125,
      "loss": 0.9711,
      "nll_loss": 0.892578125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07032470405101776,
      "rewards/margins": 0.06130218505859375,
      "rewards/rejected": -0.1317138671875,
      "step": 2620
    },
    {
      "epoch": 0.1915025303090982,
      "grad_norm": 2.21634226483171,
      "learning_rate": 1.5599551519340636e-06,
      "log_odds_chosen": 1.207910180091858,
      "log_odds_ratio": -0.4461914002895355,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -0.678417980670929,
      "logps/rejected": -1.5068359375,
      "loss": 0.9737,
      "nll_loss": 0.9068359136581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06778564304113388,
      "rewards/margins": 0.08300171047449112,
      "rewards/rejected": -0.15070800483226776,
      "step": 2630
    },
    {
      "epoch": 0.19223067681217462,
      "grad_norm": 1.683991813663088,
      "learning_rate": 1.556997888323046e-06,
      "log_odds_chosen": 1.104821801185608,
      "log_odds_ratio": -0.4869140684604645,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -0.704296886920929,
      "logps/rejected": -1.497656226158142,
      "loss": 0.9859,
      "nll_loss": 0.966992199420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07041015475988388,
      "rewards/margins": 0.07934989780187607,
      "rewards/rejected": -0.14975586533546448,
      "step": 2640
    },
    {
      "epoch": 0.19295882331525102,
      "grad_norm": 1.8103352328012394,
      "learning_rate": 1.5540573797716226e-06,
      "log_odds_chosen": 1.421484351158142,
      "log_odds_ratio": -0.380615234375,
      "logits/chosen": -1.028906226158142,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -0.6832031011581421,
      "logps/rejected": -1.6589844226837158,
      "loss": 0.9781,
      "nll_loss": 0.961132824420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06831054389476776,
      "rewards/margins": 0.09750976413488388,
      "rewards/rejected": -0.16584472358226776,
      "step": 2650
    },
    {
      "epoch": 0.19368696981832745,
      "grad_norm": 1.7308963669828101,
      "learning_rate": 1.5511334686589623e-06,
      "log_odds_chosen": 1.2762451171875,
      "log_odds_ratio": -0.43427735567092896,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -0.7164062261581421,
      "logps/rejected": -1.641015648841858,
      "loss": 0.968,
      "nll_loss": 0.9408203363418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07161865383386612,
      "rewards/margins": 0.09243468940258026,
      "rewards/rejected": -0.16396483778953552,
      "step": 2660
    },
    {
      "epoch": 0.19441511632140387,
      "grad_norm": 1.713234417882242,
      "learning_rate": 1.548225999432367e-06,
      "log_odds_chosen": 1.102270483970642,
      "log_odds_ratio": -0.47285157442092896,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8861328363418579,
      "logps/chosen": -0.6875,
      "logps/rejected": -1.4542968273162842,
      "loss": 0.9799,
      "nll_loss": 0.8736327886581421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.0767059326171875,
      "rewards/rejected": -0.14536133408546448,
      "step": 2670
    },
    {
      "epoch": 0.19514326282448027,
      "grad_norm": 2.117196734393062,
      "learning_rate": 1.5453348185725114e-06,
      "log_odds_chosen": 1.164453148841858,
      "log_odds_ratio": -0.49433594942092896,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.8814452886581421,
      "logps/chosen": -0.739062488079071,
      "logps/rejected": -1.560546875,
      "loss": 1.0065,
      "nll_loss": 1.0398437976837158,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07388915866613388,
      "rewards/margins": 0.08219604194164276,
      "rewards/rejected": -0.15607909858226776,
      "step": 2680
    },
    {
      "epoch": 0.1958714093275567,
      "grad_norm": 1.667294432246079,
      "learning_rate": 1.542459774559398e-06,
      "log_odds_chosen": 1.1334960460662842,
      "log_odds_ratio": -0.4444824159145355,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.898632824420929,
      "logps/chosen": -0.6917968988418579,
      "logps/rejected": -1.4851562976837158,
      "loss": 0.9852,
      "nll_loss": 0.9066406488418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06918945163488388,
      "rewards/margins": 0.07927856594324112,
      "rewards/rejected": -0.14846190810203552,
      "step": 2690
    },
    {
      "epoch": 0.19659955583063313,
      "grad_norm": 1.884904503906586,
      "learning_rate": 1.539600717839002e-06,
      "log_odds_chosen": 1.4392578601837158,
      "log_odds_ratio": -0.37578123807907104,
      "logits/chosen": -1.0066406726837158,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.6773437261581421,
      "logps/rejected": -1.6953125,
      "loss": 0.9791,
      "nll_loss": 0.9134765863418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.10184326022863388,
      "rewards/rejected": -0.1695556640625,
      "step": 2700
    },
    {
      "epoch": 0.19732770233370955,
      "grad_norm": 2.1456814461914755,
      "learning_rate": 1.536757500790597e-06,
      "log_odds_chosen": 1.4524414539337158,
      "log_odds_ratio": -0.41157227754592896,
      "logits/chosen": -0.9955078363418579,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.6771484613418579,
      "logps/rejected": -1.6945312023162842,
      "loss": 0.974,
      "nll_loss": 0.9195312261581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06773681938648224,
      "rewards/margins": 0.10166015475988388,
      "rewards/rejected": -0.16933593153953552,
      "step": 2710
    },
    {
      "epoch": 0.19805584883678595,
      "grad_norm": 1.9933224194500967,
      "learning_rate": 1.5339299776947407e-06,
      "log_odds_chosen": 1.044580101966858,
      "log_odds_ratio": -0.47314453125,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.729296863079071,
      "logps/rejected": -1.4314453601837158,
      "loss": 0.9788,
      "nll_loss": 0.941210925579071,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07291259616613388,
      "rewards/margins": 0.070220947265625,
      "rewards/rejected": -0.14313964545726776,
      "step": 2720
    },
    {
      "epoch": 0.19878399533986238,
      "grad_norm": 1.8882083897874273,
      "learning_rate": 1.5311180047019054e-06,
      "log_odds_chosen": 1.301855444908142,
      "log_odds_ratio": -0.4417968690395355,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.859179675579071,
      "logps/chosen": -0.634765625,
      "logps/rejected": -1.5457031726837158,
      "loss": 0.9828,
      "nll_loss": 0.876757800579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0634765625,
      "rewards/margins": 0.09110717475414276,
      "rewards/rejected": -0.15461425483226776,
      "step": 2730
    },
    {
      "epoch": 0.1995121418429388,
      "grad_norm": 1.9272035550408655,
      "learning_rate": 1.5283214398017402e-06,
      "log_odds_chosen": 1.11376953125,
      "log_odds_ratio": -0.47026365995407104,
      "logits/chosen": -0.9710937738418579,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -0.7298828363418579,
      "logps/rejected": -1.5146484375,
      "loss": 0.9807,
      "nll_loss": 0.8951171636581421,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07297363132238388,
      "rewards/margins": 0.07837524265050888,
      "rewards/rejected": -0.15146484971046448,
      "step": 2740
    },
    {
      "epoch": 0.2002402883460152,
      "grad_norm": 2.275440509102858,
      "learning_rate": 1.5255401427929477e-06,
      "log_odds_chosen": 1.2584960460662842,
      "log_odds_ratio": -0.455322265625,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.692089855670929,
      "logps/rejected": -1.5876953601837158,
      "loss": 0.9652,
      "nll_loss": 0.879101574420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06916503608226776,
      "rewards/margins": 0.08952026069164276,
      "rewards/rejected": -0.15871581435203552,
      "step": 2750
    },
    {
      "epoch": 0.20096843484909163,
      "grad_norm": 1.9131325251618527,
      "learning_rate": 1.5227739752537617e-06,
      "log_odds_chosen": 1.525390625,
      "log_odds_ratio": -0.3623046875,
      "logits/chosen": -1.04296875,
      "logits/rejected": -0.933398425579071,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.726171851158142,
      "loss": 0.9481,
      "nll_loss": 0.871289074420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.10809326171875,
      "rewards/rejected": -0.17255859076976776,
      "step": 2760
    },
    {
      "epoch": 0.20169658135216806,
      "grad_norm": 1.872622775460701,
      "learning_rate": 1.5200228005130127e-06,
      "log_odds_chosen": 1.153906226158142,
      "log_odds_ratio": -0.4588867127895355,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.9052734375,
      "logps/chosen": -0.699902355670929,
      "logps/rejected": -1.511328101158142,
      "loss": 0.951,
      "nll_loss": 0.9457031488418579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06999512016773224,
      "rewards/margins": 0.08110351860523224,
      "rewards/rejected": -0.15104980766773224,
      "step": 2770
    },
    {
      "epoch": 0.20242472785524449,
      "grad_norm": 1.8688286652629447,
      "learning_rate": 1.5172864836217631e-06,
      "log_odds_chosen": 1.135156273841858,
      "log_odds_ratio": -0.45585936307907104,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.9332031011581421,
      "logps/chosen": -0.6884765625,
      "logps/rejected": -1.46484375,
      "loss": 0.9524,
      "nll_loss": 0.9007812738418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06885986030101776,
      "rewards/margins": 0.07772521674633026,
      "rewards/rejected": -0.1466064453125,
      "step": 2780
    },
    {
      "epoch": 0.20315287435832088,
      "grad_norm": 1.9921633276397985,
      "learning_rate": 1.514564891325506e-06,
      "log_odds_chosen": 1.201269507408142,
      "log_odds_ratio": -0.4375976622104645,
      "logits/chosen": -0.9580078125,
      "logits/rejected": -0.8740234375,
      "logps/chosen": -0.689453125,
      "logps/rejected": -1.4728515148162842,
      "loss": 0.9724,
      "nll_loss": 0.942187488079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06903076171875,
      "rewards/margins": 0.07842407375574112,
      "rewards/rejected": -0.14726562798023224,
      "step": 2790
    },
    {
      "epoch": 0.2038810208613973,
      "grad_norm": 1.7622652662269196,
      "learning_rate": 1.5118578920369086e-06,
      "log_odds_chosen": 1.521240234375,
      "log_odds_ratio": -0.3876953125,
      "logits/chosen": -0.991406261920929,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -0.6944335699081421,
      "logps/rejected": -1.7703125476837158,
      "loss": 0.9604,
      "nll_loss": 0.924609363079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.10758056491613388,
      "rewards/rejected": -0.17702636122703552,
      "step": 2800
    },
    {
      "epoch": 0.20460916736447374,
      "grad_norm": 2.4610524568144307,
      "learning_rate": 1.5091653558090898e-06,
      "log_odds_chosen": 1.4079101085662842,
      "log_odds_ratio": -0.44477540254592896,
      "logits/chosen": -0.981249988079071,
      "logits/rejected": -0.8775390386581421,
      "logps/chosen": -0.721875011920929,
      "logps/rejected": -1.7531249523162842,
      "loss": 0.9698,
      "nll_loss": 1.011328101158142,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07215575873851776,
      "rewards/margins": 0.10315094143152237,
      "rewards/rejected": -0.17529296875,
      "step": 2810
    },
    {
      "epoch": 0.20533731386755016,
      "grad_norm": 1.855926984871671,
      "learning_rate": 1.506487154309419e-06,
      "log_odds_chosen": 1.3383667469024658,
      "log_odds_ratio": -0.4185546934604645,
      "logits/chosen": -0.9986327886581421,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.679980456829071,
      "logps/rejected": -1.5880858898162842,
      "loss": 0.9646,
      "nll_loss": 0.912304699420929,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06801757961511612,
      "rewards/margins": 0.09089966118335724,
      "rewards/rejected": -0.15871581435203552,
      "step": 2820
    },
    {
      "epoch": 0.20606546037062656,
      "grad_norm": 1.89142819356753,
      "learning_rate": 1.5038231607938247e-06,
      "log_odds_chosen": 1.3450195789337158,
      "log_odds_ratio": -0.4310546815395355,
      "logits/chosen": -0.980664074420929,
      "logits/rejected": -0.892773449420929,
      "logps/chosen": -0.7310546636581421,
      "logps/rejected": -1.6671874523162842,
      "loss": 0.9944,
      "nll_loss": 0.9273437261581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07313232123851776,
      "rewards/margins": 0.09367065131664276,
      "rewards/rejected": -0.16665038466453552,
      "step": 2830
    },
    {
      "epoch": 0.206793606873703,
      "grad_norm": 1.8674812372554181,
      "learning_rate": 1.501173250081603e-06,
      "log_odds_chosen": 1.2644531726837158,
      "log_odds_ratio": -0.43437498807907104,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -1.5935547351837158,
      "loss": 0.9816,
      "nll_loss": 0.9292968511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07009277492761612,
      "rewards/margins": 0.08923645317554474,
      "rewards/rejected": -0.159423828125,
      "step": 2840
    },
    {
      "epoch": 0.20752175337677942,
      "grad_norm": 1.7319261331614564,
      "learning_rate": 1.4985372985307103e-06,
      "log_odds_chosen": 1.299462914466858,
      "log_odds_ratio": -0.4154296815395355,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.680957019329071,
      "logps/rejected": -1.606835961341858,
      "loss": 0.9622,
      "nll_loss": 0.9185546636581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06815185397863388,
      "rewards/margins": 0.09235687553882599,
      "rewards/rejected": -0.16059570014476776,
      "step": 2850
    },
    {
      "epoch": 0.20824989987985582,
      "grad_norm": 1.7066555415301545,
      "learning_rate": 1.4959151840135313e-06,
      "log_odds_chosen": 1.4213135242462158,
      "log_odds_ratio": -0.37651365995407104,
      "logits/chosen": -0.9644531011581421,
      "logits/rejected": -0.8935546875,
      "logps/chosen": -0.60693359375,
      "logps/rejected": -1.5859375,
      "loss": 0.9563,
      "nll_loss": 0.883007824420929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06062011793255806,
      "rewards/margins": 0.09796142578125,
      "rewards/rejected": -0.15866699814796448,
      "step": 2860
    },
    {
      "epoch": 0.20897804638293224,
      "grad_norm": 2.040773021057978,
      "learning_rate": 1.4933067858931148e-06,
      "log_odds_chosen": 1.324804663658142,
      "log_odds_ratio": -0.4117675721645355,
      "logits/chosen": -0.977734386920929,
      "logits/rejected": -0.866015613079071,
      "logps/chosen": -0.6915038824081421,
      "logps/rejected": -1.615234375,
      "loss": 0.9692,
      "nll_loss": 0.916796863079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06910400092601776,
      "rewards/margins": 0.09234619140625,
      "rewards/rejected": -0.1614990234375,
      "step": 2870
    },
    {
      "epoch": 0.20970619288600867,
      "grad_norm": 1.978854072925717,
      "learning_rate": 1.4907119849998597e-06,
      "log_odds_chosen": 1.1728515625,
      "log_odds_ratio": -0.4541015625,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.861132800579071,
      "logps/chosen": -0.662402331829071,
      "logps/rejected": -1.4402344226837158,
      "loss": 0.9556,
      "nll_loss": 0.924023449420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06625976413488388,
      "rewards/margins": 0.07781829684972763,
      "rewards/rejected": -0.14396972954273224,
      "step": 2880
    },
    {
      "epoch": 0.2104343393890851,
      "grad_norm": 1.7054023285399005,
      "learning_rate": 1.488130663608649e-06,
      "log_odds_chosen": 1.0991332530975342,
      "log_odds_ratio": -0.45112305879592896,
      "logits/chosen": -0.9798828363418579,
      "logits/rejected": -0.8675781488418579,
      "logps/chosen": -0.6244140863418579,
      "logps/rejected": -1.2873046398162842,
      "loss": 0.9543,
      "nll_loss": 0.9546874761581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06248779222369194,
      "rewards/margins": 0.06623993068933487,
      "rewards/rejected": -0.12863770127296448,
      "step": 2890
    },
    {
      "epoch": 0.2111624858921615,
      "grad_norm": 2.107837478420817,
      "learning_rate": 1.4855627054164149e-06,
      "log_odds_chosen": 1.473046898841858,
      "log_odds_ratio": -0.41289061307907104,
      "logits/chosen": -1.0134766101837158,
      "logits/rejected": -0.9185546636581421,
      "logps/chosen": -0.6890624761581421,
      "logps/rejected": -1.744531273841858,
      "loss": 0.9433,
      "nll_loss": 0.8968750238418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06887207180261612,
      "rewards/margins": 0.10548706352710724,
      "rewards/rejected": -0.17436523735523224,
      "step": 2900
    },
    {
      "epoch": 0.21189063239523792,
      "grad_norm": 2.2537114122368997,
      "learning_rate": 1.4830079955201294e-06,
      "log_odds_chosen": 1.234094262123108,
      "log_odds_ratio": -0.4718261659145355,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.9085937738418579,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.574609398841858,
      "loss": 0.9536,
      "nll_loss": 0.8763672113418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06864013522863388,
      "rewards/margins": 0.08886337280273438,
      "rewards/rejected": -0.15739746391773224,
      "step": 2910
    },
    {
      "epoch": 0.21261877889831435,
      "grad_norm": 1.8244686245259676,
      "learning_rate": 1.4804664203952103e-06,
      "log_odds_chosen": 1.08837890625,
      "log_odds_ratio": -0.4693359434604645,
      "logits/chosen": -0.9205077886581421,
      "logits/rejected": -0.8248046636581421,
      "logps/chosen": -0.69140625,
      "logps/rejected": -1.457617163658142,
      "loss": 0.9765,
      "nll_loss": 0.968945324420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06918945163488388,
      "rewards/margins": 0.076568603515625,
      "rewards/rejected": -0.14577636122703552,
      "step": 2920
    },
    {
      "epoch": 0.21334692540139075,
      "grad_norm": 1.8006919322547585,
      "learning_rate": 1.4779378678743327e-06,
      "log_odds_chosen": 1.6355469226837158,
      "log_odds_ratio": -0.3731445372104645,
      "logits/chosen": -1.017578125,
      "logits/rejected": -0.887499988079071,
      "logps/chosen": -0.68359375,
      "logps/rejected": -1.875,
      "loss": 0.9595,
      "nll_loss": 0.8802734613418579,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06838379055261612,
      "rewards/margins": 0.11911620944738388,
      "rewards/rejected": -0.18740233778953552,
      "step": 2930
    },
    {
      "epoch": 0.21407507190446717,
      "grad_norm": 1.9131584993525343,
      "learning_rate": 1.4754222271266348e-06,
      "log_odds_chosen": 1.566796898841858,
      "log_odds_ratio": -0.38654786348342896,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.880664050579071,
      "logps/chosen": -0.662792980670929,
      "logps/rejected": -1.7802734375,
      "loss": 0.9541,
      "nll_loss": 0.9365234375,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.11158446967601776,
      "rewards/rejected": -0.17802734673023224,
      "step": 2940
    },
    {
      "epoch": 0.2148032184075436,
      "grad_norm": 1.9460384896474459,
      "learning_rate": 1.4729193886373175e-06,
      "log_odds_chosen": 1.32861328125,
      "log_odds_ratio": -0.4120117127895355,
      "logits/chosen": -0.9390624761581421,
      "logits/rejected": -0.841601550579071,
      "logps/chosen": -0.681445300579071,
      "logps/rejected": -1.6281249523162842,
      "loss": 0.9538,
      "nll_loss": 0.9683593511581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06821288913488388,
      "rewards/margins": 0.09473876655101776,
      "rewards/rejected": -0.16289062798023224,
      "step": 2950
    },
    {
      "epoch": 0.21553136491062003,
      "grad_norm": 2.142437375052006,
      "learning_rate": 1.4704292441876156e-06,
      "log_odds_chosen": 1.2626953125,
      "log_odds_ratio": -0.42509764432907104,
      "logits/chosen": -0.972851574420929,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.7411133050918579,
      "logps/rejected": -1.642578125,
      "loss": 0.9488,
      "nll_loss": 0.9574218988418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07414551079273224,
      "rewards/margins": 0.09000243991613388,
      "rewards/rejected": -0.16428223252296448,
      "step": 2960
    },
    {
      "epoch": 0.21625951141369643,
      "grad_norm": 2.416863996828751,
      "learning_rate": 1.4679516868351474e-06,
      "log_odds_chosen": 1.200097680091858,
      "log_odds_ratio": -0.46064454317092896,
      "logits/chosen": -1.010156273841858,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.687695324420929,
      "logps/rejected": -1.548828125,
      "loss": 0.9551,
      "nll_loss": 0.9525390863418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0687255859375,
      "rewards/margins": 0.086090087890625,
      "rewards/rejected": -0.15485839545726776,
      "step": 2970
    },
    {
      "epoch": 0.21698765791677285,
      "grad_norm": 1.6671627464746113,
      "learning_rate": 1.4654866108946234e-06,
      "log_odds_chosen": 1.0857422351837158,
      "log_odds_ratio": -0.46528321504592896,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.9076172113418579,
      "logps/chosen": -0.661816418170929,
      "logps/rejected": -1.388671875,
      "loss": 0.9622,
      "nll_loss": 0.943164050579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06624756008386612,
      "rewards/margins": 0.0725608840584755,
      "rewards/rejected": -0.138916015625,
      "step": 2980
    },
    {
      "epoch": 0.21771580441984928,
      "grad_norm": 2.2636494828883826,
      "learning_rate": 1.4630339119189101e-06,
      "log_odds_chosen": 1.367285132408142,
      "log_odds_ratio": -0.4156738221645355,
      "logits/chosen": -0.9654296636581421,
      "logits/rejected": -0.8583984375,
      "logps/chosen": -0.6966797113418579,
      "logps/rejected": -1.6833984851837158,
      "loss": 0.9592,
      "nll_loss": 0.950390636920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.09891357272863388,
      "rewards/rejected": -0.16855469346046448,
      "step": 2990
    },
    {
      "epoch": 0.2184439509229257,
      "grad_norm": 2.1101380163406804,
      "learning_rate": 1.4605934866804429e-06,
      "log_odds_chosen": 1.328710913658142,
      "log_odds_ratio": -0.4356933534145355,
      "logits/chosen": -0.9716796875,
      "logits/rejected": -0.8720703125,
      "logps/chosen": -0.6787109375,
      "logps/rejected": -1.6240234375,
      "loss": 0.9662,
      "nll_loss": 0.9486328363418579,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06793212890625,
      "rewards/margins": 0.09464111179113388,
      "rewards/rejected": -0.16259765625,
      "step": 3000
    },
    {
      "epoch": 0.2191720974260021,
      "grad_norm": 2.0487337106120056,
      "learning_rate": 1.4581652331529784e-06,
      "log_odds_chosen": 1.3349609375,
      "log_odds_ratio": -0.4010253846645355,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.880078136920929,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -1.585546851158142,
      "loss": 0.9323,
      "nll_loss": 0.91015625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06630859524011612,
      "rewards/margins": 0.09222412109375,
      "rewards/rejected": -0.15854492783546448,
      "step": 3010
    },
    {
      "epoch": 0.21990024392907853,
      "grad_norm": 1.7821297389824873,
      "learning_rate": 1.4557490504936778e-06,
      "log_odds_chosen": 1.4997069835662842,
      "log_odds_ratio": -0.4007812440395355,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -0.8892577886581421,
      "logps/chosen": -0.677929699420929,
      "logps/rejected": -1.708593726158142,
      "loss": 0.9473,
      "nll_loss": 0.9619140625,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.10306243598461151,
      "rewards/rejected": -0.1708984375,
      "step": 3020
    },
    {
      "epoch": 0.22062839043215496,
      "grad_norm": 2.2075025049368913,
      "learning_rate": 1.453344839025519e-06,
      "log_odds_chosen": 1.3333008289337158,
      "log_odds_ratio": -0.43901365995407104,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.8861328363418579,
      "logps/chosen": -0.707714855670929,
      "logps/rejected": -1.649023413658142,
      "loss": 0.9526,
      "nll_loss": 0.927734375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07072754204273224,
      "rewards/margins": 0.09394989162683487,
      "rewards/rejected": -0.16486816108226776,
      "step": 3030
    },
    {
      "epoch": 0.22135653693523136,
      "grad_norm": 1.8552765813231715,
      "learning_rate": 1.4509525002200234e-06,
      "log_odds_chosen": 1.34716796875,
      "log_odds_ratio": -0.4063476622104645,
      "logits/chosen": -1.0349609851837158,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.7123047113418579,
      "logps/rejected": -1.6941406726837158,
      "loss": 0.9545,
      "nll_loss": 0.9576171636581421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.07130126655101776,
      "rewards/margins": 0.09808349609375,
      "rewards/rejected": -0.16938476264476776,
      "step": 3040
    },
    {
      "epoch": 0.22208468343830778,
      "grad_norm": 2.0386303284968292,
      "learning_rate": 1.4485719366802965e-06,
      "log_odds_chosen": 1.5283203125,
      "log_odds_ratio": -0.3778320252895355,
      "logits/chosen": -1.051367163658142,
      "logits/rejected": -0.966992199420929,
      "logps/chosen": -0.68408203125,
      "logps/rejected": -1.8097655773162842,
      "loss": 0.9659,
      "nll_loss": 0.9164062738418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06837157905101776,
      "rewards/margins": 0.112548828125,
      "rewards/rejected": -0.18100586533546448,
      "step": 3050
    },
    {
      "epoch": 0.2228128299413842,
      "grad_norm": 1.9756674859249355,
      "learning_rate": 1.4462030521243742e-06,
      "log_odds_chosen": 1.2770507335662842,
      "log_odds_ratio": -0.4183105528354645,
      "logits/chosen": -1.0255858898162842,
      "logits/rejected": -0.9388672113418579,
      "logps/chosen": -0.6094726324081421,
      "logps/rejected": -1.441992163658142,
      "loss": 0.9493,
      "nll_loss": 0.8822265863418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06102294847369194,
      "rewards/margins": 0.08314208686351776,
      "rewards/rejected": -0.14418944716453552,
      "step": 3060
    },
    {
      "epoch": 0.22354097644446064,
      "grad_norm": 2.026828091317089,
      "learning_rate": 1.443845751368867e-06,
      "log_odds_chosen": 1.4765625,
      "log_odds_ratio": -0.4332031309604645,
      "logits/chosen": -0.9624999761581421,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -0.6673828363418579,
      "logps/rejected": -1.730078101158142,
      "loss": 0.9682,
      "nll_loss": 0.902148425579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06671142578125,
      "rewards/margins": 0.1063232421875,
      "rewards/rejected": -0.17294922471046448,
      "step": 3070
    },
    {
      "epoch": 0.22426912294753704,
      "grad_norm": 2.0704026248128464,
      "learning_rate": 1.4414999403128943e-06,
      "log_odds_chosen": 1.27685546875,
      "log_odds_ratio": -0.38818359375,
      "logits/chosen": -0.992382824420929,
      "logits/rejected": -0.8880859613418579,
      "logps/chosen": -0.642871081829071,
      "logps/rejected": -1.485937476158142,
      "loss": 0.9409,
      "nll_loss": 0.889453113079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.064208984375,
      "rewards/margins": 0.08423309028148651,
      "rewards/rejected": -0.14853516221046448,
      "step": 3080
    },
    {
      "epoch": 0.22499726945061346,
      "grad_norm": 1.6596722166646387,
      "learning_rate": 1.439165525922309e-06,
      "log_odds_chosen": 1.521875023841858,
      "log_odds_ratio": -0.3441406190395355,
      "logits/chosen": -0.9955078363418579,
      "logits/rejected": -0.87890625,
      "logps/chosen": -0.63427734375,
      "logps/rejected": -1.6306641101837158,
      "loss": 0.9517,
      "nll_loss": 0.9166015386581421,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06345214694738388,
      "rewards/margins": 0.09968261420726776,
      "rewards/rejected": -0.16303710639476776,
      "step": 3090
    },
    {
      "epoch": 0.2257254159536899,
      "grad_norm": 2.4207485804016358,
      "learning_rate": 1.4368424162141992e-06,
      "log_odds_chosen": 1.219140648841858,
      "log_odds_ratio": -0.4796386659145355,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.713671863079071,
      "logps/rejected": -1.608007788658142,
      "loss": 0.9715,
      "nll_loss": 0.972851574420929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07130126655101776,
      "rewards/margins": 0.08950195461511612,
      "rewards/rejected": -0.16074219346046448,
      "step": 3100
    },
    {
      "epoch": 0.2264535624567663,
      "grad_norm": 1.8252249236954992,
      "learning_rate": 1.434530520241665e-06,
      "log_odds_chosen": 1.3927733898162842,
      "log_odds_ratio": -0.4014648497104645,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.6524413824081421,
      "logps/rejected": -1.5988280773162842,
      "loss": 0.9402,
      "nll_loss": 0.886523425579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06525878608226776,
      "rewards/margins": 0.094482421875,
      "rewards/rejected": -0.1597900390625,
      "step": 3110
    },
    {
      "epoch": 0.22718170895984272,
      "grad_norm": 1.8812273728775581,
      "learning_rate": 1.4322297480788657e-06,
      "log_odds_chosen": 1.1748535633087158,
      "log_odds_ratio": -0.46625977754592896,
      "logits/chosen": -0.985156238079071,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.6996093988418579,
      "logps/rejected": -1.526953101158142,
      "loss": 0.9517,
      "nll_loss": 0.945507824420929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06998290866613388,
      "rewards/margins": 0.08274994045495987,
      "rewards/rejected": -0.1527099609375,
      "step": 3120
    },
    {
      "epoch": 0.22790985546291914,
      "grad_norm": 2.7344401519505057,
      "learning_rate": 1.4299400108063247e-06,
      "log_odds_chosen": 1.3515625,
      "log_odds_ratio": -0.44465333223342896,
      "logits/chosen": -0.9927734136581421,
      "logits/rejected": -0.8744140863418579,
      "logps/chosen": -0.732421875,
      "logps/rejected": -1.7041015625,
      "loss": 0.9683,
      "nll_loss": 0.920703113079071,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07322998344898224,
      "rewards/margins": 0.09711913764476776,
      "rewards/rejected": -0.17045898735523224,
      "step": 3130
    },
    {
      "epoch": 0.22863800196599557,
      "grad_norm": 1.9995693445540086,
      "learning_rate": 1.4276612204964992e-06,
      "log_odds_chosen": 1.285375952720642,
      "log_odds_ratio": -0.43500977754592896,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.6634765863418579,
      "logps/rejected": -1.5261719226837158,
      "loss": 0.9348,
      "nll_loss": 0.853320300579071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06633301079273224,
      "rewards/margins": 0.086273193359375,
      "rewards/rejected": -0.152587890625,
      "step": 3140
    },
    {
      "epoch": 0.22936614846907197,
      "grad_norm": 1.9731818610147924,
      "learning_rate": 1.4253932901995967e-06,
      "log_odds_chosen": 1.2380859851837158,
      "log_odds_ratio": -0.4385742247104645,
      "logits/chosen": -1.0076172351837158,
      "logits/rejected": -0.909375011920929,
      "logps/chosen": -0.7046874761581421,
      "logps/rejected": -1.595312476158142,
      "loss": 0.9175,
      "nll_loss": 0.8804687261581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07048340141773224,
      "rewards/margins": 0.08892212063074112,
      "rewards/rejected": -0.15939942002296448,
      "step": 3150
    },
    {
      "epoch": 0.2300942949721484,
      "grad_norm": 1.96315338446132,
      "learning_rate": 1.42313613392964e-06,
      "log_odds_chosen": 1.506250023841858,
      "log_odds_ratio": -0.36601561307907104,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.9140625,
      "logps/chosen": -0.656445324420929,
      "logps/rejected": -1.695898413658142,
      "loss": 0.9576,
      "nll_loss": 0.9486328363418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06558837741613388,
      "rewards/margins": 0.10384521633386612,
      "rewards/rejected": -0.16940918564796448,
      "step": 3160
    },
    {
      "epoch": 0.23082244147522482,
      "grad_norm": 1.9582604422326573,
      "learning_rate": 1.4208896666507756e-06,
      "log_odds_chosen": 1.4503905773162842,
      "log_odds_ratio": -0.40576171875,
      "logits/chosen": -1.0183594226837158,
      "logits/rejected": -0.8890625238418579,
      "logps/chosen": -0.7232421636581421,
      "logps/rejected": -1.7511718273162842,
      "loss": 0.9388,
      "nll_loss": 0.90234375,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07225342094898224,
      "rewards/margins": 0.10303344577550888,
      "rewards/rejected": -0.17524413764476776,
      "step": 3170
    },
    {
      "epoch": 0.23155058797830125,
      "grad_norm": 1.9001038651532063,
      "learning_rate": 1.4186538042638173e-06,
      "log_odds_chosen": 1.2781250476837158,
      "log_odds_ratio": -0.43427735567092896,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.8832031488418579,
      "logps/chosen": -0.687304675579071,
      "logps/rejected": -1.58203125,
      "loss": 0.9531,
      "nll_loss": 0.959179699420929,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06873778998851776,
      "rewards/margins": 0.089447021484375,
      "rewards/rejected": -0.15822753310203552,
      "step": 3180
    },
    {
      "epoch": 0.23227873448137765,
      "grad_norm": 1.8761442443485807,
      "learning_rate": 1.416428463593022e-06,
      "log_odds_chosen": 1.21142578125,
      "log_odds_ratio": -0.4537597596645355,
      "logits/chosen": -1.042578101158142,
      "logits/rejected": -0.927539050579071,
      "logps/chosen": -0.67724609375,
      "logps/rejected": -1.530664086341858,
      "loss": 0.9525,
      "nll_loss": 0.9273437261581421,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06779785454273224,
      "rewards/margins": 0.08539581298828125,
      "rewards/rejected": -0.15324707329273224,
      "step": 3190
    },
    {
      "epoch": 0.23300688098445407,
      "grad_norm": 2.102780096954173,
      "learning_rate": 1.414213562373095e-06,
      "log_odds_chosen": 1.2815430164337158,
      "log_odds_ratio": -0.4295410215854645,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.908984363079071,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.608984351158142,
      "loss": 0.9245,
      "nll_loss": 0.95703125,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06760253757238388,
      "rewards/margins": 0.09331665188074112,
      "rewards/rejected": -0.1610107421875,
      "step": 3200
    },
    {
      "epoch": 0.2337350274875305,
      "grad_norm": 2.0596656084325704,
      "learning_rate": 1.4120090192364154e-06,
      "log_odds_chosen": 1.218847632408142,
      "log_odds_ratio": -0.43657225370407104,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.7001953125,
      "logps/rejected": -1.5652344226837158,
      "loss": 0.9467,
      "nll_loss": 0.891796886920929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07003173977136612,
      "rewards/margins": 0.08646240085363388,
      "rewards/rejected": -0.15642090141773224,
      "step": 3210
    },
    {
      "epoch": 0.2344631739906069,
      "grad_norm": 1.7776871068480575,
      "learning_rate": 1.4098147537004828e-06,
      "log_odds_chosen": 1.5828125476837158,
      "log_odds_ratio": -0.382080078125,
      "logits/chosen": -1.0400390625,
      "logits/rejected": -0.947460949420929,
      "logps/chosen": -0.667187511920929,
      "logps/rejected": -1.776757836341858,
      "loss": 0.9231,
      "nll_loss": 0.8199218511581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06669922173023224,
      "rewards/margins": 0.11105956882238388,
      "rewards/rejected": -0.17778320610523224,
      "step": 3220
    },
    {
      "epoch": 0.23519132049368333,
      "grad_norm": 2.2488850443210056,
      "learning_rate": 1.4076306861555735e-06,
      "log_odds_chosen": 1.41796875,
      "log_odds_ratio": -0.40283203125,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.8779296875,
      "logps/chosen": -0.6537109613418579,
      "logps/rejected": -1.617773413658142,
      "loss": 0.9367,
      "nll_loss": 0.900390625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06531982123851776,
      "rewards/margins": 0.09633026272058487,
      "rewards/rejected": -0.16169433295726776,
      "step": 3230
    },
    {
      "epoch": 0.23591946699675975,
      "grad_norm": 1.9473739415490248,
      "learning_rate": 1.405456737852613e-06,
      "log_odds_chosen": 1.30908203125,
      "log_odds_ratio": -0.4225097596645355,
      "logits/chosen": -0.9857422113418579,
      "logits/rejected": -0.8968750238418579,
      "logps/chosen": -0.6573241949081421,
      "logps/rejected": -1.591406226158142,
      "loss": 0.9486,
      "nll_loss": 0.9212890863418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06569824367761612,
      "rewards/margins": 0.09355469048023224,
      "rewards/rejected": -0.15927734971046448,
      "step": 3240
    },
    {
      "epoch": 0.23664761349983618,
      "grad_norm": 1.9792943929256113,
      "learning_rate": 1.4032928308912468e-06,
      "log_odds_chosen": 1.3916015625,
      "log_odds_ratio": -0.4039062559604645,
      "logits/chosen": -0.9466797113418579,
      "logits/rejected": -0.8560546636581421,
      "logps/chosen": -0.658203125,
      "logps/rejected": -1.6072266101837158,
      "loss": 0.9077,
      "nll_loss": 0.819140613079071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06582031399011612,
      "rewards/margins": 0.09481201320886612,
      "rewards/rejected": -0.16062012314796448,
      "step": 3250
    },
    {
      "epoch": 0.23737576000291258,
      "grad_norm": 1.9243620305329163,
      "learning_rate": 1.4011388882081175e-06,
      "log_odds_chosen": 1.3654296398162842,
      "log_odds_ratio": -0.43818360567092896,
      "logits/chosen": -0.9710937738418579,
      "logits/rejected": -0.910937488079071,
      "logps/chosen": -0.6949218511581421,
      "logps/rejected": -1.65234375,
      "loss": 0.9609,
      "nll_loss": 0.916210949420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06950683891773224,
      "rewards/margins": 0.09586334228515625,
      "rewards/rejected": -0.16533203423023224,
      "step": 3260
    },
    {
      "epoch": 0.238103906505989,
      "grad_norm": 1.8789820602704654,
      "learning_rate": 1.3989948335653378e-06,
      "log_odds_chosen": 1.082421898841858,
      "log_odds_ratio": -0.45166015625,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.8671875,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.3624999523162842,
      "loss": 0.9315,
      "nll_loss": 0.915234386920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06593017280101776,
      "rewards/margins": 0.07033081352710724,
      "rewards/rejected": -0.13623046875,
      "step": 3270
    },
    {
      "epoch": 0.23883205300906543,
      "grad_norm": 1.8270238165002866,
      "learning_rate": 1.3968605915391564e-06,
      "log_odds_chosen": 0.9653564691543579,
      "log_odds_ratio": -0.510009765625,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -0.854687511920929,
      "logps/chosen": -0.7044922113418579,
      "logps/rejected": -1.341796875,
      "loss": 0.9267,
      "nll_loss": 0.9300781488418579,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0704345703125,
      "rewards/margins": 0.06384124606847763,
      "rewards/rejected": -0.13430175185203552,
      "step": 3280
    },
    {
      "epoch": 0.23956019951214183,
      "grad_norm": 1.7458016586679614,
      "learning_rate": 1.3947360875088132e-06,
      "log_odds_chosen": 1.3389160633087158,
      "log_odds_ratio": -0.4267822206020355,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.882031261920929,
      "logps/chosen": -0.655078113079071,
      "logps/rejected": -1.5925781726837158,
      "loss": 0.9279,
      "nll_loss": 0.9091796875,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0655517578125,
      "rewards/margins": 0.09366454929113388,
      "rewards/rejected": -0.15922851860523224,
      "step": 3290
    },
    {
      "epoch": 0.24028834601521826,
      "grad_norm": 1.9535989792097175,
      "learning_rate": 1.3926212476455828e-06,
      "log_odds_chosen": 1.395410180091858,
      "log_odds_ratio": -0.39301759004592896,
      "logits/chosen": -0.988085925579071,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.716113269329071,
      "logps/rejected": -1.6892578601837158,
      "loss": 0.9456,
      "nll_loss": 0.9443359375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07160644233226776,
      "rewards/margins": 0.09724120795726776,
      "rewards/rejected": -0.1689453125,
      "step": 3300
    },
    {
      "epoch": 0.24101649251829468,
      "grad_norm": 2.098551401687419,
      "learning_rate": 1.3905159989019964e-06,
      "log_odds_chosen": 1.3742187023162842,
      "log_odds_ratio": -0.3951171934604645,
      "logits/chosen": -0.958789050579071,
      "logits/rejected": -0.864062488079071,
      "logps/chosen": -0.6548827886581421,
      "logps/rejected": -1.6208984851837158,
      "loss": 0.9566,
      "nll_loss": 0.8724609613418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06551513820886612,
      "rewards/margins": 0.09650878608226776,
      "rewards/rejected": -0.16203613579273224,
      "step": 3310
    },
    {
      "epoch": 0.2417446390213711,
      "grad_norm": 2.2667028369549502,
      "learning_rate": 1.3884202690012465e-06,
      "log_odds_chosen": 1.1271851062774658,
      "log_odds_ratio": -0.49238282442092896,
      "logits/chosen": -0.9697265625,
      "logits/rejected": -0.9140625,
      "logps/chosen": -0.70703125,
      "logps/rejected": -1.492773413658142,
      "loss": 0.9255,
      "nll_loss": 0.883007824420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.07071533054113388,
      "rewards/margins": 0.07855224609375,
      "rewards/rejected": -0.14926758408546448,
      "step": 3320
    },
    {
      "epoch": 0.2424727855244475,
      "grad_norm": 1.8684644804444257,
      "learning_rate": 1.3863339864267636e-06,
      "log_odds_chosen": 1.131445288658142,
      "log_odds_ratio": -0.47309571504592896,
      "logits/chosen": -0.979687511920929,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.663281261920929,
      "logps/rejected": -1.4275391101837158,
      "loss": 0.9517,
      "nll_loss": 0.908984363079071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06638183444738388,
      "rewards/margins": 0.07631225883960724,
      "rewards/rejected": -0.14262695610523224,
      "step": 3330
    },
    {
      "epoch": 0.24320093202752394,
      "grad_norm": 1.9487258090702118,
      "learning_rate": 1.3842570804119655e-06,
      "log_odds_chosen": 1.220239281654358,
      "log_odds_ratio": -0.4520019590854645,
      "logits/chosen": -1.0353515148162842,
      "logits/rejected": -0.938671886920929,
      "logps/chosen": -0.694531261920929,
      "logps/rejected": -1.5656249523162842,
      "loss": 0.9226,
      "nll_loss": 0.8564453125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06944580376148224,
      "rewards/margins": 0.08715057373046875,
      "rewards/rejected": -0.1566162109375,
      "step": 3340
    },
    {
      "epoch": 0.24392907853060036,
      "grad_norm": 2.0037519577994223,
      "learning_rate": 1.3821894809301763e-06,
      "log_odds_chosen": 1.4854857921600342,
      "log_odds_ratio": -0.4117187559604645,
      "logits/chosen": -1.0166015625,
      "logits/rejected": -0.9117187261581421,
      "logps/chosen": -0.679882824420929,
      "logps/rejected": -1.7296874523162842,
      "loss": 0.9265,
      "nll_loss": 0.858691394329071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06796874850988388,
      "rewards/margins": 0.10483551025390625,
      "rewards/rejected": -0.17290039360523224,
      "step": 3350
    },
    {
      "epoch": 0.2446572250336768,
      "grad_norm": 2.3791373909526206,
      "learning_rate": 1.3801311186847081e-06,
      "log_odds_chosen": 1.338281273841858,
      "log_odds_ratio": -0.40966796875,
      "logits/chosen": -1.057031273841858,
      "logits/rejected": -0.9261718988418579,
      "logps/chosen": -0.6436523199081421,
      "logps/rejected": -1.578515648841858,
      "loss": 0.9323,
      "nll_loss": 0.8744140863418579,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06434325873851776,
      "rewards/margins": 0.09346923977136612,
      "rewards/rejected": -0.157958984375,
      "step": 3360
    },
    {
      "epoch": 0.2453853715367532,
      "grad_norm": 2.9431341239057374,
      "learning_rate": 1.378081925099109e-06,
      "log_odds_chosen": 1.499414086341858,
      "log_odds_ratio": -0.3798828125,
      "logits/chosen": -1.043554663658142,
      "logits/rejected": -0.905468761920929,
      "logps/chosen": -0.66748046875,
      "logps/rejected": -1.739843726158142,
      "loss": 0.9167,
      "nll_loss": 0.8755859136581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0667724609375,
      "rewards/margins": 0.107208251953125,
      "rewards/rejected": -0.17404785752296448,
      "step": 3370
    },
    {
      "epoch": 0.24611351803982962,
      "grad_norm": 1.9785449319589474,
      "learning_rate": 1.376041832307563e-06,
      "log_odds_chosen": 1.524999976158142,
      "log_odds_ratio": -0.37958985567092896,
      "logits/chosen": -1.0068359375,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.6705077886581421,
      "logps/rejected": -1.7296874523162842,
      "loss": 0.9534,
      "nll_loss": 0.874804675579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06705322116613388,
      "rewards/margins": 0.10585937649011612,
      "rewards/rejected": -0.17312011122703552,
      "step": 3380
    },
    {
      "epoch": 0.24684166454290604,
      "grad_norm": 1.870208371949718,
      "learning_rate": 1.3740107731454524e-06,
      "log_odds_chosen": 1.1873047351837158,
      "log_odds_ratio": -0.4505371153354645,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.9296875,
      "logps/chosen": -0.669628918170929,
      "logps/rejected": -1.508398413658142,
      "loss": 0.9537,
      "nll_loss": 0.8500000238418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06699218600988388,
      "rewards/margins": 0.08387450873851776,
      "rewards/rejected": -0.15097656846046448,
      "step": 3390
    },
    {
      "epoch": 0.24756981104598244,
      "grad_norm": 2.070363334418945,
      "learning_rate": 1.3719886811400705e-06,
      "log_odds_chosen": 1.3897705078125,
      "log_odds_ratio": -0.42094725370407104,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.6768554449081421,
      "logps/rejected": -1.6730468273162842,
      "loss": 0.9404,
      "nll_loss": 0.9085937738418579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06765136867761612,
      "rewards/margins": 0.09942932426929474,
      "rewards/rejected": -0.1671142578125,
      "step": 3400
    },
    {
      "epoch": 0.24829795754905887,
      "grad_norm": 2.1482349420441684,
      "learning_rate": 1.3699754905014834e-06,
      "log_odds_chosen": 1.43603515625,
      "log_odds_ratio": -0.36176759004592896,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.862109363079071,
      "logps/chosen": -0.6426757574081421,
      "logps/rejected": -1.6052734851837158,
      "loss": 0.9344,
      "nll_loss": 0.911914050579071,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.09621582180261612,
      "rewards/rejected": -0.16049805283546448,
      "step": 3410
    },
    {
      "epoch": 0.2490261040521353,
      "grad_norm": 2.141510424454952,
      "learning_rate": 1.3679711361135388e-06,
      "log_odds_chosen": 1.211181640625,
      "log_odds_ratio": -0.4310546815395355,
      "logits/chosen": -0.9970703125,
      "logits/rejected": -0.920703113079071,
      "logps/chosen": -0.7007812261581421,
      "logps/rejected": -1.550390601158142,
      "loss": 0.9168,
      "nll_loss": 0.8329101800918579,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.070068359375,
      "rewards/margins": 0.08489990234375,
      "rewards/rejected": -0.15500488877296448,
      "step": 3420
    },
    {
      "epoch": 0.24975425055521172,
      "grad_norm": 2.4080216592754056,
      "learning_rate": 1.3659755535250212e-06,
      "log_odds_chosen": 1.6338379383087158,
      "log_odds_ratio": -0.38127440214157104,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.885546863079071,
      "logps/chosen": -0.6625000238418579,
      "logps/rejected": -1.8162109851837158,
      "loss": 0.918,
      "nll_loss": 0.832812488079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0662841796875,
      "rewards/margins": 0.11539916694164276,
      "rewards/rejected": -0.18173828721046448,
      "step": 3430
    },
    {
      "epoch": 0.2504823970582881,
      "grad_norm": 1.7590568219662346,
      "learning_rate": 1.3639886789409469e-06,
      "log_odds_chosen": 1.2498047351837158,
      "log_odds_ratio": -0.41923826932907104,
      "logits/chosen": -1.018164038658142,
      "logits/rejected": -0.9339843988418579,
      "logps/chosen": -0.666796863079071,
      "logps/rejected": -1.492578148841858,
      "loss": 0.939,
      "nll_loss": 0.8726562261581421,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06666259467601776,
      "rewards/margins": 0.08251953125,
      "rewards/rejected": -0.149169921875,
      "step": 3440
    },
    {
      "epoch": 0.2512105435613645,
      "grad_norm": 1.8599517577101523,
      "learning_rate": 1.3620104492139977e-06,
      "log_odds_chosen": 1.6375000476837158,
      "log_odds_ratio": -0.37993162870407104,
      "logits/chosen": -1.040429711341858,
      "logits/rejected": -0.948046863079071,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -1.831640601158142,
      "loss": 0.9036,
      "nll_loss": 0.90625,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06843261420726776,
      "rewards/margins": 0.11461181938648224,
      "rewards/rejected": -0.18310546875,
      "step": 3450
    },
    {
      "epoch": 0.251938690064441,
      "grad_norm": 2.438198042937003,
      "learning_rate": 1.3600408018360918e-06,
      "log_odds_chosen": 1.582617163658142,
      "log_odds_ratio": -0.38139647245407104,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.920117199420929,
      "logps/chosen": -0.671875,
      "logps/rejected": -1.796484351158142,
      "loss": 0.9084,
      "nll_loss": 0.8744140863418579,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06718750298023224,
      "rewards/margins": 0.11269531399011612,
      "rewards/rejected": -0.17988280951976776,
      "step": 3460
    },
    {
      "epoch": 0.25266683656751737,
      "grad_norm": 1.7893320646909474,
      "learning_rate": 1.3580796749300878e-06,
      "log_odds_chosen": 1.408203125,
      "log_odds_ratio": -0.409423828125,
      "logits/chosen": -1.0066406726837158,
      "logits/rejected": -0.9253906011581421,
      "logps/chosen": -0.708789050579071,
      "logps/rejected": -1.661718726158142,
      "loss": 0.9128,
      "nll_loss": 0.887890636920929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07076416164636612,
      "rewards/margins": 0.0953369140625,
      "rewards/rejected": -0.16618652641773224,
      "step": 3470
    },
    {
      "epoch": 0.2533949830705938,
      "grad_norm": 2.128657610487306,
      "learning_rate": 1.3561270072416209e-06,
      "log_odds_chosen": 1.4201171398162842,
      "log_odds_ratio": -0.41533201932907104,
      "logits/chosen": -1.011132836341858,
      "logits/rejected": -0.913867175579071,
      "logps/chosen": -0.692187488079071,
      "logps/rejected": -1.728906273841858,
      "loss": 0.9361,
      "nll_loss": 0.8677734136581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06925048679113388,
      "rewards/margins": 0.10357666015625,
      "rewards/rejected": -0.1727294921875,
      "step": 3480
    },
    {
      "epoch": 0.2541231295736702,
      "grad_norm": 1.8041119682970694,
      "learning_rate": 1.3541827381310652e-06,
      "log_odds_chosen": 1.2841796875,
      "log_odds_ratio": -0.4425292909145355,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.8539062738418579,
      "logps/chosen": -0.7339843511581421,
      "logps/rejected": -1.633203148841858,
      "loss": 0.9178,
      "nll_loss": 0.9322265386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0733642578125,
      "rewards/margins": 0.089874267578125,
      "rewards/rejected": -0.16330567002296448,
      "step": 3490
    },
    {
      "epoch": 0.2548512760767466,
      "grad_norm": 1.8833387818705296,
      "learning_rate": 1.3522468075656264e-06,
      "log_odds_chosen": 1.2472655773162842,
      "log_odds_ratio": -0.42578125,
      "logits/chosen": -1.010156273841858,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.7027343511581421,
      "logps/rejected": -1.543554663658142,
      "loss": 0.9257,
      "nll_loss": 0.826953113079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07027588039636612,
      "rewards/margins": 0.08406372368335724,
      "rewards/rejected": -0.15451660752296448,
      "step": 3500
    },
    {
      "epoch": 0.2555794225798231,
      "grad_norm": 1.9071480388205753,
      "learning_rate": 1.3503191561115553e-06,
      "log_odds_chosen": 1.1837890148162842,
      "log_odds_ratio": -0.4224609434604645,
      "logits/chosen": -1.055273413658142,
      "logits/rejected": -0.965039074420929,
      "logps/chosen": -0.680957019329071,
      "logps/rejected": -1.4607422351837158,
      "loss": 0.9192,
      "nll_loss": 0.809765636920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06807861477136612,
      "rewards/margins": 0.07801513373851776,
      "rewards/rejected": -0.14614257216453552,
      "step": 3510
    },
    {
      "epoch": 0.2563075690828995,
      "grad_norm": 2.219931969049714,
      "learning_rate": 1.348399724926484e-06,
      "log_odds_chosen": 1.265039086341858,
      "log_odds_ratio": -0.43745118379592896,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.884570300579071,
      "logps/chosen": -0.649121105670929,
      "logps/rejected": -1.5558593273162842,
      "loss": 0.9313,
      "nll_loss": 0.8404296636581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06491699069738388,
      "rewards/margins": 0.09067382663488388,
      "rewards/rejected": -0.15556640923023224,
      "step": 3520
    },
    {
      "epoch": 0.2570357155859759,
      "grad_norm": 1.8339356722724671,
      "learning_rate": 1.346488455751882e-06,
      "log_odds_chosen": 1.3312499523162842,
      "log_odds_ratio": -0.4427734315395355,
      "logits/chosen": -1.008398413658142,
      "logits/rejected": -0.9072265625,
      "logps/chosen": -0.7255859375,
      "logps/rejected": -1.672265648841858,
      "loss": 0.9429,
      "nll_loss": 0.888476550579071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07257080078125,
      "rewards/margins": 0.0946044921875,
      "rewards/rejected": -0.16730956733226776,
      "step": 3530
    },
    {
      "epoch": 0.25776386208905233,
      "grad_norm": 2.051741764411779,
      "learning_rate": 1.3445852909056286e-06,
      "log_odds_chosen": 1.495019555091858,
      "log_odds_ratio": -0.37114256620407104,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.9111328125,
      "logps/chosen": -0.68115234375,
      "logps/rejected": -1.71484375,
      "loss": 0.9199,
      "nll_loss": 0.8687499761581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06800536811351776,
      "rewards/margins": 0.10347900539636612,
      "rewards/rejected": -0.17143554985523224,
      "step": 3540
    },
    {
      "epoch": 0.25849200859212873,
      "grad_norm": 2.1376568213975564,
      "learning_rate": 1.3426901732747024e-06,
      "log_odds_chosen": 1.2824218273162842,
      "log_odds_ratio": -0.42626953125,
      "logits/chosen": -1.003515601158142,
      "logits/rejected": -0.9208984375,
      "logps/chosen": -0.6385742425918579,
      "logps/rejected": -1.525781273841858,
      "loss": 0.9193,
      "nll_loss": 0.840624988079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06389160454273224,
      "rewards/margins": 0.08857421576976776,
      "rewards/rejected": -0.15244141221046448,
      "step": 3550
    },
    {
      "epoch": 0.25922015509520513,
      "grad_norm": 2.1283031384308146,
      "learning_rate": 1.3408030463079818e-06,
      "log_odds_chosen": 1.439062476158142,
      "log_odds_ratio": -0.3955078125,
      "logits/chosen": -0.988476574420929,
      "logits/rejected": -0.893359363079071,
      "logps/chosen": -0.6976562738418579,
      "logps/rejected": -1.6873047351837158,
      "loss": 0.9367,
      "nll_loss": 0.8794921636581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06972656399011612,
      "rewards/margins": 0.09901122748851776,
      "rewards/rejected": -0.16879883408546448,
      "step": 3560
    },
    {
      "epoch": 0.2599483015982816,
      "grad_norm": 1.9317890240768365,
      "learning_rate": 1.3389238540091568e-06,
      "log_odds_chosen": 1.162109375,
      "log_odds_ratio": -0.4800781309604645,
      "logits/chosen": -0.999804675579071,
      "logits/rejected": -0.9144531488418579,
      "logps/chosen": -0.695117175579071,
      "logps/rejected": -1.4783203601837158,
      "loss": 0.9294,
      "nll_loss": 0.878222644329071,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06940917670726776,
      "rewards/margins": 0.0784759521484375,
      "rewards/rejected": -0.14775390923023224,
      "step": 3570
    },
    {
      "epoch": 0.260676448101358,
      "grad_norm": 2.145466794618356,
      "learning_rate": 1.337052540929751e-06,
      "log_odds_chosen": 1.366308569908142,
      "log_odds_ratio": -0.42158204317092896,
      "logits/chosen": -1.0291016101837158,
      "logits/rejected": -0.952343761920929,
      "logps/chosen": -0.698046863079071,
      "logps/rejected": -1.630468726158142,
      "loss": 0.9262,
      "nll_loss": 0.890625,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06981201469898224,
      "rewards/margins": 0.09336242824792862,
      "rewards/rejected": -0.16318359971046448,
      "step": 3580
    },
    {
      "epoch": 0.26140459460443444,
      "grad_norm": 2.2878805210850652,
      "learning_rate": 1.33518905216225e-06,
      "log_odds_chosen": 1.3230469226837158,
      "log_odds_ratio": -0.4310058653354645,
      "logits/chosen": -1.018164038658142,
      "logits/rejected": -0.912890613079071,
      "logps/chosen": -0.73046875,
      "logps/rejected": -1.6613280773162842,
      "loss": 0.9049,
      "nll_loss": 0.862500011920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07310791313648224,
      "rewards/margins": 0.09298095852136612,
      "rewards/rejected": -0.16616210341453552,
      "step": 3590
    },
    {
      "epoch": 0.26213274110751084,
      "grad_norm": 2.529065082726043,
      "learning_rate": 1.3333333333333332e-06,
      "log_odds_chosen": 1.219824194908142,
      "log_odds_ratio": -0.4585937559604645,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -0.7060546875,
      "logps/rejected": -1.5496094226837158,
      "loss": 0.9263,
      "nll_loss": 0.8666015863418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07060547173023224,
      "rewards/margins": 0.08421631157398224,
      "rewards/rejected": -0.15488281846046448,
      "step": 3600
    },
    {
      "epoch": 0.26286088761058723,
      "grad_norm": 2.14329552263589,
      "learning_rate": 1.3314853305972122e-06,
      "log_odds_chosen": 1.574804663658142,
      "log_odds_ratio": -0.36455076932907104,
      "logits/chosen": -0.979296863079071,
      "logits/rejected": -0.851367175579071,
      "logps/chosen": -0.660937488079071,
      "logps/rejected": -1.767578125,
      "loss": 0.9349,
      "nll_loss": 0.93359375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06612548977136612,
      "rewards/margins": 0.11071167141199112,
      "rewards/rejected": -0.17680664360523224,
      "step": 3610
    },
    {
      "epoch": 0.2635890341136637,
      "grad_norm": 2.054969353533555,
      "learning_rate": 1.3296449906290671e-06,
      "log_odds_chosen": 1.6687500476837158,
      "log_odds_ratio": -0.34404295682907104,
      "logits/chosen": -1.037500023841858,
      "logits/rejected": -0.8951171636581421,
      "logps/chosen": -0.665332019329071,
      "logps/rejected": -1.8347656726837158,
      "loss": 0.9152,
      "nll_loss": 0.8707031011581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06657715141773224,
      "rewards/margins": 0.11689452826976776,
      "rewards/rejected": -0.18349608778953552,
      "step": 3620
    },
    {
      "epoch": 0.2643171806167401,
      "grad_norm": 2.0298369960702627,
      "learning_rate": 1.3278122606185844e-06,
      "log_odds_chosen": 1.4845702648162842,
      "log_odds_ratio": -0.43681639432907104,
      "logits/chosen": -1.007226586341858,
      "logits/rejected": -0.883593738079071,
      "logps/chosen": -0.654296875,
      "logps/rejected": -1.68359375,
      "loss": 0.9293,
      "nll_loss": 0.833984375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06545410305261612,
      "rewards/margins": 0.10294189304113388,
      "rewards/rejected": -0.16826172173023224,
      "step": 3630
    },
    {
      "epoch": 0.2650453271198165,
      "grad_norm": 2.0786852622533303,
      "learning_rate": 1.3259870882635918e-06,
      "log_odds_chosen": 1.4045898914337158,
      "log_odds_ratio": -0.4500732421875,
      "logits/chosen": -0.9847656488418579,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.68505859375,
      "logps/rejected": -1.722070336341858,
      "loss": 0.9263,
      "nll_loss": 0.8955078125,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.10381774604320526,
      "rewards/rejected": -0.17229004204273224,
      "step": 3640
    },
    {
      "epoch": 0.26577347362289294,
      "grad_norm": 2.065679727412843,
      "learning_rate": 1.3241694217637886e-06,
      "log_odds_chosen": 1.4490234851837158,
      "log_odds_ratio": -0.402587890625,
      "logits/chosen": -1.0089843273162842,
      "logits/rejected": -0.910937488079071,
      "logps/chosen": -0.6944335699081421,
      "logps/rejected": -1.706640601158142,
      "loss": 0.9267,
      "nll_loss": 0.916015625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06939697265625,
      "rewards/margins": 0.10113525390625,
      "rewards/rejected": -0.17048339545726776,
      "step": 3650
    },
    {
      "epoch": 0.26650162012596934,
      "grad_norm": 2.1227982487132735,
      "learning_rate": 1.3223592098145723e-06,
      "log_odds_chosen": 1.2158203125,
      "log_odds_ratio": -0.4048828184604645,
      "logits/chosen": -1.023828148841858,
      "logits/rejected": -0.932812511920929,
      "logps/chosen": -0.6910156011581421,
      "logps/rejected": -1.528906226158142,
      "loss": 0.9229,
      "nll_loss": 0.900585949420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06914062798023224,
      "rewards/margins": 0.08385010063648224,
      "rewards/rejected": -0.15285643935203552,
      "step": 3660
    },
    {
      "epoch": 0.26722976662904574,
      "grad_norm": 1.880816951977088,
      "learning_rate": 1.3205564016009555e-06,
      "log_odds_chosen": 1.0674560070037842,
      "log_odds_ratio": -0.4695800840854645,
      "logits/chosen": -1.016210913658142,
      "logits/rejected": -0.9375,
      "logps/chosen": -0.642578125,
      "logps/rejected": -1.3699219226837158,
      "loss": 0.9333,
      "nll_loss": 0.828320324420929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06424560397863388,
      "rewards/margins": 0.07275619357824326,
      "rewards/rejected": -0.13706055283546448,
      "step": 3670
    },
    {
      "epoch": 0.2679579131321222,
      "grad_norm": 2.078438689330952,
      "learning_rate": 1.318760946791574e-06,
      "log_odds_chosen": 1.455322265625,
      "log_odds_ratio": -0.44965821504592896,
      "logits/chosen": -1.031835913658142,
      "logits/rejected": -0.869921863079071,
      "logps/chosen": -0.724316418170929,
      "logps/rejected": -1.792578101158142,
      "loss": 0.9201,
      "nll_loss": 0.915234386920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07241211086511612,
      "rewards/margins": 0.10657958686351776,
      "rewards/rejected": -0.17890624701976776,
      "step": 3680
    },
    {
      "epoch": 0.2686860596351986,
      "grad_norm": 1.770180295412941,
      "learning_rate": 1.316972795532786e-06,
      "log_odds_chosen": 1.380761742591858,
      "log_odds_ratio": -0.4234375059604645,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.9085937738418579,
      "logps/chosen": -0.706835925579071,
      "logps/rejected": -1.6550781726837158,
      "loss": 0.9225,
      "nll_loss": 0.84521484375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07066650688648224,
      "rewards/margins": 0.09483642876148224,
      "rewards/rejected": -0.16557617485523224,
      "step": 3690
    },
    {
      "epoch": 0.26941420613827505,
      "grad_norm": 2.080720067998385,
      "learning_rate": 1.3151918984428582e-06,
      "log_odds_chosen": 1.280615210533142,
      "log_odds_ratio": -0.44807130098342896,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.890625,
      "logps/chosen": -0.6983398199081421,
      "logps/rejected": -1.616601586341858,
      "loss": 0.9095,
      "nll_loss": 0.9105468988418579,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06979980319738388,
      "rewards/margins": 0.09171752631664276,
      "rewards/rejected": -0.16159668564796448,
      "step": 3700
    },
    {
      "epoch": 0.27014235264135145,
      "grad_norm": 2.4581478536055124,
      "learning_rate": 1.313418206606237e-06,
      "log_odds_chosen": 1.448144555091858,
      "log_odds_ratio": -0.39179688692092896,
      "logits/chosen": -1.032812476158142,
      "logits/rejected": -0.875,
      "logps/chosen": -0.646191418170929,
      "logps/rejected": -1.6785156726837158,
      "loss": 0.8938,
      "nll_loss": 0.780078113079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0645751953125,
      "rewards/margins": 0.103118896484375,
      "rewards/rejected": -0.16774901747703552,
      "step": 3710
    },
    {
      "epoch": 0.27087049914442785,
      "grad_norm": 1.8616826962580717,
      "learning_rate": 1.3116516715679057e-06,
      "log_odds_chosen": 1.378515601158142,
      "log_odds_ratio": -0.39208984375,
      "logits/chosen": -1.0302734375,
      "logits/rejected": -0.943359375,
      "logps/chosen": -0.630566418170929,
      "logps/rejected": -1.560156226158142,
      "loss": 0.9291,
      "nll_loss": 0.887499988079071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06309814751148224,
      "rewards/margins": 0.09299316257238388,
      "rewards/rejected": -0.15598145127296448,
      "step": 3720
    },
    {
      "epoch": 0.2715986456475043,
      "grad_norm": 2.0681530201690546,
      "learning_rate": 1.3098922453278258e-06,
      "log_odds_chosen": 1.2501952648162842,
      "log_odds_ratio": -0.3738769590854645,
      "logits/chosen": -0.9966796636581421,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.6156250238418579,
      "logps/rejected": -1.4021484851837158,
      "loss": 0.9013,
      "nll_loss": 0.83203125,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.06157226487994194,
      "rewards/margins": 0.078765869140625,
      "rewards/rejected": -0.14033202826976776,
      "step": 3730
    },
    {
      "epoch": 0.2723267921505807,
      "grad_norm": 2.308783637756246,
      "learning_rate": 1.3081398803354573e-06,
      "log_odds_chosen": 1.2775390148162842,
      "log_odds_ratio": -0.4527831971645355,
      "logits/chosen": -1.035742163658142,
      "logits/rejected": -0.92578125,
      "logps/chosen": -0.7310546636581421,
      "logps/rejected": -1.651953101158142,
      "loss": 0.9044,
      "nll_loss": 0.903515636920929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07314453274011612,
      "rewards/margins": 0.09195556491613388,
      "rewards/rejected": -0.16506347060203552,
      "step": 3740
    },
    {
      "epoch": 0.2730549386536571,
      "grad_norm": 2.319177821523628,
      "learning_rate": 1.3063945294843617e-06,
      "log_odds_chosen": 1.370996117591858,
      "log_odds_ratio": -0.38725584745407104,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.851757824420929,
      "logps/chosen": -0.658984363079071,
      "logps/rejected": -1.611718773841858,
      "loss": 0.8906,
      "nll_loss": 0.855664074420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06588134914636612,
      "rewards/margins": 0.09519042819738388,
      "rewards/rejected": -0.1611328125,
      "step": 3750
    },
    {
      "epoch": 0.27378308515673355,
      "grad_norm": 1.761299683171171,
      "learning_rate": 1.3046561461068843e-06,
      "log_odds_chosen": 1.4573853015899658,
      "log_odds_ratio": -0.41669923067092896,
      "logits/chosen": -1.0388672351837158,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.693359375,
      "logps/rejected": -1.772851586341858,
      "loss": 0.927,
      "nll_loss": 0.8705078363418579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06943359225988388,
      "rewards/margins": 0.10798950493335724,
      "rewards/rejected": -0.17720946669578552,
      "step": 3760
    },
    {
      "epoch": 0.27451123165980995,
      "grad_norm": 2.8612497602223397,
      "learning_rate": 1.3029246839689124e-06,
      "log_odds_chosen": 1.4542968273162842,
      "log_odds_ratio": -0.3822265565395355,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.8949218988418579,
      "logps/chosen": -0.658007800579071,
      "logps/rejected": -1.65625,
      "loss": 0.9125,
      "nll_loss": 0.887890636920929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.099945068359375,
      "rewards/rejected": -0.16567382216453552,
      "step": 3770
    },
    {
      "epoch": 0.27523937816288635,
      "grad_norm": 2.555510954711944,
      "learning_rate": 1.3012000972647109e-06,
      "log_odds_chosen": 1.423681616783142,
      "log_odds_ratio": -0.4099365174770355,
      "logits/chosen": -0.9964843988418579,
      "logits/rejected": -0.8597656488418579,
      "logps/chosen": -0.6548827886581421,
      "logps/rejected": -1.695703148841858,
      "loss": 0.9301,
      "nll_loss": 0.867382824420929,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06550292670726776,
      "rewards/margins": 0.10407409816980362,
      "rewards/rejected": -0.16945800185203552,
      "step": 3780
    },
    {
      "epoch": 0.2759675246659628,
      "grad_norm": 1.9698754464810986,
      "learning_rate": 1.299482340611832e-06,
      "log_odds_chosen": 1.4599609375,
      "log_odds_ratio": -0.45283204317092896,
      "logits/chosen": -0.963671863079071,
      "logits/rejected": -0.8607422113418579,
      "logps/chosen": -0.711230456829071,
      "logps/rejected": -1.763281226158142,
      "loss": 0.9143,
      "nll_loss": 0.890820324420929,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07105712592601776,
      "rewards/margins": 0.10535888373851776,
      "rewards/rejected": -0.1763916015625,
      "step": 3790
    },
    {
      "epoch": 0.2766956711690392,
      "grad_norm": 2.0439385608108673,
      "learning_rate": 1.2977713690461003e-06,
      "log_odds_chosen": 1.425927758216858,
      "log_odds_ratio": -0.44755858182907104,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.7183593511581421,
      "logps/rejected": -1.7746093273162842,
      "loss": 0.9203,
      "nll_loss": 0.903515636920929,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.07191161811351776,
      "rewards/margins": 0.10572052001953125,
      "rewards/rejected": -0.17753906548023224,
      "step": 3800
    },
    {
      "epoch": 0.2774238176721156,
      "grad_norm": 2.1809033960735817,
      "learning_rate": 1.296067138016669e-06,
      "log_odds_chosen": 1.251953125,
      "log_odds_ratio": -0.42158204317092896,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -0.649121105670929,
      "logps/rejected": -1.493749976158142,
      "loss": 0.9036,
      "nll_loss": 0.8109375238418579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06490478664636612,
      "rewards/margins": 0.08449707180261612,
      "rewards/rejected": -0.14948730170726776,
      "step": 3810
    },
    {
      "epoch": 0.27815196417519206,
      "grad_norm": 2.383663367704226,
      "learning_rate": 1.294369603381147e-06,
      "log_odds_chosen": 1.4202148914337158,
      "log_odds_ratio": -0.3775390684604645,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.932421863079071,
      "logps/chosen": -0.6888672113418579,
      "logps/rejected": -1.6677734851837158,
      "loss": 0.9208,
      "nll_loss": 0.879101574420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06892089545726776,
      "rewards/margins": 0.09795226901769638,
      "rewards/rejected": -0.166748046875,
      "step": 3820
    },
    {
      "epoch": 0.27888011067826846,
      "grad_norm": 1.862087057097791,
      "learning_rate": 1.2926787214007981e-06,
      "log_odds_chosen": 1.414160132408142,
      "log_odds_ratio": -0.4173339903354645,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.870312511920929,
      "logps/chosen": -0.6859375238418579,
      "logps/rejected": -1.6845703125,
      "loss": 0.9364,
      "nll_loss": 0.9281250238418579,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06862793117761612,
      "rewards/margins": 0.09991760551929474,
      "rewards/rejected": -0.16848143935203552,
      "step": 3830
    },
    {
      "epoch": 0.2796082571813449,
      "grad_norm": 1.8573254019846177,
      "learning_rate": 1.2909944487358056e-06,
      "log_odds_chosen": 1.089941382408142,
      "log_odds_ratio": -0.46923828125,
      "logits/chosen": -0.9828125238418579,
      "logits/rejected": -0.865429699420929,
      "logps/chosen": -0.677734375,
      "logps/rejected": -1.4169921875,
      "loss": 0.909,
      "nll_loss": 0.867382824420929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06781005859375,
      "rewards/margins": 0.07388915866613388,
      "rewards/rejected": -0.14157715439796448,
      "step": 3840
    },
    {
      "epoch": 0.2803364036844213,
      "grad_norm": 2.9560278293757656,
      "learning_rate": 1.2893167424406084e-06,
      "log_odds_chosen": 1.584375023841858,
      "log_odds_ratio": -0.37211912870407104,
      "logits/chosen": -1.0294921398162842,
      "logits/rejected": -0.8984375,
      "logps/chosen": -0.6563476324081421,
      "logps/rejected": -1.7556641101837158,
      "loss": 0.916,
      "nll_loss": 0.873242199420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06560058891773224,
      "rewards/margins": 0.10994873195886612,
      "rewards/rejected": -0.17558594048023224,
      "step": 3850
    },
    {
      "epoch": 0.2810645501874977,
      "grad_norm": 2.301857896508706,
      "learning_rate": 1.2876455599593008e-06,
      "log_odds_chosen": 1.272314429283142,
      "log_odds_ratio": -0.4703125059604645,
      "logits/chosen": -0.96875,
      "logits/rejected": -0.8929687738418579,
      "logps/chosen": -0.7123047113418579,
      "logps/rejected": -1.6369140148162842,
      "loss": 0.899,
      "nll_loss": 0.907421886920929,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07126464694738388,
      "rewards/margins": 0.092315673828125,
      "rewards/rejected": -0.16362304985523224,
      "step": 3860
    },
    {
      "epoch": 0.28179269669057416,
      "grad_norm": 1.9693252976660351,
      "learning_rate": 1.285980859121099e-06,
      "log_odds_chosen": 1.489843726158142,
      "log_odds_ratio": -0.3890136778354645,
      "logits/chosen": -0.996874988079071,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -0.6749023199081421,
      "logps/rejected": -1.7138671875,
      "loss": 0.912,
      "nll_loss": 0.927539050579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06744384765625,
      "rewards/margins": 0.10399170219898224,
      "rewards/rejected": -0.17145995795726776,
      "step": 3870
    },
    {
      "epoch": 0.28252084319365056,
      "grad_norm": 1.8250221689884194,
      "learning_rate": 1.2843225981358712e-06,
      "log_odds_chosen": 1.29486083984375,
      "log_odds_ratio": -0.3955078125,
      "logits/chosen": -1.0125000476837158,
      "logits/rejected": -0.8667968511581421,
      "logps/chosen": -0.661914050579071,
      "logps/rejected": -1.5587890148162842,
      "loss": 0.903,
      "nll_loss": 0.8369140625,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06622314453125,
      "rewards/margins": 0.0897369384765625,
      "rewards/rejected": -0.15590819716453552,
      "step": 3880
    },
    {
      "epoch": 0.28324898969672696,
      "grad_norm": 1.9583934401346017,
      "learning_rate": 1.2826707355897317e-06,
      "log_odds_chosen": 1.431249976158142,
      "log_odds_ratio": -0.40107423067092896,
      "logits/chosen": -0.948437511920929,
      "logits/rejected": -0.8599609136581421,
      "logps/chosen": -0.7049804925918579,
      "logps/rejected": -1.7117187976837158,
      "loss": 0.9119,
      "nll_loss": 0.982617199420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07045898586511612,
      "rewards/margins": 0.10072021186351776,
      "rewards/rejected": -0.1712646484375,
      "step": 3890
    },
    {
      "epoch": 0.2839771361998034,
      "grad_norm": 2.390947632672128,
      "learning_rate": 1.281025230440697e-06,
      "log_odds_chosen": 1.03076171875,
      "log_odds_ratio": -0.46064454317092896,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.6996093988418579,
      "logps/rejected": -1.390625,
      "loss": 0.9179,
      "nll_loss": 0.862109363079071,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0699462890625,
      "rewards/margins": 0.06906585395336151,
      "rewards/rejected": -0.13908691704273224,
      "step": 3900
    },
    {
      "epoch": 0.2847052827028798,
      "grad_norm": 1.906944086757529,
      "learning_rate": 1.2793860420144025e-06,
      "log_odds_chosen": 1.515039086341858,
      "log_odds_ratio": -0.4058593809604645,
      "logits/chosen": -0.9525390863418579,
      "logits/rejected": -0.833789050579071,
      "logps/chosen": -0.6563476324081421,
      "logps/rejected": -1.704687476158142,
      "loss": 0.8985,
      "nll_loss": 0.836132824420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.10487060248851776,
      "rewards/rejected": -0.17050781846046448,
      "step": 3910
    },
    {
      "epoch": 0.2854334292059562,
      "grad_norm": 2.3331581164239474,
      "learning_rate": 1.2777531299998798e-06,
      "log_odds_chosen": 1.26416015625,
      "log_odds_ratio": -0.46088868379592896,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.887499988079071,
      "logps/chosen": -0.6910156011581421,
      "logps/rejected": -1.564453125,
      "loss": 0.9413,
      "nll_loss": 0.849414050579071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06914062798023224,
      "rewards/margins": 0.0873565673828125,
      "rewards/rejected": -0.15642090141773224,
      "step": 3920
    },
    {
      "epoch": 0.28616157570903267,
      "grad_norm": 2.088158930255884,
      "learning_rate": 1.2761264544453928e-06,
      "log_odds_chosen": 1.27783203125,
      "log_odds_ratio": -0.42866212129592896,
      "logits/chosen": -0.9876953363418579,
      "logits/rejected": -0.868359386920929,
      "logps/chosen": -0.6851562261581421,
      "logps/rejected": -1.587890625,
      "loss": 0.9007,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.09032592922449112,
      "rewards/rejected": -0.15895995497703552,
      "step": 3930
    },
    {
      "epoch": 0.28688972221210907,
      "grad_norm": 2.0013029666363837,
      "learning_rate": 1.2745059757543324e-06,
      "log_odds_chosen": 1.3886229991912842,
      "log_odds_ratio": -0.43559569120407104,
      "logits/chosen": -0.989062488079071,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -0.7183593511581421,
      "logps/rejected": -1.6789062023162842,
      "loss": 0.8902,
      "nll_loss": 0.859570324420929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07181396335363388,
      "rewards/margins": 0.09621276706457138,
      "rewards/rejected": -0.16787108778953552,
      "step": 3940
    },
    {
      "epoch": 0.2876178687151855,
      "grad_norm": 1.980447538853639,
      "learning_rate": 1.272891654681168e-06,
      "log_odds_chosen": 1.460302710533142,
      "log_odds_ratio": -0.4136718809604645,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.8843749761581421,
      "logps/chosen": -0.739453136920929,
      "logps/rejected": -1.808984398841858,
      "loss": 0.8962,
      "nll_loss": 0.882617175579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07402344048023224,
      "rewards/margins": 0.10699462890625,
      "rewards/rejected": -0.18085937201976776,
      "step": 3950
    },
    {
      "epoch": 0.2883460152182619,
      "grad_norm": 2.1029680609042822,
      "learning_rate": 1.2712834523274563e-06,
      "log_odds_chosen": 1.3596680164337158,
      "log_odds_ratio": -0.4193359315395355,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.6722656488418579,
      "logps/rejected": -1.596093773841858,
      "loss": 0.904,
      "nll_loss": 0.8792968988418579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06724853813648224,
      "rewards/margins": 0.09249267727136612,
      "rewards/rejected": -0.15961913764476776,
      "step": 3960
    },
    {
      "epoch": 0.2890741617213383,
      "grad_norm": 2.010968121301939,
      "learning_rate": 1.2696813301379032e-06,
      "log_odds_chosen": 1.6116211414337158,
      "log_odds_ratio": -0.35407716035842896,
      "logits/chosen": -0.9740234613418579,
      "logits/rejected": -0.8427734375,
      "logps/chosen": -0.625,
      "logps/rejected": -1.7361328601837158,
      "loss": 0.9081,
      "nll_loss": 0.854687511920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06253661960363388,
      "rewards/margins": 0.11113891750574112,
      "rewards/rejected": -0.17360839247703552,
      "step": 3970
    },
    {
      "epoch": 0.2898023082244148,
      "grad_norm": 2.19757893074996,
      "learning_rate": 1.2680852498964829e-06,
      "log_odds_chosen": 1.2998046875,
      "log_odds_ratio": -0.45917969942092896,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.8765624761581421,
      "logps/chosen": -0.6751953363418579,
      "logps/rejected": -1.6300780773162842,
      "loss": 0.8971,
      "nll_loss": 0.8603515625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0675048828125,
      "rewards/margins": 0.09555206447839737,
      "rewards/rejected": -0.16301269829273224,
      "step": 3980
    },
    {
      "epoch": 0.29053045472749117,
      "grad_norm": 2.114679185301981,
      "learning_rate": 1.266495173722607e-06,
      "log_odds_chosen": 1.539648413658142,
      "log_odds_ratio": -0.3841308653354645,
      "logits/chosen": -1.01171875,
      "logits/rejected": -0.90234375,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -1.697851538658142,
      "loss": 0.8915,
      "nll_loss": 0.8187500238418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06541748344898224,
      "rewards/margins": 0.10440673679113388,
      "rewards/rejected": -0.16989746689796448,
      "step": 3990
    },
    {
      "epoch": 0.29125860123056757,
      "grad_norm": 3.003774127739079,
      "learning_rate": 1.2649110640673517e-06,
      "log_odds_chosen": 1.4013671875,
      "log_odds_ratio": -0.4100097715854645,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.850781261920929,
      "logps/chosen": -0.6455078125,
      "logps/rejected": -1.581640601158142,
      "loss": 0.9143,
      "nll_loss": 0.8765624761581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.09369506686925888,
      "rewards/rejected": -0.15817871689796448,
      "step": 4000
    },
    {
      "epoch": 0.291986747733644,
      "grad_norm": 1.9432782525194665,
      "learning_rate": 1.2633328837097308e-06,
      "log_odds_chosen": 1.6609375476837158,
      "log_odds_ratio": -0.35004884004592896,
      "logits/chosen": -1.032617211341858,
      "logits/rejected": -0.9076172113418579,
      "logps/chosen": -0.647656261920929,
      "logps/rejected": -1.8406250476837158,
      "loss": 0.9006,
      "nll_loss": 0.8480468988418579,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06474609673023224,
      "rewards/margins": 0.1192626953125,
      "rewards/rejected": -0.18405762314796448,
      "step": 4010
    },
    {
      "epoch": 0.2927148942367204,
      "grad_norm": 1.9009380432104825,
      "learning_rate": 1.2617605957530233e-06,
      "log_odds_chosen": 1.289648413658142,
      "log_odds_ratio": -0.43144530057907104,
      "logits/chosen": -1.006445288658142,
      "logits/rejected": -0.8998047113418579,
      "logps/chosen": -0.6607421636581421,
      "logps/rejected": -1.551171898841858,
      "loss": 0.8871,
      "nll_loss": 0.876171886920929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0660400390625,
      "rewards/margins": 0.08900757133960724,
      "rewards/rejected": -0.15500488877296448,
      "step": 4020
    },
    {
      "epoch": 0.2934430407397968,
      "grad_norm": 2.1431399739267727,
      "learning_rate": 1.2601941636211516e-06,
      "log_odds_chosen": 1.6111328601837158,
      "log_odds_ratio": -0.37641602754592896,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.872265636920929,
      "logps/chosen": -0.6869140863418579,
      "logps/rejected": -1.8468749523162842,
      "loss": 0.8911,
      "nll_loss": 0.808789074420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06871338188648224,
      "rewards/margins": 0.11597900092601776,
      "rewards/rejected": -0.18471679091453552,
      "step": 4030
    },
    {
      "epoch": 0.2941711872428733,
      "grad_norm": 1.782542270707798,
      "learning_rate": 1.2586335510551052e-06,
      "log_odds_chosen": 1.3860352039337158,
      "log_odds_ratio": -0.43657225370407104,
      "logits/chosen": -1.0087890625,
      "logits/rejected": -0.897656261920929,
      "logps/chosen": -0.7105468511581421,
      "logps/rejected": -1.7150390148162842,
      "loss": 0.9062,
      "nll_loss": 0.832812488079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.07109375298023224,
      "rewards/margins": 0.10028076171875,
      "rewards/rejected": -0.17131347954273224,
      "step": 4040
    },
    {
      "epoch": 0.2948993337459497,
      "grad_norm": 2.023179698986602,
      "learning_rate": 1.2570787221094177e-06,
      "log_odds_chosen": 1.715429663658142,
      "log_odds_ratio": -0.3423828184604645,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.8359375,
      "logps/chosen": -0.637011706829071,
      "logps/rejected": -1.837499976158142,
      "loss": 0.9159,
      "nll_loss": 0.8511718511581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.063720703125,
      "rewards/margins": 0.1201171875,
      "rewards/rejected": -0.18393555283546448,
      "step": 4050
    },
    {
      "epoch": 0.29562748024902613,
      "grad_norm": 1.9733336284907965,
      "learning_rate": 1.255529641148689e-06,
      "log_odds_chosen": 1.424218773841858,
      "log_odds_ratio": -0.3931518495082855,
      "logits/chosen": -0.970507800579071,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.620312511920929,
      "logps/rejected": -1.560937523841858,
      "loss": 0.9037,
      "nll_loss": 0.8568359613418579,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06204833835363388,
      "rewards/margins": 0.09418945014476776,
      "rewards/rejected": -0.15605469048023224,
      "step": 4060
    },
    {
      "epoch": 0.29635562675210253,
      "grad_norm": 2.334298693947026,
      "learning_rate": 1.2539862728441536e-06,
      "log_odds_chosen": 1.18115234375,
      "log_odds_ratio": -0.4518066346645355,
      "logits/chosen": -0.9761718511581421,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.6849609613418579,
      "logps/rejected": -1.5076172351837158,
      "loss": 0.8958,
      "nll_loss": 0.897656261920929,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.08220825344324112,
      "rewards/rejected": -0.15073242783546448,
      "step": 4070
    },
    {
      "epoch": 0.29708377325517893,
      "grad_norm": 2.0335569719244746,
      "learning_rate": 1.252448582170299e-06,
      "log_odds_chosen": 1.3641235828399658,
      "log_odds_ratio": -0.4237304627895355,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.6128906011581421,
      "logps/rejected": -1.5457031726837158,
      "loss": 0.8798,
      "nll_loss": 0.785937488079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06124267727136612,
      "rewards/margins": 0.09330139309167862,
      "rewards/rejected": -0.154541015625,
      "step": 4080
    },
    {
      "epoch": 0.2978119197582554,
      "grad_norm": 2.149184128933456,
      "learning_rate": 1.2509165344015243e-06,
      "log_odds_chosen": 1.2607421875,
      "log_odds_ratio": -0.3938964903354645,
      "logits/chosen": -0.9789062738418579,
      "logits/rejected": -0.852343738079071,
      "logps/chosen": -0.6298828125,
      "logps/rejected": -1.476171851158142,
      "loss": 0.8922,
      "nll_loss": 0.8187500238418579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06297607719898224,
      "rewards/margins": 0.084564208984375,
      "rewards/rejected": -0.14755859971046448,
      "step": 4090
    },
    {
      "epoch": 0.2985400662613318,
      "grad_norm": 2.4756301036059116,
      "learning_rate": 1.2493900951088486e-06,
      "log_odds_chosen": 1.378625512123108,
      "log_odds_ratio": -0.37871092557907104,
      "logits/chosen": -0.9292968511581421,
      "logits/rejected": -0.830078125,
      "logps/chosen": -0.6573241949081421,
      "logps/rejected": -1.5830078125,
      "loss": 0.9153,
      "nll_loss": 0.859570324420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06575927883386612,
      "rewards/margins": 0.0926666259765625,
      "rewards/rejected": -0.15837402641773224,
      "step": 4100
    },
    {
      "epoch": 0.2992682127644082,
      "grad_norm": 2.3346024578564335,
      "learning_rate": 1.2478692301566601e-06,
      "log_odds_chosen": 1.7351562976837158,
      "log_odds_ratio": -0.35966795682907104,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.8623046875,
      "logps/chosen": -0.6783202886581421,
      "logps/rejected": -1.9562499523162842,
      "loss": 0.8957,
      "nll_loss": 0.820507824420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06782226264476776,
      "rewards/margins": 0.12806396186351776,
      "rewards/rejected": -0.19584961235523224,
      "step": 4110
    },
    {
      "epoch": 0.29999635926748464,
      "grad_norm": 2.093899272092189,
      "learning_rate": 1.2463539056995116e-06,
      "log_odds_chosen": 1.4303710460662842,
      "log_odds_ratio": -0.42631834745407104,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8685547113418579,
      "logps/chosen": -0.6587890386581421,
      "logps/rejected": -1.679296851158142,
      "loss": 0.9119,
      "nll_loss": 0.8560546636581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.10211181640625,
      "rewards/rejected": -0.16801758110523224,
      "step": 4120
    },
    {
      "epoch": 0.30072450577056103,
      "grad_norm": 2.1160267548154827,
      "learning_rate": 1.2448440881789541e-06,
      "log_odds_chosen": 1.455078125,
      "log_odds_ratio": -0.38618165254592896,
      "logits/chosen": -0.991406261920929,
      "logits/rejected": -0.8824218511581421,
      "logps/chosen": -0.6650390625,
      "logps/rejected": -1.6867187023162842,
      "loss": 0.8647,
      "nll_loss": 0.803906261920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06651611626148224,
      "rewards/margins": 0.10212402045726776,
      "rewards/rejected": -0.16862793266773224,
      "step": 4130
    },
    {
      "epoch": 0.30145265227363743,
      "grad_norm": 2.0121108703410564,
      "learning_rate": 1.2433397443204184e-06,
      "log_odds_chosen": 1.251855492591858,
      "log_odds_ratio": -0.4278808534145355,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.8490234613418579,
      "logps/chosen": -0.6845703125,
      "logps/rejected": -1.5427734851837158,
      "loss": 0.8885,
      "nll_loss": 0.8521484136581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06850586086511612,
      "rewards/margins": 0.08576659858226776,
      "rewards/rejected": -0.15432128310203552,
      "step": 4140
    },
    {
      "epoch": 0.3021807987767139,
      "grad_norm": 1.8965159591039014,
      "learning_rate": 1.2418408411301324e-06,
      "log_odds_chosen": 1.3927733898162842,
      "log_odds_ratio": -0.38300782442092896,
      "logits/chosen": -0.9759765863418579,
      "logits/rejected": -0.841601550579071,
      "logps/chosen": -0.652050793170929,
      "logps/rejected": -1.6193358898162842,
      "loss": 0.8692,
      "nll_loss": 0.873046875,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06522216647863388,
      "rewards/margins": 0.09677734225988388,
      "rewards/rejected": -0.16193847358226776,
      "step": 4150
    },
    {
      "epoch": 0.3029089452797903,
      "grad_norm": 2.031109667589268,
      "learning_rate": 1.2403473458920844e-06,
      "log_odds_chosen": 1.578222632408142,
      "log_odds_ratio": -0.4445556700229645,
      "logits/chosen": -0.9703124761581421,
      "logits/rejected": -0.8609374761581421,
      "logps/chosen": -0.716992199420929,
      "logps/rejected": -1.824609398841858,
      "loss": 0.9008,
      "nll_loss": 0.777148425579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07174072414636612,
      "rewards/margins": 0.11082763969898224,
      "rewards/rejected": -0.18242187798023224,
      "step": 4160
    },
    {
      "epoch": 0.3036370917828667,
      "grad_norm": 2.124461824682984,
      "learning_rate": 1.2388592261650217e-06,
      "log_odds_chosen": 1.429540991783142,
      "log_odds_ratio": -0.4297851622104645,
      "logits/chosen": -1.0119140148162842,
      "logits/rejected": -0.8755859136581421,
      "logps/chosen": -0.743945300579071,
      "logps/rejected": -1.7462890148162842,
      "loss": 0.8695,
      "nll_loss": 0.8453124761581421,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0743408203125,
      "rewards/margins": 0.10030059516429901,
      "rewards/rejected": -0.17470702528953552,
      "step": 4170
    },
    {
      "epoch": 0.30436523828594314,
      "grad_norm": 2.076458114968415,
      "learning_rate": 1.2373764497794918e-06,
      "log_odds_chosen": 1.379003882408142,
      "log_odds_ratio": -0.42900389432907104,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.799023449420929,
      "logps/chosen": -0.6749023199081421,
      "logps/rejected": -1.6179687976837158,
      "loss": 0.882,
      "nll_loss": 0.86328125,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06744384765625,
      "rewards/margins": 0.09448547661304474,
      "rewards/rejected": -0.16181640326976776,
      "step": 4180
    },
    {
      "epoch": 0.30509338478901954,
      "grad_norm": 2.48336575201867,
      "learning_rate": 1.2358989848349217e-06,
      "log_odds_chosen": 1.4592773914337158,
      "log_odds_ratio": -0.388671875,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.8876953125,
      "logps/chosen": -0.645312488079071,
      "logps/rejected": -1.64453125,
      "loss": 0.879,
      "nll_loss": 0.80078125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06447754055261612,
      "rewards/margins": 0.09986571967601776,
      "rewards/rejected": -0.16440430283546448,
      "step": 4190
    },
    {
      "epoch": 0.305821531292096,
      "grad_norm": 2.2489361849729494,
      "learning_rate": 1.2344267996967353e-06,
      "log_odds_chosen": 1.3708984851837158,
      "log_odds_ratio": -0.4219726622104645,
      "logits/chosen": -0.943164050579071,
      "logits/rejected": -0.8589843511581421,
      "logps/chosen": -0.6449218988418579,
      "logps/rejected": -1.6124999523162842,
      "loss": 0.8731,
      "nll_loss": 0.8408203125,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.09684447944164276,
      "rewards/rejected": -0.16147461533546448,
      "step": 4200
    },
    {
      "epoch": 0.3065496777951724,
      "grad_norm": 2.1237729151961005,
      "learning_rate": 1.2329598629935076e-06,
      "log_odds_chosen": 1.2802734375,
      "log_odds_ratio": -0.42949217557907104,
      "logits/chosen": -0.9775390625,
      "logits/rejected": -0.8671875,
      "logps/chosen": -0.636523425579071,
      "logps/rejected": -1.4904296398162842,
      "loss": 0.8851,
      "nll_loss": 0.8203125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06364746391773224,
      "rewards/margins": 0.08545227348804474,
      "rewards/rejected": -0.14907225966453552,
      "step": 4210
    },
    {
      "epoch": 0.3072778242982488,
      "grad_norm": 2.065119171306644,
      "learning_rate": 1.2314981436141583e-06,
      "log_odds_chosen": 1.4506103992462158,
      "log_odds_ratio": -0.417724609375,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.6475585699081421,
      "logps/rejected": -1.6593749523162842,
      "loss": 0.8656,
      "nll_loss": 0.8218749761581421,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06474609673023224,
      "rewards/margins": 0.1010536178946495,
      "rewards/rejected": -0.165771484375,
      "step": 4220
    },
    {
      "epoch": 0.30800597080132525,
      "grad_norm": 2.0380344009811897,
      "learning_rate": 1.2300416107051802e-06,
      "log_odds_chosen": 1.5849609375,
      "log_odds_ratio": -0.3866210877895355,
      "logits/chosen": -1.0109374523162842,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -0.6751953363418579,
      "logps/rejected": -1.793359398841858,
      "loss": 0.9131,
      "nll_loss": 0.829882800579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06752929836511612,
      "rewards/margins": 0.11180420219898224,
      "rewards/rejected": -0.1793212890625,
      "step": 4230
    },
    {
      "epoch": 0.30873411730440165,
      "grad_norm": 2.285835766158039,
      "learning_rate": 1.2285902336679024e-06,
      "log_odds_chosen": 1.495703101158142,
      "log_odds_ratio": -0.4005371034145355,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.9013671875,
      "logps/chosen": -0.6353515386581421,
      "logps/rejected": -1.6248047351837158,
      "loss": 0.8744,
      "nll_loss": 0.8404296636581421,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.0634765625,
      "rewards/margins": 0.09897460788488388,
      "rewards/rejected": -0.1624755859375,
      "step": 4240
    },
    {
      "epoch": 0.30946226380747804,
      "grad_norm": 1.9718872704477093,
      "learning_rate": 1.2271439821557926e-06,
      "log_odds_chosen": 1.4089844226837158,
      "log_odds_ratio": -0.4078613221645355,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.6654297113418579,
      "logps/rejected": -1.701562523841858,
      "loss": 0.8766,
      "nll_loss": 0.805859386920929,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06647948920726776,
      "rewards/margins": 0.10369873046875,
      "rewards/rejected": -0.17023925483226776,
      "step": 4250
    },
    {
      "epoch": 0.3101904103105545,
      "grad_norm": 2.252332268191545,
      "learning_rate": 1.225702826071791e-06,
      "log_odds_chosen": 1.5226562023162842,
      "log_odds_ratio": -0.3954101502895355,
      "logits/chosen": -0.954882800579071,
      "logits/rejected": -0.839648425579071,
      "logps/chosen": -0.698925793170929,
      "logps/rejected": -1.732812523841858,
      "loss": 0.889,
      "nll_loss": 0.8902343511581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06986083835363388,
      "rewards/margins": 0.10344238579273224,
      "rewards/rejected": -0.17336425185203552,
      "step": 4260
    },
    {
      "epoch": 0.3109185568136309,
      "grad_norm": 1.9211632394092473,
      "learning_rate": 1.2242667355656797e-06,
      "log_odds_chosen": 1.467187523841858,
      "log_odds_ratio": -0.383544921875,
      "logits/chosen": -0.946484386920929,
      "logits/rejected": -0.859375,
      "logps/chosen": -0.636523425579071,
      "logps/rejected": -1.6613280773162842,
      "loss": 0.8854,
      "nll_loss": 0.864062488079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06357421725988388,
      "rewards/margins": 0.10262451320886612,
      "rewards/rejected": -0.16630859673023224,
      "step": 4270
    },
    {
      "epoch": 0.3116467033167073,
      "grad_norm": 3.1950683903101416,
      "learning_rate": 1.2228356810314862e-06,
      "log_odds_chosen": 1.430090308189392,
      "log_odds_ratio": -0.44428712129592896,
      "logits/chosen": -0.9615234136581421,
      "logits/rejected": -0.8720703125,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.752343773841858,
      "loss": 0.9026,
      "nll_loss": 0.8701171875,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06983642280101776,
      "rewards/margins": 0.10538940131664276,
      "rewards/rejected": -0.17524413764476776,
      "step": 4280
    },
    {
      "epoch": 0.31237484981978375,
      "grad_norm": 2.3402498377339014,
      "learning_rate": 1.2214096331049186e-06,
      "log_odds_chosen": 1.1662108898162842,
      "log_odds_ratio": -0.45561522245407104,
      "logits/chosen": -0.9818359613418579,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -0.6988281011581421,
      "logps/rejected": -1.468164086341858,
      "loss": 0.8798,
      "nll_loss": 0.868945300579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06993408501148224,
      "rewards/margins": 0.07685241848230362,
      "rewards/rejected": -0.14670410752296448,
      "step": 4290
    },
    {
      "epoch": 0.31310299632286015,
      "grad_norm": 1.9221800339246609,
      "learning_rate": 1.2199885626608373e-06,
      "log_odds_chosen": 1.63818359375,
      "log_odds_ratio": -0.35249024629592896,
      "logits/chosen": -0.961718738079071,
      "logits/rejected": -0.845507800579071,
      "logps/chosen": -0.644726574420929,
      "logps/rejected": -1.7761719226837158,
      "loss": 0.8947,
      "nll_loss": 0.9326171875,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06450195610523224,
      "rewards/margins": 0.11326904594898224,
      "rewards/rejected": -0.1776123046875,
      "step": 4300
    },
    {
      "epoch": 0.3138311428259366,
      "grad_norm": 2.2775132313788222,
      "learning_rate": 1.2185724408107546e-06,
      "log_odds_chosen": 1.50390625,
      "log_odds_ratio": -0.41743165254592896,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -0.7251952886581421,
      "logps/rejected": -1.7644531726837158,
      "loss": 0.8963,
      "nll_loss": 0.877148449420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.07254638522863388,
      "rewards/margins": 0.10397949069738388,
      "rewards/rejected": -0.17641600966453552,
      "step": 4310
    },
    {
      "epoch": 0.314559289329013,
      "grad_norm": 2.464766740645121,
      "learning_rate": 1.2171612389003689e-06,
      "log_odds_chosen": 1.5910155773162842,
      "log_odds_ratio": -0.3652099668979645,
      "logits/chosen": -0.9908202886581421,
      "logits/rejected": -0.87109375,
      "logps/chosen": -0.6219726800918579,
      "logps/rejected": -1.7374999523162842,
      "loss": 0.8766,
      "nll_loss": 0.831835925579071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.062255859375,
      "rewards/margins": 0.11158446967601776,
      "rewards/rejected": -0.17397460341453552,
      "step": 4320
    },
    {
      "epoch": 0.3152874358320894,
      "grad_norm": 2.0525809499950127,
      "learning_rate": 1.2157549285071297e-06,
      "log_odds_chosen": 1.4373047351837158,
      "log_odds_ratio": -0.410888671875,
      "logits/chosen": -0.9339843988418579,
      "logits/rejected": -0.836718738079071,
      "logps/chosen": -0.654101550579071,
      "logps/rejected": -1.668359398841858,
      "loss": 0.9031,
      "nll_loss": 0.8726562261581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06540527194738388,
      "rewards/margins": 0.1014404296875,
      "rewards/rejected": -0.16696777939796448,
      "step": 4330
    },
    {
      "epoch": 0.31601558233516586,
      "grad_norm": 1.896973951984371,
      "learning_rate": 1.2143534814378327e-06,
      "log_odds_chosen": 1.4669921398162842,
      "log_odds_ratio": -0.3897949159145355,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.893359363079071,
      "logps/chosen": -0.661328136920929,
      "logps/rejected": -1.6886718273162842,
      "loss": 0.8857,
      "nll_loss": 0.841015636920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06613769382238388,
      "rewards/margins": 0.10271759331226349,
      "rewards/rejected": -0.16879883408546448,
      "step": 4340
    },
    {
      "epoch": 0.31674372883824226,
      "grad_norm": 1.9657386303024802,
      "learning_rate": 1.2129568697262454e-06,
      "log_odds_chosen": 1.68994140625,
      "log_odds_ratio": -0.34111326932907104,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.8470703363418579,
      "logps/chosen": -0.668164074420929,
      "logps/rejected": -1.866796851158142,
      "loss": 0.8917,
      "nll_loss": 0.8833984136581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06678466498851776,
      "rewards/margins": 0.11994628608226776,
      "rewards/rejected": -0.186767578125,
      "step": 4350
    },
    {
      "epoch": 0.31747187534131865,
      "grad_norm": 3.417863502867882,
      "learning_rate": 1.2115650656307653e-06,
      "log_odds_chosen": 1.6335937976837158,
      "log_odds_ratio": -0.36455076932907104,
      "logits/chosen": -1.0027344226837158,
      "logits/rejected": -0.864453136920929,
      "logps/chosen": -0.664843738079071,
      "logps/rejected": -1.7878906726837158,
      "loss": 0.898,
      "nll_loss": 0.762499988079071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06635741889476776,
      "rewards/margins": 0.11236572265625,
      "rewards/rejected": -0.17880859971046448,
      "step": 4360
    },
    {
      "epoch": 0.3182000218443951,
      "grad_norm": 2.504394611462869,
      "learning_rate": 1.210178041632103e-06,
      "log_odds_chosen": 1.4513671398162842,
      "log_odds_ratio": -0.4164062440395355,
      "logits/chosen": -0.9330078363418579,
      "logits/rejected": -0.824023425579071,
      "logps/chosen": -0.6561523675918579,
      "logps/rejected": -1.6628906726837158,
      "loss": 0.8858,
      "nll_loss": 0.832226574420929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.065673828125,
      "rewards/margins": 0.100738525390625,
      "rewards/rejected": -0.16628417372703552,
      "step": 4370
    },
    {
      "epoch": 0.3189281683474715,
      "grad_norm": 1.8810634976890226,
      "learning_rate": 1.2087957704309988e-06,
      "log_odds_chosen": 1.55859375,
      "log_odds_ratio": -0.3944091796875,
      "logits/chosen": -0.923046886920929,
      "logits/rejected": -0.8140624761581421,
      "logps/chosen": -0.6456054449081421,
      "logps/rejected": -1.705078125,
      "loss": 0.8665,
      "nll_loss": 0.788867175579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06455077975988388,
      "rewards/margins": 0.10596923530101776,
      "rewards/rejected": -0.17060546576976776,
      "step": 4380
    },
    {
      "epoch": 0.3196563148505479,
      "grad_norm": 2.2102323782714106,
      "learning_rate": 1.2074182249459642e-06,
      "log_odds_chosen": 1.691308617591858,
      "log_odds_ratio": -0.3719726502895355,
      "logits/chosen": -0.9886718988418579,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -0.6771484613418579,
      "logps/rejected": -1.8957030773162842,
      "loss": 0.8648,
      "nll_loss": 0.8330078125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06770019233226776,
      "rewards/margins": 0.12166748195886612,
      "rewards/rejected": -0.1895751953125,
      "step": 4390
    },
    {
      "epoch": 0.32038446135362436,
      "grad_norm": 2.1829900430349154,
      "learning_rate": 1.2060453783110545e-06,
      "log_odds_chosen": 1.3127930164337158,
      "log_odds_ratio": -0.40400391817092896,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.850781261920929,
      "logps/chosen": -0.634570300579071,
      "logps/rejected": -1.5166015625,
      "loss": 0.8658,
      "nll_loss": 0.888476550579071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06341552734375,
      "rewards/margins": 0.08817748725414276,
      "rewards/rejected": -0.15166015923023224,
      "step": 4400
    },
    {
      "epoch": 0.32111260785670076,
      "grad_norm": 2.2476756720566375,
      "learning_rate": 1.2046772038736682e-06,
      "log_odds_chosen": 1.4152343273162842,
      "log_odds_ratio": -0.405029296875,
      "logits/chosen": -0.974804699420929,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -0.6502929925918579,
      "logps/rejected": -1.6064453125,
      "loss": 0.8988,
      "nll_loss": 0.807812511920929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06501464545726776,
      "rewards/margins": 0.0955810546875,
      "rewards/rejected": -0.16066893935203552,
      "step": 4410
    },
    {
      "epoch": 0.3218407543597772,
      "grad_norm": 2.0704307264745987,
      "learning_rate": 1.2033136751923736e-06,
      "log_odds_chosen": 1.3499023914337158,
      "log_odds_ratio": -0.4185546934604645,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.873828113079071,
      "logps/chosen": -0.7044922113418579,
      "logps/rejected": -1.6570312976837158,
      "loss": 0.8806,
      "nll_loss": 0.819140613079071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07038573920726776,
      "rewards/margins": 0.09517212212085724,
      "rewards/rejected": -0.1656494140625,
      "step": 4420
    },
    {
      "epoch": 0.3225689008628536,
      "grad_norm": 1.9892425531399507,
      "learning_rate": 1.201954766034762e-06,
      "log_odds_chosen": 1.424414038658142,
      "log_odds_ratio": -0.44121092557907104,
      "logits/chosen": -0.9779297113418579,
      "logits/rejected": -0.8753906488418579,
      "logps/chosen": -0.6836913824081421,
      "logps/rejected": -1.696874976158142,
      "loss": 0.8941,
      "nll_loss": 0.821484386920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 0.10122070461511612,
      "rewards/rejected": -0.16960449516773224,
      "step": 4430
    },
    {
      "epoch": 0.32329704736593,
      "grad_norm": 3.6006385258020206,
      "learning_rate": 1.2006004503753285e-06,
      "log_odds_chosen": 1.3595702648162842,
      "log_odds_ratio": -0.4202636778354645,
      "logits/chosen": -0.9095703363418579,
      "logits/rejected": -0.8275390863418579,
      "logps/chosen": -0.675000011920929,
      "logps/rejected": -1.607812523841858,
      "loss": 0.886,
      "nll_loss": 0.888867199420929,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06749267876148224,
      "rewards/margins": 0.09318237006664276,
      "rewards/rejected": -0.16054686903953552,
      "step": 4440
    },
    {
      "epoch": 0.32402519386900647,
      "grad_norm": 2.5830185875644744,
      "learning_rate": 1.1992507023933782e-06,
      "log_odds_chosen": 1.631445288658142,
      "log_odds_ratio": -0.3603515625,
      "logits/chosen": -0.965039074420929,
      "logits/rejected": -0.8134765625,
      "logps/chosen": -0.667285144329071,
      "logps/rejected": -1.824609398841858,
      "loss": 0.9005,
      "nll_loss": 0.8824218511581421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06674804538488388,
      "rewards/margins": 0.11579589545726776,
      "rewards/rejected": -0.18242187798023224,
      "step": 4450
    },
    {
      "epoch": 0.32475334037208287,
      "grad_norm": 2.1135191752351172,
      "learning_rate": 1.1979054964709597e-06,
      "log_odds_chosen": 1.4690673351287842,
      "log_odds_ratio": -0.3809570372104645,
      "logits/chosen": -1.044921875,
      "logits/rejected": -0.838671863079071,
      "logps/chosen": -0.64990234375,
      "logps/rejected": -1.685156226158142,
      "loss": 0.8842,
      "nll_loss": 0.818652331829071,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06496582180261612,
      "rewards/margins": 0.10357513278722763,
      "rewards/rejected": -0.16848143935203552,
      "step": 4460
    },
    {
      "epoch": 0.32548148687515926,
      "grad_norm": 2.07056986607828,
      "learning_rate": 1.1965648071908207e-06,
      "log_odds_chosen": 1.7470703125,
      "log_odds_ratio": -0.32958984375,
      "logits/chosen": -1.0134766101837158,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -0.63916015625,
      "logps/rejected": -1.865625023841858,
      "loss": 0.8803,
      "nll_loss": 0.7881835699081421,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06387939304113388,
      "rewards/margins": 0.12250976264476776,
      "rewards/rejected": -0.18635253608226776,
      "step": 4470
    },
    {
      "epoch": 0.3262096333782357,
      "grad_norm": 2.3024637185666488,
      "learning_rate": 1.1952286093343935e-06,
      "log_odds_chosen": 1.3767578601837158,
      "log_odds_ratio": -0.39287108182907104,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.8423827886581421,
      "logps/chosen": -0.6591796875,
      "logps/rejected": -1.626562476158142,
      "loss": 0.9054,
      "nll_loss": 0.854687511920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06594238430261612,
      "rewards/margins": 0.09675292670726776,
      "rewards/rejected": -0.16276855766773224,
      "step": 4480
    },
    {
      "epoch": 0.3269377798813121,
      "grad_norm": 2.564076321797089,
      "learning_rate": 1.1938968778798005e-06,
      "log_odds_chosen": 1.4123046398162842,
      "log_odds_ratio": -0.429931640625,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.8746093511581421,
      "logps/chosen": -0.65576171875,
      "logps/rejected": -1.6462891101837158,
      "loss": 0.879,
      "nll_loss": 0.83984375,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06557617336511612,
      "rewards/margins": 0.098968505859375,
      "rewards/rejected": -0.1646728515625,
      "step": 4490
    },
    {
      "epoch": 0.3276659263843885,
      "grad_norm": 2.2782524821209047,
      "learning_rate": 1.1925695879998878e-06,
      "log_odds_chosen": 1.3969237804412842,
      "log_odds_ratio": -0.41254884004592896,
      "logits/chosen": -0.9498046636581421,
      "logits/rejected": -0.798828125,
      "logps/chosen": -0.6552734375,
      "logps/rejected": -1.6296875476837158,
      "loss": 0.8699,
      "nll_loss": 0.939648449420929,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06550292670726776,
      "rewards/margins": 0.09744872897863388,
      "rewards/rejected": -0.16291503608226776,
      "step": 4500
    },
    {
      "epoch": 0.32839407288746497,
      "grad_norm": 2.4019593561693893,
      "learning_rate": 1.1912467150602794e-06,
      "log_odds_chosen": 1.332421898841858,
      "log_odds_ratio": -0.40092772245407104,
      "logits/chosen": -0.9521484375,
      "logits/rejected": -0.863085925579071,
      "logps/chosen": -0.650585949420929,
      "logps/rejected": -1.593164086341858,
      "loss": 0.8698,
      "nll_loss": 0.786914050579071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06497802585363388,
      "rewards/margins": 0.094268798828125,
      "rewards/rejected": -0.15935058891773224,
      "step": 4510
    },
    {
      "epoch": 0.32912221939054137,
      "grad_norm": 2.0853329680026103,
      "learning_rate": 1.189928234617459e-06,
      "log_odds_chosen": 1.3391602039337158,
      "log_odds_ratio": -0.4043945372104645,
      "logits/chosen": -0.936328113079071,
      "logits/rejected": -0.804492175579071,
      "logps/chosen": -0.628710925579071,
      "logps/rejected": -1.5166015625,
      "loss": 0.8857,
      "nll_loss": 0.808789074420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06289062649011612,
      "rewards/margins": 0.08879394829273224,
      "rewards/rejected": -0.15163573622703552,
      "step": 4520
    },
    {
      "epoch": 0.32985036589361777,
      "grad_norm": 2.239330843999101,
      "learning_rate": 1.1886141224168716e-06,
      "log_odds_chosen": 1.5867187976837158,
      "log_odds_ratio": -0.3687500059604645,
      "logits/chosen": -1.0099608898162842,
      "logits/rejected": -0.889453113079071,
      "logps/chosen": -0.593066394329071,
      "logps/rejected": -1.6427733898162842,
      "loss": 0.8563,
      "nll_loss": 0.797656238079071,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05930175632238388,
      "rewards/margins": 0.10496826469898224,
      "rewards/rejected": -0.164306640625,
      "step": 4530
    },
    {
      "epoch": 0.3305785123966942,
      "grad_norm": 2.4957550857359103,
      "learning_rate": 1.1873043543910495e-06,
      "log_odds_chosen": 1.469335913658142,
      "log_odds_ratio": -0.43779295682907104,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.846484363079071,
      "logps/chosen": -0.6878906488418579,
      "logps/rejected": -1.716406226158142,
      "loss": 0.8721,
      "nll_loss": 0.8662109375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06878662109375,
      "rewards/margins": 0.10273437201976776,
      "rewards/rejected": -0.171630859375,
      "step": 4540
    },
    {
      "epoch": 0.3313066588997706,
      "grad_norm": 2.038865881363933,
      "learning_rate": 1.1859989066577617e-06,
      "log_odds_chosen": 1.75830078125,
      "log_odds_ratio": -0.35771483182907104,
      "logits/chosen": -0.991992175579071,
      "logits/rejected": -0.852734386920929,
      "logps/chosen": -0.635546863079071,
      "logps/rejected": -1.853906273841858,
      "loss": 0.8895,
      "nll_loss": 0.8270508050918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06352539360523224,
      "rewards/margins": 0.12188720703125,
      "rewards/rejected": -0.1854248046875,
      "step": 4550
    },
    {
      "epoch": 0.3320348054028471,
      "grad_norm": 2.6070418764448737,
      "learning_rate": 1.1846977555181846e-06,
      "log_odds_chosen": 1.376074194908142,
      "log_odds_ratio": -0.4466308653354645,
      "logits/chosen": -1.0263671875,
      "logits/rejected": -0.9052734375,
      "logps/chosen": -0.6991211175918579,
      "logps/rejected": -1.6882812976837158,
      "loss": 0.8794,
      "nll_loss": 0.8667968511581421,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.06978759914636612,
      "rewards/margins": 0.098968505859375,
      "rewards/rejected": -0.16875000298023224,
      "step": 4560
    },
    {
      "epoch": 0.3327629519059235,
      "grad_norm": 2.407773651801329,
      "learning_rate": 1.1834008774550946e-06,
      "log_odds_chosen": 1.5021483898162842,
      "log_odds_ratio": -0.3895507752895355,
      "logits/chosen": -0.941210925579071,
      "logits/rejected": -0.8443359136581421,
      "logps/chosen": -0.6607421636581421,
      "logps/rejected": -1.7197265625,
      "loss": 0.8819,
      "nll_loss": 0.8734375238418579,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06606445461511612,
      "rewards/margins": 0.10600586235523224,
      "rewards/rejected": -0.17219237983226776,
      "step": 4570
    },
    {
      "epoch": 0.3334910984089999,
      "grad_norm": 2.277772879150597,
      "learning_rate": 1.1821082491310835e-06,
      "log_odds_chosen": 1.3922851085662842,
      "log_odds_ratio": -0.4399169981479645,
      "logits/chosen": -0.9505859613418579,
      "logits/rejected": -0.845703125,
      "logps/chosen": -0.644824206829071,
      "logps/rejected": -1.626367211341858,
      "loss": 0.8681,
      "nll_loss": 0.864062488079071,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06452636420726776,
      "rewards/margins": 0.09818725287914276,
      "rewards/rejected": -0.16252441704273224,
      "step": 4580
    },
    {
      "epoch": 0.33421924491207633,
      "grad_norm": 2.3603339414829607,
      "learning_rate": 1.1808198473867937e-06,
      "log_odds_chosen": 1.340673804283142,
      "log_odds_ratio": -0.4244628846645355,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.87890625,
      "logps/chosen": -0.6871093511581421,
      "logps/rejected": -1.626953125,
      "loss": 0.8486,
      "nll_loss": 0.8720703125,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06873778998851776,
      "rewards/margins": 0.09398803859949112,
      "rewards/rejected": -0.16264648735523224,
      "step": 4590
    },
    {
      "epoch": 0.33494739141515273,
      "grad_norm": 2.5944839025076405,
      "learning_rate": 1.179535649239177e-06,
      "log_odds_chosen": 1.4851562976837158,
      "log_odds_ratio": -0.3910156190395355,
      "logits/chosen": -1.013671875,
      "logits/rejected": -0.896679699420929,
      "logps/chosen": -0.6802734136581421,
      "logps/rejected": -1.7449219226837158,
      "loss": 0.8736,
      "nll_loss": 0.7977539300918579,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.068115234375,
      "rewards/margins": 0.10650940239429474,
      "rewards/rejected": -0.17453613877296448,
      "step": 4600
    },
    {
      "epoch": 0.3356755379182291,
      "grad_norm": 2.260596482611632,
      "learning_rate": 1.178255631879771e-06,
      "log_odds_chosen": 1.3974120616912842,
      "log_odds_ratio": -0.3992675840854645,
      "logits/chosen": -0.968554675579071,
      "logits/rejected": -0.8482421636581421,
      "logps/chosen": -0.6962890625,
      "logps/rejected": -1.6623046398162842,
      "loss": 0.8749,
      "nll_loss": 0.811328113079071,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06964111328125,
      "rewards/margins": 0.09650573879480362,
      "rewards/rejected": -0.16606445610523224,
      "step": 4610
    },
    {
      "epoch": 0.3364036844213056,
      "grad_norm": 1.9536849245962038,
      "learning_rate": 1.1769797726729992e-06,
      "log_odds_chosen": 1.396337866783142,
      "log_odds_ratio": -0.41582030057907104,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.877734363079071,
      "logps/chosen": -0.7064453363418579,
      "logps/rejected": -1.671484351158142,
      "loss": 0.8526,
      "nll_loss": 0.840624988079071,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.07059326022863388,
      "rewards/margins": 0.09651489555835724,
      "rewards/rejected": -0.16706542670726776,
      "step": 4620
    },
    {
      "epoch": 0.337131830924382,
      "grad_norm": 2.230046565070823,
      "learning_rate": 1.1757080491544881e-06,
      "log_odds_chosen": 1.401757836341858,
      "log_odds_ratio": -0.39111328125,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.741992175579071,
      "logps/rejected": -1.6984374523162842,
      "loss": 0.8935,
      "nll_loss": 0.9501953125,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07418213039636612,
      "rewards/margins": 0.09567870944738388,
      "rewards/rejected": -0.16989746689796448,
      "step": 4630
    },
    {
      "epoch": 0.3378599774274584,
      "grad_norm": 1.9072093471572202,
      "learning_rate": 1.1744404390294068e-06,
      "log_odds_chosen": 1.7760741710662842,
      "log_odds_ratio": -0.33867186307907104,
      "logits/chosen": -0.9867187738418579,
      "logits/rejected": -0.856249988079071,
      "logps/chosen": -0.684374988079071,
      "logps/rejected": -1.946874976158142,
      "loss": 0.8751,
      "nll_loss": 0.841503918170929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06846924126148224,
      "rewards/margins": 0.12633056938648224,
      "rewards/rejected": -0.1947021484375,
      "step": 4640
    },
    {
      "epoch": 0.33858812393053483,
      "grad_norm": 2.9103737377139853,
      "learning_rate": 1.1731769201708264e-06,
      "log_odds_chosen": 1.50244140625,
      "log_odds_ratio": -0.3816894590854645,
      "logits/chosen": -1.046289086341858,
      "logits/rejected": -0.912890613079071,
      "logps/chosen": -0.6953125,
      "logps/rejected": -1.7990233898162842,
      "loss": 0.8579,
      "nll_loss": 0.8062499761581421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06950683891773224,
      "rewards/margins": 0.11051025241613388,
      "rewards/rejected": -0.18017578125,
      "step": 4650
    },
    {
      "epoch": 0.33931627043361123,
      "grad_norm": 2.6792034460083145,
      "learning_rate": 1.1719174706180952e-06,
      "log_odds_chosen": 1.474707007408142,
      "log_odds_ratio": -0.43242186307907104,
      "logits/chosen": -0.9800781011581421,
      "logits/rejected": -0.9125000238418579,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -1.6671874523162842,
      "loss": 0.9048,
      "nll_loss": 0.78955078125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0640869140625,
      "rewards/margins": 0.10258789360523224,
      "rewards/rejected": -0.16660156846046448,
      "step": 4660
    },
    {
      "epoch": 0.3400444169366877,
      "grad_norm": 2.5399208455884783,
      "learning_rate": 1.1706620685752386e-06,
      "log_odds_chosen": 1.257177710533142,
      "log_odds_ratio": -0.4805664122104645,
      "logits/chosen": -0.956835925579071,
      "logits/rejected": -0.867382824420929,
      "logps/chosen": -0.723828136920929,
      "logps/rejected": -1.591406226158142,
      "loss": 0.8877,
      "nll_loss": 0.84912109375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07232666015625,
      "rewards/margins": 0.08670349419116974,
      "rewards/rejected": -0.15913085639476776,
      "step": 4670
    },
    {
      "epoch": 0.3407725634397641,
      "grad_norm": 2.047834837396672,
      "learning_rate": 1.1694106924093723e-06,
      "log_odds_chosen": 1.4031250476837158,
      "log_odds_ratio": -0.42241209745407104,
      "logits/chosen": -0.9638671875,
      "logits/rejected": -0.8892577886581421,
      "logps/chosen": -0.6836913824081421,
      "logps/rejected": -1.6593749523162842,
      "loss": 0.8832,
      "nll_loss": 0.8324218988418579,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06833495944738388,
      "rewards/margins": 0.09761810302734375,
      "rewards/rejected": -0.16604003310203552,
      "step": 4680
    },
    {
      "epoch": 0.3415007099428405,
      "grad_norm": 2.137081132408231,
      "learning_rate": 1.1681633206491381e-06,
      "log_odds_chosen": 1.425878882408142,
      "log_odds_ratio": -0.4407714903354645,
      "logits/chosen": -0.9945312738418579,
      "logits/rejected": -0.896484375,
      "logps/chosen": -0.6192382574081421,
      "logps/rejected": -1.577539086341858,
      "loss": 0.8742,
      "nll_loss": 0.796679675579071,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06186523288488388,
      "rewards/margins": 0.09583739936351776,
      "rewards/rejected": -0.15771484375,
      "step": 4690
    },
    {
      "epoch": 0.34222885644591694,
      "grad_norm": 2.268331223595524,
      "learning_rate": 1.1669199319831564e-06,
      "log_odds_chosen": 1.5496094226837158,
      "log_odds_ratio": -0.3848632872104645,
      "logits/chosen": -0.967968761920929,
      "logits/rejected": -0.826367199420929,
      "logps/chosen": -0.67578125,
      "logps/rejected": -1.769921898841858,
      "loss": 0.8945,
      "nll_loss": 0.921093761920929,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06759033352136612,
      "rewards/margins": 0.10950927436351776,
      "rewards/rejected": -0.17702636122703552,
      "step": 4700
    },
    {
      "epoch": 0.34295700294899334,
      "grad_norm": 2.199788262175794,
      "learning_rate": 1.1656805052584958e-06,
      "log_odds_chosen": 1.501074194908142,
      "log_odds_ratio": -0.3841796815395355,
      "logits/chosen": -0.969921886920929,
      "logits/rejected": -0.8642578125,
      "logps/chosen": -0.658496081829071,
      "logps/rejected": -1.702539086341858,
      "loss": 0.915,
      "nll_loss": 0.903515636920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06584472954273224,
      "rewards/margins": 0.10447998344898224,
      "rewards/rejected": -0.17021484673023224,
      "step": 4710
    },
    {
      "epoch": 0.34368514945206974,
      "grad_norm": 1.9882092965372364,
      "learning_rate": 1.164445019479164e-06,
      "log_odds_chosen": 1.6267578601837158,
      "log_odds_ratio": -0.36640626192092896,
      "logits/chosen": -1.016210913658142,
      "logits/rejected": -0.873046875,
      "logps/chosen": -0.6875,
      "logps/rejected": -1.8671875,
      "loss": 0.8537,
      "nll_loss": 0.8765624761581421,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06871338188648224,
      "rewards/margins": 0.11800537258386612,
      "rewards/rejected": -0.18671874701976776,
      "step": 4720
    },
    {
      "epoch": 0.3444132959551462,
      "grad_norm": 2.354587239165765,
      "learning_rate": 1.1632134538046105e-06,
      "log_odds_chosen": 1.417626976966858,
      "log_odds_ratio": -0.4208984375,
      "logits/chosen": -0.9916015863418579,
      "logits/rejected": -0.8705078363418579,
      "logps/chosen": -0.710156261920929,
      "logps/rejected": -1.7265625,
      "loss": 0.8859,
      "nll_loss": 0.879101574420929,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07102050632238388,
      "rewards/margins": 0.10151366889476776,
      "rewards/rejected": -0.17268066108226776,
      "step": 4730
    },
    {
      "epoch": 0.3451414424582226,
      "grad_norm": 1.9464886596116469,
      "learning_rate": 1.1619857875482536e-06,
      "log_odds_chosen": 1.6466796398162842,
      "log_odds_ratio": -0.33378905057907104,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.8714843988418579,
      "logps/chosen": -0.6234375238418579,
      "logps/rejected": -1.748046875,
      "loss": 0.8798,
      "nll_loss": 0.804882824420929,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06234131008386612,
      "rewards/margins": 0.11249999701976776,
      "rewards/rejected": -0.17478027939796448,
      "step": 4740
    },
    {
      "epoch": 0.345869588961299,
      "grad_norm": 3.075646143944494,
      "learning_rate": 1.1607620001760185e-06,
      "log_odds_chosen": 1.3464844226837158,
      "log_odds_ratio": -0.45014649629592896,
      "logits/chosen": -1.003320336341858,
      "logits/rejected": -0.9019531011581421,
      "logps/chosen": -0.678906261920929,
      "logps/rejected": -1.6335937976837158,
      "loss": 0.868,
      "nll_loss": 0.8482421636581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06790771335363388,
      "rewards/margins": 0.09535522758960724,
      "rewards/rejected": -0.16337890923023224,
      "step": 4750
    },
    {
      "epoch": 0.34659773546437544,
      "grad_norm": 2.59789663066583,
      "learning_rate": 1.1595420713048968e-06,
      "log_odds_chosen": 1.5583984851837158,
      "log_odds_ratio": -0.3541503846645355,
      "logits/chosen": -1.022851586341858,
      "logits/rejected": -0.9115234613418579,
      "logps/chosen": -0.631152331829071,
      "logps/rejected": -1.6746094226837158,
      "loss": 0.8678,
      "nll_loss": 0.8160156011581421,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06315918266773224,
      "rewards/margins": 0.10427246242761612,
      "rewards/rejected": -0.16745606064796448,
      "step": 4760
    },
    {
      "epoch": 0.34732588196745184,
      "grad_norm": 2.225948237085341,
      "learning_rate": 1.1583259807015182e-06,
      "log_odds_chosen": 1.2248046398162842,
      "log_odds_ratio": -0.44438475370407104,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.884960949420929,
      "logps/chosen": -0.646191418170929,
      "logps/rejected": -1.4443359375,
      "loss": 0.872,
      "nll_loss": 0.8084961175918579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06462402641773224,
      "rewards/margins": 0.07988281548023224,
      "rewards/rejected": -0.14439697563648224,
      "step": 4770
    },
    {
      "epoch": 0.34805402847052824,
      "grad_norm": 2.128644866021711,
      "learning_rate": 1.1571137082807434e-06,
      "log_odds_chosen": 1.5554687976837158,
      "log_odds_ratio": -0.4063964784145355,
      "logits/chosen": -0.976367175579071,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -0.6356445550918579,
      "logps/rejected": -1.732812523841858,
      "loss": 0.8591,
      "nll_loss": 0.792187511920929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06363525241613388,
      "rewards/margins": 0.10974731296300888,
      "rewards/rejected": -0.17331543564796448,
      "step": 4780
    },
    {
      "epoch": 0.3487821749736047,
      "grad_norm": 2.3594232231758054,
      "learning_rate": 1.155905234104269e-06,
      "log_odds_chosen": 1.499230980873108,
      "log_odds_ratio": -0.3814697265625,
      "logits/chosen": -0.9603515863418579,
      "logits/rejected": -0.8681640625,
      "logps/chosen": -0.676953136920929,
      "logps/rejected": -1.678125023841858,
      "loss": 0.8651,
      "nll_loss": 0.8160156011581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06767578423023224,
      "rewards/margins": 0.10012435913085938,
      "rewards/rejected": -0.16779784858226776,
      "step": 4790
    },
    {
      "epoch": 0.3495103214766811,
      "grad_norm": 2.3491760577048253,
      "learning_rate": 1.1547005383792516e-06,
      "log_odds_chosen": 1.634124755859375,
      "log_odds_ratio": -0.330810546875,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.840624988079071,
      "logps/chosen": -0.645800769329071,
      "logps/rejected": -1.775390625,
      "loss": 0.8776,
      "nll_loss": 0.8687499761581421,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.06458739936351776,
      "rewards/margins": 0.113037109375,
      "rewards/rejected": -0.17751464247703552,
      "step": 4800
    },
    {
      "epoch": 0.35023846797975755,
      "grad_norm": 1.973863230179253,
      "learning_rate": 1.1534996014569446e-06,
      "log_odds_chosen": 1.4659912586212158,
      "log_odds_ratio": -0.38203126192092896,
      "logits/chosen": -1.0087890625,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -0.6405273675918579,
      "logps/rejected": -1.6513671875,
      "loss": 0.8743,
      "nll_loss": 0.793749988079071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06407471001148224,
      "rewards/margins": 0.101226806640625,
      "rewards/rejected": -0.165283203125,
      "step": 4810
    },
    {
      "epoch": 0.35096661448283395,
      "grad_norm": 2.0577446395614634,
      "learning_rate": 1.1523024038313547e-06,
      "log_odds_chosen": 1.294921875,
      "log_odds_ratio": -0.435791015625,
      "logits/chosen": -0.9732421636581421,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -0.6376953125,
      "logps/rejected": -1.5041015148162842,
      "loss": 0.8745,
      "nll_loss": 0.828320324420929,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06373290717601776,
      "rewards/margins": 0.08674316108226776,
      "rewards/rejected": -0.15046386420726776,
      "step": 4820
    },
    {
      "epoch": 0.35169476098591035,
      "grad_norm": 2.6926917489276163,
      "learning_rate": 1.1511089261379083e-06,
      "log_odds_chosen": 1.4591796398162842,
      "log_odds_ratio": -0.39726561307907104,
      "logits/chosen": -1.0671875476837158,
      "logits/rejected": -0.907031238079071,
      "logps/chosen": -0.7035156488418579,
      "logps/rejected": -1.751562476158142,
      "loss": 0.8633,
      "nll_loss": 0.8384765386581421,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07032470405101776,
      "rewards/margins": 0.1048583984375,
      "rewards/rejected": -0.17524413764476776,
      "step": 4830
    },
    {
      "epoch": 0.3524229074889868,
      "grad_norm": 2.291019400783551,
      "learning_rate": 1.149919149152138e-06,
      "log_odds_chosen": 1.4088866710662842,
      "log_odds_ratio": -0.4153808653354645,
      "logits/chosen": -0.9287109375,
      "logits/rejected": -0.84375,
      "logps/chosen": -0.645703136920929,
      "logps/rejected": -1.598046898841858,
      "loss": 0.863,
      "nll_loss": 0.804492175579071,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06461181491613388,
      "rewards/margins": 0.09534911811351776,
      "rewards/rejected": -0.15986327826976776,
      "step": 4840
    },
    {
      "epoch": 0.3531510539920632,
      "grad_norm": 2.2606220666895687,
      "learning_rate": 1.148733053788381e-06,
      "log_odds_chosen": 1.420800805091858,
      "log_odds_ratio": -0.3935546875,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.8619140386581421,
      "logps/chosen": -0.6590820550918579,
      "logps/rejected": -1.666406273841858,
      "loss": 0.8788,
      "nll_loss": 0.817187488079071,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06586913764476776,
      "rewards/margins": 0.10074005275964737,
      "rewards/rejected": -0.16667480766773224,
      "step": 4850
    },
    {
      "epoch": 0.3538792004951396,
      "grad_norm": 2.3504972713576477,
      "learning_rate": 1.1475506210984938e-06,
      "log_odds_chosen": 1.300878882408142,
      "log_odds_ratio": -0.45026856660842896,
      "logits/chosen": -1.018945336341858,
      "logits/rejected": -0.9291015863418579,
      "logps/chosen": -0.6982421875,
      "logps/rejected": -1.591406226158142,
      "loss": 0.8908,
      "nll_loss": 0.865039050579071,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06978759914636612,
      "rewards/margins": 0.0892486572265625,
      "rewards/rejected": -0.1590576171875,
      "step": 4860
    },
    {
      "epoch": 0.35460734699821606,
      "grad_norm": 2.2880743419705962,
      "learning_rate": 1.1463718322705807e-06,
      "log_odds_chosen": 0.985644519329071,
      "log_odds_ratio": -0.53369140625,
      "logits/chosen": -0.959765613079071,
      "logits/rejected": -0.8207031488418579,
      "logps/chosen": -0.722460925579071,
      "logps/rejected": -1.4189453125,
      "loss": 0.8452,
      "nll_loss": 0.8466796875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07225342094898224,
      "rewards/margins": 0.06957397609949112,
      "rewards/rejected": -0.14189453423023224,
      "step": 4870
    },
    {
      "epoch": 0.35533549350129245,
      "grad_norm": 2.8795386427103398,
      "learning_rate": 1.1451966686277364e-06,
      "log_odds_chosen": 1.190820336341858,
      "log_odds_ratio": -0.45061033964157104,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.883984386920929,
      "logps/chosen": -0.6895507574081421,
      "logps/rejected": -1.504296898841858,
      "loss": 0.8681,
      "nll_loss": 0.8501952886581421,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0689697265625,
      "rewards/margins": 0.08150024712085724,
      "rewards/rejected": -0.15048828721046448,
      "step": 4880
    },
    {
      "epoch": 0.35606364000436885,
      "grad_norm": 1.9986231423832015,
      "learning_rate": 1.1440251116268034e-06,
      "log_odds_chosen": 1.464746117591858,
      "log_odds_ratio": -0.40302735567092896,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.8837890625,
      "logps/chosen": -0.656933605670929,
      "logps/rejected": -1.705468773841858,
      "loss": 0.8518,
      "nll_loss": 0.8050781488418579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06569824367761612,
      "rewards/margins": 0.10481567680835724,
      "rewards/rejected": -0.17043456435203552,
      "step": 4890
    },
    {
      "epoch": 0.3567917865074453,
      "grad_norm": 2.6600615434643466,
      "learning_rate": 1.1428571428571428e-06,
      "log_odds_chosen": 1.7117187976837158,
      "log_odds_ratio": -0.3340820372104645,
      "logits/chosen": -0.989062488079071,
      "logits/rejected": -0.879101574420929,
      "logps/chosen": -0.610546886920929,
      "logps/rejected": -1.823828101158142,
      "loss": 0.8539,
      "nll_loss": 0.7662109136581421,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.06103515625,
      "rewards/margins": 0.121337890625,
      "rewards/rejected": -0.18256835639476776,
      "step": 4900
    },
    {
      "epoch": 0.3575199330105217,
      "grad_norm": 2.214181737715167,
      "learning_rate": 1.14169274403942e-06,
      "log_odds_chosen": 1.3644530773162842,
      "log_odds_ratio": -0.4229492247104645,
      "logits/chosen": -0.9537109136581421,
      "logits/rejected": -0.858593761920929,
      "logps/chosen": -0.6826171875,
      "logps/rejected": -1.6257812976837158,
      "loss": 0.8884,
      "nll_loss": 0.8960937261581421,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.06820068508386612,
      "rewards/margins": 0.0943603515625,
      "rewards/rejected": -0.16249999403953552,
      "step": 4910
    },
    {
      "epoch": 0.35824807951359816,
      "grad_norm": 2.0686486680448475,
      "learning_rate": 1.140531897024402e-06,
      "log_odds_chosen": 1.179235816001892,
      "log_odds_ratio": -0.46845704317092896,
      "logits/chosen": -1.0265624523162842,
      "logits/rejected": -0.951171875,
      "logps/chosen": -0.6571289300918579,
      "logps/rejected": -1.465234398841858,
      "loss": 0.8642,
      "nll_loss": 0.830859363079071,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.06571044772863388,
      "rewards/margins": 0.08081817626953125,
      "rewards/rejected": -0.14655761420726776,
      "step": 4920
    },
    {
      "epoch": 0.35897622601667456,
      "grad_norm": 2.1231901713936514,
      "learning_rate": 1.13937458379177e-06,
      "log_odds_chosen": 1.32666015625,
      "log_odds_ratio": -0.4737792909145355,
      "logits/chosen": -0.9955078363418579,
      "logits/rejected": -0.8853515386581421,
      "logps/chosen": -0.741406261920929,
      "logps/rejected": -1.7136719226837158,
      "loss": 0.8584,
      "nll_loss": 0.8648437261581421,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.07415771484375,
      "rewards/margins": 0.09732665866613388,
      "rewards/rejected": -0.17141112685203552,
      "step": 4930
    },
    {
      "epoch": 0.35970437251975096,
      "grad_norm": 2.7596862788320116,
      "learning_rate": 1.1382207864489444e-06,
      "log_odds_chosen": 1.460546851158142,
      "log_odds_ratio": -0.37553709745407104,
      "logits/chosen": -0.9556640386581421,
      "logits/rejected": -0.8578125238418579,
      "logps/chosen": -0.6451171636581421,
      "logps/rejected": -1.633203148841858,
      "loss": 0.8596,
      "nll_loss": 0.82568359375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06451416015625,
      "rewards/margins": 0.09881591796875,
      "rewards/rejected": -0.16323241591453552,
      "step": 4940
    },
    {
      "epoch": 0.3604325190228274,
      "grad_norm": 2.521389359075402,
      "learning_rate": 1.1370704872299223e-06,
      "log_odds_chosen": 1.4609375,
      "log_odds_ratio": -0.4232421815395355,
      "logits/chosen": -1.005273461341858,
      "logits/rejected": -0.9105468988418579,
      "logps/chosen": -0.678417980670929,
      "logps/rejected": -1.701562523841858,
      "loss": 0.8595,
      "nll_loss": 0.8326171636581421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06784667819738388,
      "rewards/margins": 0.10222168266773224,
      "rewards/rejected": -0.16994628310203552,
      "step": 4950
    },
    {
      "epoch": 0.3611606655259038,
      "grad_norm": 2.584395192815235,
      "learning_rate": 1.1359236684941295e-06,
      "log_odds_chosen": 1.677636742591858,
      "log_odds_ratio": -0.35871583223342896,
      "logits/chosen": -0.928906261920929,
      "logits/rejected": -0.8324218988418579,
      "logps/chosen": -0.6353515386581421,
      "logps/rejected": -1.790429711341858,
      "loss": 0.8568,
      "nll_loss": 0.818164050579071,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.06345214694738388,
      "rewards/margins": 0.11562804877758026,
      "rewards/rejected": -0.17917481064796448,
      "step": 4960
    },
    {
      "epoch": 0.3618888120289802,
      "grad_norm": 2.27351688374288,
      "learning_rate": 1.1347803127252839e-06,
      "log_odds_chosen": 1.357031226158142,
      "log_odds_ratio": -0.4119628965854645,
      "logits/chosen": -0.9048827886581421,
      "logits/rejected": -0.831250011920929,
      "logps/chosen": -0.6314452886581421,
      "logps/rejected": -1.5712890625,
      "loss": 0.8505,
      "nll_loss": 0.792187511920929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06313476711511612,
      "rewards/margins": 0.09393920749425888,
      "rewards/rejected": -0.1571044921875,
      "step": 4970
    },
    {
      "epoch": 0.36261695853205667,
      "grad_norm": 2.7553717501760637,
      "learning_rate": 1.1336404025302715e-06,
      "log_odds_chosen": 1.403417944908142,
      "log_odds_ratio": -0.4190917909145355,
      "logits/chosen": -0.9869140386581421,
      "logits/rejected": -0.848828136920929,
      "logps/chosen": -0.7333008050918579,
      "logps/rejected": -1.7468750476837158,
      "loss": 0.8738,
      "nll_loss": 0.808789074420929,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.07329101860523224,
      "rewards/margins": 0.10150146484375,
      "rewards/rejected": -0.1746826171875,
      "step": 4980
    },
    {
      "epoch": 0.36334510503513306,
      "grad_norm": 2.0247286834775333,
      "learning_rate": 1.1325039206380352e-06,
      "log_odds_chosen": 1.5318725109100342,
      "log_odds_ratio": -0.402587890625,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.902539074420929,
      "logps/chosen": -0.64697265625,
      "logps/rejected": -1.6769530773162842,
      "loss": 0.8402,
      "nll_loss": 0.785351574420929,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.064697265625,
      "rewards/margins": 0.10296936333179474,
      "rewards/rejected": -0.16777344048023224,
      "step": 4990
    },
    {
      "epoch": 0.36407325153820946,
      "grad_norm": 2.926531941062753,
      "learning_rate": 1.131370849898476e-06,
      "log_odds_chosen": 1.635156273841858,
      "log_odds_ratio": -0.3763183653354645,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.8794921636581421,
      "logps/chosen": -0.680957019329071,
      "logps/rejected": -1.8406250476837158,
      "loss": 0.8686,
      "nll_loss": 0.830078125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06817626953125,
      "rewards/margins": 0.115966796875,
      "rewards/rejected": -0.18410643935203552,
      "step": 5000
    },
    {
      "epoch": 0.36407325153820946,
      "eval_log_odds_chosen": 1.1701252460479736,
      "eval_log_odds_ratio": -0.47759079933166504,
      "eval_logits/chosen": -0.8645420074462891,
      "eval_logits/rejected": -0.7723644971847534,
      "eval_logps/chosen": -0.7588062882423401,
      "eval_logps/rejected": -1.6227697134017944,
      "eval_loss": 1.188714861869812,
      "eval_nll_loss": 1.136938452720642,
      "eval_rewards/accuracies": 0.7338129281997681,
      "eval_rewards/chosen": -0.07588158547878265,
      "eval_rewards/margins": 0.08640365302562714,
      "eval_rewards/rejected": -0.16227255761623383,
      "eval_runtime": 1383.4765,
      "eval_samples_per_second": 70.728,
      "eval_steps_per_second": 1.105,
      "step": 5000
    }
  ],
  "logging_steps": 10,
  "max_steps": 13733,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}