diff --git "a/checkpoint-10000/trainer_state.json" "b/checkpoint-10000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-10000/trainer_state.json" @@ -0,0 +1,18071 @@ +{ + "best_metric": 1.1802747249603271, + "best_model_checkpoint": "models/llama3.2-3b-orpo-coarse/checkpoint-5000", + "epoch": 0.7588404917286387, + "eval_steps": 5000, + "global_step": 10000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0007588404917286386, + "grad_norm": 16.28905678970287, + "learning_rate": 8e-07, + "log_odds_chosen": 0.6555420160293579, + "log_odds_ratio": -0.68408203125, + "logits/chosen": -1.0910155773162842, + "logits/rejected": -1.1103515625, + "logps/chosen": -2.21875, + "logps/rejected": -2.825000047683716, + "loss": 2.518, + "nll_loss": 2.332812547683716, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.22182616591453552, + "rewards/margins": 0.060699462890625, + "rewards/rejected": -0.28266602754592896, + "step": 10 + }, + { + "epoch": 0.0015176809834572772, + "grad_norm": 8.977044251578562, + "learning_rate": 1.6e-06, + "log_odds_chosen": 0.5552002191543579, + "log_odds_ratio": -0.7049804925918579, + "logits/chosen": -1.138085961341858, + "logits/rejected": -1.131445288658142, + "logps/chosen": -2.3125, + "logps/rejected": -2.8218750953674316, + "loss": 2.3322, + "nll_loss": 2.2734375, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.23154297471046448, + "rewards/margins": 0.0509796142578125, + "rewards/rejected": -0.28271484375, + "step": 20 + }, + { + "epoch": 0.002276521475185916, + "grad_norm": 3.4584431033537384, + "learning_rate": 2.4e-06, + "log_odds_chosen": 0.6031738519668579, + "log_odds_ratio": -0.614941418170929, + "logits/chosen": -1.235742211341858, + "logits/rejected": -1.194726586341858, + "logps/chosen": -1.784765601158142, + "logps/rejected": -2.30078125, + "loss": 1.9747, + "nll_loss": 1.8878905773162842, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.178466796875, + "rewards/margins": 0.05157470703125, + "rewards/rejected": -0.22998046875, + "step": 30 + }, + { + "epoch": 0.0030353619669145544, + "grad_norm": 2.779230061008798, + "learning_rate": 3.2e-06, + "log_odds_chosen": 0.37614136934280396, + "log_odds_ratio": -0.7298828363418579, + "logits/chosen": -1.0929687023162842, + "logits/rejected": -1.0183594226837158, + "logps/chosen": -1.552734375, + "logps/rejected": -1.8781249523162842, + "loss": 1.795, + "nll_loss": 1.745703101158142, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15510253608226776, + "rewards/margins": 0.03264465183019638, + "rewards/rejected": -0.187744140625, + "step": 40 + }, + { + "epoch": 0.0037942024586431933, + "grad_norm": 2.5592134050477653, + "learning_rate": 4e-06, + "log_odds_chosen": 0.559399425983429, + "log_odds_ratio": -0.5757812261581421, + "logits/chosen": -0.9652343988418579, + "logits/rejected": -0.919726550579071, + "logps/chosen": -1.413671851158142, + "logps/rejected": -1.8718750476837158, + "loss": 1.7396, + "nll_loss": 1.596093773841858, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.141357421875, + "rewards/margins": 0.04580993577837944, + "rewards/rejected": -0.18730469048023224, + "step": 50 + }, + { + "epoch": 0.004553042950371832, + "grad_norm": 2.8450488945476566, + "learning_rate": 4.8e-06, + "log_odds_chosen": 0.5027710199356079, + "log_odds_ratio": -0.5972656011581421, + "logits/chosen": -1.0324218273162842, + "logits/rejected": -0.975390613079071, + "logps/chosen": -1.277929663658142, + "logps/rejected": -1.692968726158142, + "loss": 1.6585, + "nll_loss": 1.4763672351837158, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.12788085639476776, + "rewards/margins": 0.04137573391199112, + "rewards/rejected": -0.16914062201976776, + "step": 60 + }, + { + "epoch": 0.005311883442100471, + "grad_norm": 2.4981511166293764, + "learning_rate": 5.6e-06, + "log_odds_chosen": 0.5220092535018921, + "log_odds_ratio": -0.5845702886581421, + "logits/chosen": -1.106054663658142, + "logits/rejected": -1.019140601158142, + "logps/chosen": -1.224218726158142, + "logps/rejected": -1.671484351158142, + "loss": 1.6213, + "nll_loss": 1.501562476158142, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.12246093899011612, + "rewards/margins": 0.0447998046875, + "rewards/rejected": -0.167236328125, + "step": 70 + }, + { + "epoch": 0.006070723933829109, + "grad_norm": 2.6645809181218567, + "learning_rate": 6.4e-06, + "log_odds_chosen": 0.555920422077179, + "log_odds_ratio": -0.568652331829071, + "logits/chosen": -1.048437476158142, + "logits/rejected": -0.991992175579071, + "logps/chosen": -1.229882836341858, + "logps/rejected": -1.6925780773162842, + "loss": 1.5998, + "nll_loss": 1.505273461341858, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.12290038913488388, + "rewards/margins": 0.04631652683019638, + "rewards/rejected": -0.16923828423023224, + "step": 80 + }, + { + "epoch": 0.006829564425557748, + "grad_norm": 2.467324514034339, + "learning_rate": 7.2e-06, + "log_odds_chosen": 0.3549560606479645, + "log_odds_ratio": -0.6615234613418579, + "logits/chosen": -1.115625023841858, + "logits/rejected": -1.0144531726837158, + "logps/chosen": -1.228906273841858, + "logps/rejected": -1.5417969226837158, + "loss": 1.51, + "nll_loss": 1.5164062976837158, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -0.1229248046875, + "rewards/margins": 0.031341552734375, + "rewards/rejected": -0.154296875, + "step": 90 + }, + { + "epoch": 0.007588404917286387, + "grad_norm": 2.47471355596972, + "learning_rate": 8e-06, + "log_odds_chosen": 0.39606934785842896, + "log_odds_ratio": -0.6527343988418579, + "logits/chosen": -1.009179711341858, + "logits/rejected": -0.9609375, + "logps/chosen": -1.2267577648162842, + "logps/rejected": -1.552734375, + "loss": 1.5495, + "nll_loss": 1.5304687023162842, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.12258300930261612, + "rewards/margins": 0.03266448900103569, + "rewards/rejected": -0.15522460639476776, + "step": 100 + }, + { + "epoch": 0.008347245409015025, + "grad_norm": 2.02338598823769, + "learning_rate": 7.627700713964738e-06, + "log_odds_chosen": 0.5322631597518921, + "log_odds_ratio": -0.566601574420929, + "logits/chosen": -0.953125, + "logits/rejected": -0.888867199420929, + "logps/chosen": -1.1130859851837158, + "logps/rejected": -1.5285155773162842, + "loss": 1.4784, + "nll_loss": 1.435937523841858, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11125488579273224, + "rewards/margins": 0.04149017482995987, + "rewards/rejected": -0.15275879204273224, + "step": 110 + }, + { + "epoch": 0.009106085900743664, + "grad_norm": 1.7600136940465556, + "learning_rate": 7.3029674334022146e-06, + "log_odds_chosen": 0.585436999797821, + "log_odds_ratio": -0.55029296875, + "logits/chosen": -1.0187499523162842, + "logits/rejected": -0.9371093511581421, + "logps/chosen": -1.0310547351837158, + "logps/rejected": -1.4699218273162842, + "loss": 1.4271, + "nll_loss": 1.293554663658142, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.10307617485523224, + "rewards/margins": 0.0438995361328125, + "rewards/rejected": -0.14692382514476776, + "step": 120 + }, + { + "epoch": 0.009864926392472302, + "grad_norm": 1.6794517773793187, + "learning_rate": 7.016464154456233e-06, + "log_odds_chosen": 0.35838621854782104, + "log_odds_ratio": -0.6317383050918579, + "logits/chosen": -0.9859374761581421, + "logits/rejected": -0.9013671875, + "logps/chosen": -1.074609398841858, + "logps/rejected": -1.357031226158142, + "loss": 1.4294, + "nll_loss": 1.3332030773162842, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.10749511420726776, + "rewards/margins": 0.02816314622759819, + "rewards/rejected": -0.1356201171875, + "step": 130 + }, + { + "epoch": 0.010623766884200941, + "grad_norm": 1.515687206113085, + "learning_rate": 6.7612340378281325e-06, + "log_odds_chosen": 0.49846190214157104, + "log_odds_ratio": -0.5927734375, + "logits/chosen": -0.960742175579071, + "logits/rejected": -0.9111328125, + "logps/chosen": -0.993359386920929, + "logps/rejected": -1.381445288658142, + "loss": 1.3889, + "nll_loss": 1.2716796398162842, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09931640326976776, + "rewards/margins": 0.03876953199505806, + "rewards/rejected": -0.13808593153953552, + "step": 140 + }, + { + "epoch": 0.01138260737592958, + "grad_norm": 1.5746644221408093, + "learning_rate": 6.531972647421809e-06, + "log_odds_chosen": 0.4237060546875, + "log_odds_ratio": -0.609375, + "logits/chosen": -0.996874988079071, + "logits/rejected": -0.950976550579071, + "logps/chosen": -0.9847656488418579, + "logps/rejected": -1.307031273841858, + "loss": 1.3694, + "nll_loss": 1.302343726158142, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09843750298023224, + "rewards/margins": 0.032186128199100494, + "rewards/rejected": -0.13068847358226776, + "step": 150 + }, + { + "epoch": 0.012141447867658217, + "grad_norm": 1.7303046396071229, + "learning_rate": 6.3245553203367575e-06, + "log_odds_chosen": 0.4165405333042145, + "log_odds_ratio": -0.6290527582168579, + "logits/chosen": -1.0310547351837158, + "logits/rejected": -0.9527343511581421, + "logps/chosen": -1.0068359375, + "logps/rejected": -1.3210937976837158, + "loss": 1.3639, + "nll_loss": 1.32421875, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.10068359225988388, + "rewards/margins": 0.03147735446691513, + "rewards/rejected": -0.13212890923023224, + "step": 160 + }, + { + "epoch": 0.012900288359386856, + "grad_norm": 1.5892750852136275, + "learning_rate": 6.135719910778963e-06, + "log_odds_chosen": 0.43085938692092896, + "log_odds_ratio": -0.638378918170929, + "logits/chosen": -0.9833984375, + "logits/rejected": -0.942578136920929, + "logps/chosen": -0.980664074420929, + "logps/rejected": -1.329687476158142, + "loss": 1.3067, + "nll_loss": 1.208984375, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -0.09804687649011612, + "rewards/margins": 0.03497924655675888, + "rewards/rejected": -0.13300780951976776, + "step": 170 + }, + { + "epoch": 0.013659128851115495, + "grad_norm": 1.9895227129073771, + "learning_rate": 5.962847939999439e-06, + "log_odds_chosen": 0.4945312440395355, + "log_odds_ratio": -0.589648425579071, + "logits/chosen": -0.932421863079071, + "logits/rejected": -0.873828113079071, + "logps/chosen": -0.9701172113418579, + "logps/rejected": -1.3488280773162842, + "loss": 1.3382, + "nll_loss": 1.2804687023162842, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.09707031399011612, + "rewards/margins": 0.03778686374425888, + "rewards/rejected": -0.13481445610523224, + "step": 180 + }, + { + "epoch": 0.014417969342844134, + "grad_norm": 1.6480729869591368, + "learning_rate": 5.803810000880094e-06, + "log_odds_chosen": 0.4837402403354645, + "log_odds_ratio": -0.5904296636581421, + "logits/chosen": -0.9173828363418579, + "logits/rejected": -0.8873046636581421, + "logps/chosen": -0.9761718511581421, + "logps/rejected": -1.346289038658142, + "loss": 1.3017, + "nll_loss": 1.285546898841858, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.09760741889476776, + "rewards/margins": 0.03700103610754013, + "rewards/rejected": -0.13461914658546448, + "step": 190 + }, + { + "epoch": 0.015176809834572773, + "grad_norm": 1.503264102872236, + "learning_rate": 5.65685424949238e-06, + "log_odds_chosen": 0.4825683534145355, + "log_odds_ratio": -0.582226574420929, + "logits/chosen": -0.9466797113418579, + "logits/rejected": -0.870898425579071, + "logps/chosen": -0.939648449420929, + "logps/rejected": -1.3035156726837158, + "loss": 1.2894, + "nll_loss": 1.217187523841858, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.09396972507238388, + "rewards/margins": 0.036403656005859375, + "rewards/rejected": -0.13041992485523224, + "step": 200 + }, + { + "epoch": 0.015935650326301412, + "grad_norm": 1.6703995784714376, + "learning_rate": 5.5205244747388325e-06, + "log_odds_chosen": 0.6786133050918579, + "log_odds_ratio": -0.579882800579071, + "logits/chosen": -0.8832031488418579, + "logits/rejected": -0.862109363079071, + "logps/chosen": -0.9300781488418579, + "logps/rejected": -1.4582030773162842, + "loss": 1.2816, + "nll_loss": 1.2117187976837158, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -0.09296874701976776, + "rewards/margins": 0.05272827297449112, + "rewards/rejected": -0.14572754502296448, + "step": 210 + }, + { + "epoch": 0.01669449081803005, + "grad_norm": 1.5169765274707603, + "learning_rate": 5.393598899705936e-06, + "log_odds_chosen": 0.517822265625, + "log_odds_ratio": -0.616503894329071, + "logits/chosen": -0.9017578363418579, + "logits/rejected": -0.8421875238418579, + "logps/chosen": -0.894335925579071, + "logps/rejected": -1.3212890625, + "loss": 1.2526, + "nll_loss": 1.177148461341858, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.08945312350988388, + "rewards/margins": 0.04265594482421875, + "rewards/rejected": -0.13203124701976776, + "step": 220 + }, + { + "epoch": 0.01745333130975869, + "grad_norm": 1.4368746262060774, + "learning_rate": 5.275043787166296e-06, + "log_odds_chosen": 0.4863037168979645, + "log_odds_ratio": -0.578808605670929, + "logits/chosen": -0.8785156011581421, + "logits/rejected": -0.7623046636581421, + "logps/chosen": -0.9287109375, + "logps/rejected": -1.2966797351837158, + "loss": 1.2595, + "nll_loss": 1.2150390148162842, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.09282226860523224, + "rewards/margins": 0.03685913234949112, + "rewards/rejected": -0.129638671875, + "step": 230 + }, + { + "epoch": 0.018212171801487327, + "grad_norm": 1.5611472145780005, + "learning_rate": 5.163977794943223e-06, + "log_odds_chosen": 0.4346069395542145, + "log_odds_ratio": -0.597363293170929, + "logits/chosen": -0.8232421875, + "logits/rejected": -0.8089843988418579, + "logps/chosen": -0.871874988079071, + "logps/rejected": -1.18359375, + "loss": 1.2569, + "nll_loss": 1.143164038658142, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.08718261867761612, + "rewards/margins": 0.031207656487822533, + "rewards/rejected": -0.11833496391773224, + "step": 240 + }, + { + "epoch": 0.018971012293215964, + "grad_norm": 1.460241600287341, + "learning_rate": 5.059644256269407e-06, + "log_odds_chosen": 0.40449219942092896, + "log_odds_ratio": -0.616015613079071, + "logits/chosen": -0.8539062738418579, + "logits/rejected": -0.804492175579071, + "logps/chosen": -0.8667968511581421, + "logps/rejected": -1.163476586341858, + "loss": 1.2829, + "nll_loss": 1.1749999523162842, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08671875298023224, + "rewards/margins": 0.02962798997759819, + "rewards/rejected": -0.11635742336511612, + "step": 250 + }, + { + "epoch": 0.019729852784944605, + "grad_norm": 1.6013195827393891, + "learning_rate": 4.961389383568338e-06, + "log_odds_chosen": 0.56024169921875, + "log_odds_ratio": -0.573925793170929, + "logits/chosen": -0.863085925579071, + "logits/rejected": -0.7964843511581421, + "logps/chosen": -0.834765613079071, + "logps/rejected": -1.236328125, + "loss": 1.2298, + "nll_loss": 1.2373046875, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.08352050930261612, + "rewards/margins": 0.04019775241613388, + "rewards/rejected": -0.12368164211511612, + "step": 260 + }, + { + "epoch": 0.020488693276673242, + "grad_norm": 1.5935105244091246, + "learning_rate": 4.8686449556014755e-06, + "log_odds_chosen": 0.693359375, + "log_odds_ratio": -0.531054675579071, + "logits/chosen": -0.857617199420929, + "logits/rejected": -0.811718761920929, + "logps/chosen": -0.777148425579071, + "logps/rejected": -1.2861328125, + "loss": 1.2069, + "nll_loss": 1.126562476158142, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07766113430261612, + "rewards/margins": 0.05088958889245987, + "rewards/rejected": -0.12858887016773224, + "step": 270 + }, + { + "epoch": 0.021247533768401883, + "grad_norm": 1.3821618104135565, + "learning_rate": 4.780914437337574e-06, + "log_odds_chosen": 0.5213378667831421, + "log_odds_ratio": -0.592480480670929, + "logits/chosen": -0.8648437261581421, + "logits/rejected": -0.8041015863418579, + "logps/chosen": -0.811718761920929, + "logps/rejected": -1.173437476158142, + "loss": 1.2352, + "nll_loss": 1.216406226158142, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.08118896186351776, + "rewards/margins": 0.03626251220703125, + "rewards/rejected": -0.11740722507238388, + "step": 280 + }, + { + "epoch": 0.02200637426013052, + "grad_norm": 1.6247124157263477, + "learning_rate": 4.697761756117627e-06, + "log_odds_chosen": 0.590527355670929, + "log_odds_ratio": -0.5450195074081421, + "logits/chosen": -0.862109363079071, + "logits/rejected": -0.7880859375, + "logps/chosen": -0.7593749761581421, + "logps/rejected": -1.153906226158142, + "loss": 1.2422, + "nll_loss": 1.1925780773162842, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07601318508386612, + "rewards/margins": 0.03940429538488388, + "rewards/rejected": -0.1153564453125, + "step": 290 + }, + { + "epoch": 0.02276521475185916, + "grad_norm": 1.4425392059458433, + "learning_rate": 4.618802153517006e-06, + "log_odds_chosen": 0.539794921875, + "log_odds_ratio": -0.584277331829071, + "logits/chosen": -0.8628906011581421, + "logits/rejected": -0.816601574420929, + "logps/chosen": -0.7953125238418579, + "logps/rejected": -1.162109375, + "loss": 1.2105, + "nll_loss": 1.14453125, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -0.07958984375, + "rewards/margins": 0.03663330152630806, + "rewards/rejected": -0.11616210639476776, + "step": 300 + }, + { + "epoch": 0.023524055243587798, + "grad_norm": 1.4229245984267798, + "learning_rate": 4.543694673976518e-06, + "log_odds_chosen": 0.553759753704071, + "log_odds_ratio": -0.567187488079071, + "logits/chosen": -0.891406238079071, + "logits/rejected": -0.8333984613418579, + "logps/chosen": -0.7958984375, + "logps/rejected": -1.169335961341858, + "loss": 1.2145, + "nll_loss": 1.260351538658142, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07958984375, + "rewards/margins": 0.03729400783777237, + "rewards/rejected": -0.11689452826976776, + "step": 310 + }, + { + "epoch": 0.024282895735316435, + "grad_norm": 1.5337475875154933, + "learning_rate": 4.472135954999579e-06, + "log_odds_chosen": 0.5230346918106079, + "log_odds_ratio": -0.5665038824081421, + "logits/chosen": -0.8939453363418579, + "logits/rejected": -0.9115234613418579, + "logps/chosen": -0.8011718988418579, + "logps/rejected": -1.1394531726837158, + "loss": 1.2126, + "nll_loss": 1.1082031726837158, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.08017577975988388, + "rewards/margins": 0.03389739990234375, + "rewards/rejected": -0.11411132663488388, + "step": 320 + }, + { + "epoch": 0.025041736227045076, + "grad_norm": 1.4941611010961415, + "learning_rate": 4.403855060505443e-06, + "log_odds_chosen": 0.5206054449081421, + "log_odds_ratio": -0.6089843511581421, + "logits/chosen": -0.9638671875, + "logits/rejected": -0.9195312261581421, + "logps/chosen": -0.776562511920929, + "logps/rejected": -1.1339843273162842, + "loss": 1.1647, + "nll_loss": 1.0783202648162842, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -0.07766113430261612, + "rewards/margins": 0.03580169752240181, + "rewards/rejected": -0.11337890475988388, + "step": 330 + }, + { + "epoch": 0.025800576718773713, + "grad_norm": 8.234299330038246, + "learning_rate": 4.338609156373123e-06, + "log_odds_chosen": 0.559277355670929, + "log_odds_ratio": -0.579296886920929, + "logits/chosen": -0.9681640863418579, + "logits/rejected": -0.8949218988418579, + "logps/chosen": -0.7923828363418579, + "logps/rejected": -1.1804687976837158, + "loss": 1.1985, + "nll_loss": 1.120703101158142, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07926025241613388, + "rewards/margins": 0.03888397291302681, + "rewards/rejected": -0.11806640774011612, + "step": 340 + }, + { + "epoch": 0.026559417210502353, + "grad_norm": 1.4986198531651644, + "learning_rate": 4.27617987059879e-06, + "log_odds_chosen": 0.5880126953125, + "log_odds_ratio": -0.5694335699081421, + "logits/chosen": -0.9203125238418579, + "logits/rejected": -0.850390613079071, + "logps/chosen": -0.798828125, + "logps/rejected": -1.200781226158142, + "loss": 1.1852, + "nll_loss": 1.150976538658142, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07993163913488388, + "rewards/margins": 0.04015808179974556, + "rewards/rejected": -0.12001953274011612, + "step": 350 + }, + { + "epoch": 0.02731825770223099, + "grad_norm": 1.4802049838530484, + "learning_rate": 4.216370213557839e-06, + "log_odds_chosen": 0.5188354253768921, + "log_odds_ratio": -0.607617199420929, + "logits/chosen": -0.9332031011581421, + "logits/rejected": -0.862500011920929, + "logps/chosen": -0.8257812261581421, + "logps/rejected": -1.1884765625, + "loss": 1.2022, + "nll_loss": 1.246484398841858, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.08256836235523224, + "rewards/margins": 0.03617553785443306, + "rewards/rejected": -0.1187744140625, + "step": 360 + }, + { + "epoch": 0.02807709819395963, + "grad_norm": 1.4330692363937183, + "learning_rate": 4.15900195928029e-06, + "log_odds_chosen": 0.586865246295929, + "log_odds_ratio": -0.60498046875, + "logits/chosen": -0.9009765386581421, + "logits/rejected": -0.858203113079071, + "logps/chosen": -0.797656238079071, + "logps/rejected": -1.1896483898162842, + "loss": 1.2002, + "nll_loss": 1.1806640625, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.07974853366613388, + "rewards/margins": 0.03930206224322319, + "rewards/rejected": -0.11899413913488388, + "step": 370 + }, + { + "epoch": 0.02883593868568827, + "grad_norm": 1.4912577480503046, + "learning_rate": 4.103913408340617e-06, + "log_odds_chosen": 0.540576159954071, + "log_odds_ratio": -0.6109863519668579, + "logits/chosen": -0.9781249761581421, + "logits/rejected": -0.9087890386581421, + "logps/chosen": -0.785937488079071, + "logps/rejected": -1.1513671875, + "loss": 1.1835, + "nll_loss": 1.194726586341858, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07860107719898224, + "rewards/margins": 0.03663025051355362, + "rewards/rejected": -0.11527099460363388, + "step": 380 + }, + { + "epoch": 0.029594779177416906, + "grad_norm": 1.344518207009396, + "learning_rate": 4.050957468334666e-06, + "log_odds_chosen": 0.5313720703125, + "log_odds_ratio": -0.606249988079071, + "logits/chosen": -0.9189453125, + "logits/rejected": -0.8687499761581421, + "logps/chosen": -0.779101550579071, + "logps/rejected": -1.145117163658142, + "loss": 1.1957, + "nll_loss": 1.102929711341858, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -0.07797851413488388, + "rewards/margins": 0.03666992112994194, + "rewards/rejected": -0.11447753757238388, + "step": 390 + }, + { + "epoch": 0.030353619669145546, + "grad_norm": 4.3130074404618135, + "learning_rate": 4e-06, + "log_odds_chosen": 0.45671385526657104, + "log_odds_ratio": -0.6246093511581421, + "logits/chosen": -0.918164074420929, + "logits/rejected": -0.857617199420929, + "logps/chosen": -0.7632812261581421, + "logps/rejected": -1.0556640625, + "loss": 1.1896, + "nll_loss": 1.123632788658142, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.07639160007238388, + "rewards/margins": 0.02919769287109375, + "rewards/rejected": -0.10543213039636612, + "step": 400 + }, + { + "epoch": 0.031112460160874184, + "grad_norm": 1.3095212395183442, + "learning_rate": 3.950918386598359e-06, + "log_odds_chosen": 0.7026733160018921, + "log_odds_ratio": -0.5523437261581421, + "logits/chosen": -0.970507800579071, + "logits/rejected": -0.898632824420929, + "logps/chosen": -0.783007800579071, + "logps/rejected": -1.248437523841858, + "loss": 1.1875, + "nll_loss": 1.142187476158142, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07823486626148224, + "rewards/margins": 0.04652862623333931, + "rewards/rejected": -0.12485351413488388, + "step": 410 + }, + { + "epoch": 0.031871300652602824, + "grad_norm": 1.538770613712715, + "learning_rate": 3.903600291794132e-06, + "log_odds_chosen": 0.9018310308456421, + "log_odds_ratio": -0.4969726502895355, + "logits/chosen": -0.8892577886581421, + "logits/rejected": -0.81640625, + "logps/chosen": -0.752734363079071, + "logps/rejected": -1.372656226158142, + "loss": 1.1881, + "nll_loss": 1.146875023841858, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07523193210363388, + "rewards/margins": 0.06215057522058487, + "rewards/rejected": -0.13740234076976776, + "step": 420 + }, + { + "epoch": 0.032630141144331465, + "grad_norm": 1.5135129410829358, + "learning_rate": 3.857942577363297e-06, + "log_odds_chosen": 0.5770508050918579, + "log_odds_ratio": -0.5787109136581421, + "logits/chosen": -0.9457031488418579, + "logits/rejected": -0.872851550579071, + "logps/chosen": -0.789257824420929, + "logps/rejected": -1.173828125, + "loss": 1.1877, + "nll_loss": 1.115234375, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.07891845703125, + "rewards/margins": 0.03848876804113388, + "rewards/rejected": -0.11740722507238388, + "step": 430 + }, + { + "epoch": 0.0333889816360601, + "grad_norm": 1.5495511200199545, + "learning_rate": 3.813850356982369e-06, + "log_odds_chosen": 0.7400146722793579, + "log_odds_ratio": -0.5404297113418579, + "logits/chosen": -0.9517577886581421, + "logits/rejected": -0.914257824420929, + "logps/chosen": -0.779101550579071, + "logps/rejected": -1.2970702648162842, + "loss": 1.1755, + "nll_loss": 1.0986328125, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07794189453125, + "rewards/margins": 0.05182952806353569, + "rewards/rejected": -0.12973633408546448, + "step": 440 + }, + { + "epoch": 0.03414782212778874, + "grad_norm": 1.434457768263082, + "learning_rate": 3.7712361663282537e-06, + "log_odds_chosen": 0.7149658203125, + "log_odds_ratio": -0.53125, + "logits/chosen": -0.892382800579071, + "logits/rejected": -0.848437488079071, + "logps/chosen": -0.7767578363418579, + "logps/rejected": -1.2646484375, + "loss": 1.1691, + "nll_loss": 1.088476538658142, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07757568359375, + "rewards/margins": 0.04888000339269638, + "rewards/rejected": -0.12656250596046448, + "step": 450 + }, + { + "epoch": 0.03490666261951738, + "grad_norm": 1.4647624075894172, + "learning_rate": 3.730019232961255e-06, + "log_odds_chosen": 0.598742663860321, + "log_odds_ratio": -0.5794922113418579, + "logits/chosen": -0.846875011920929, + "logits/rejected": -0.7669922113418579, + "logps/chosen": -0.788281261920929, + "logps/rejected": -1.1970703601837158, + "loss": 1.1697, + "nll_loss": 1.101171851158142, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -0.07880859076976776, + "rewards/margins": 0.04080658033490181, + "rewards/rejected": -0.11972656100988388, + "step": 460 + }, + { + "epoch": 0.035665503111246014, + "grad_norm": 1.460032211323527, + "learning_rate": 3.6901248321155403e-06, + "log_odds_chosen": 0.535937488079071, + "log_odds_ratio": -0.59423828125, + "logits/chosen": -0.857226550579071, + "logits/rejected": -0.783984363079071, + "logps/chosen": -0.793164074420929, + "logps/rejected": -1.162695288658142, + "loss": 1.1848, + "nll_loss": 1.1906249523162842, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.07929687201976776, + "rewards/margins": 0.03700103610754013, + "rewards/rejected": -0.11623535305261612, + "step": 470 + }, + { + "epoch": 0.036424343602974654, + "grad_norm": 1.3524653524950496, + "learning_rate": 3.6514837167011073e-06, + "log_odds_chosen": 0.646289050579071, + "log_odds_ratio": -0.5396484136581421, + "logits/chosen": -0.888867199420929, + "logits/rejected": -0.8472656011581421, + "logps/chosen": -0.755664050579071, + "logps/rejected": -1.179296851158142, + "loss": 1.1176, + "nll_loss": 1.028906226158142, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.0755615234375, + "rewards/margins": 0.04240112379193306, + "rewards/rejected": -0.11789550632238388, + "step": 480 + }, + { + "epoch": 0.037183184094703295, + "grad_norm": 1.575634651364888, + "learning_rate": 3.6140316116210052e-06, + "log_odds_chosen": 0.5822998285293579, + "log_odds_ratio": -0.582324206829071, + "logits/chosen": -0.8666015863418579, + "logits/rejected": -0.790820300579071, + "logps/chosen": -0.757617175579071, + "logps/rejected": -1.1457030773162842, + "loss": 1.1698, + "nll_loss": 1.113671898841858, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07574462890625, + "rewards/margins": 0.03886108472943306, + "rewards/rejected": -0.1146240234375, + "step": 490 + }, + { + "epoch": 0.03794202458643193, + "grad_norm": 1.4509304015827453, + "learning_rate": 3.5777087639996634e-06, + "log_odds_chosen": 0.682324230670929, + "log_odds_ratio": -0.526074230670929, + "logits/chosen": -0.888867199420929, + "logits/rejected": -0.8194335699081421, + "logps/chosen": -0.7503906488418579, + "logps/rejected": -1.1746094226837158, + "loss": 1.1828, + "nll_loss": 1.1652343273162842, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.0750732421875, + "rewards/margins": 0.0423583984375, + "rewards/rejected": -0.117431640625, + "step": 500 + }, + { + "epoch": 0.03870086507816057, + "grad_norm": 1.8399283965320115, + "learning_rate": 3.5424595421603814e-06, + "log_odds_chosen": 0.82373046875, + "log_odds_ratio": -0.5107421875, + "logits/chosen": -0.8783203363418579, + "logits/rejected": -0.8070312738418579, + "logps/chosen": -0.736328125, + "logps/rejected": -1.3039062023162842, + "loss": 1.1434, + "nll_loss": 1.0751953125, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07369384914636612, + "rewards/margins": 0.05676879733800888, + "rewards/rejected": -0.13034668564796448, + "step": 510 + }, + { + "epoch": 0.03945970556988921, + "grad_norm": 1.476828967448819, + "learning_rate": 3.5082320772281165e-06, + "log_odds_chosen": 0.8194335699081421, + "log_odds_ratio": -0.5054687261581421, + "logits/chosen": -0.9585937261581421, + "logits/rejected": -0.8929687738418579, + "logps/chosen": -0.7845703363418579, + "logps/rejected": -1.348046898841858, + "loss": 1.1622, + "nll_loss": 1.1238281726837158, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07841797173023224, + "rewards/margins": 0.05634155124425888, + "rewards/rejected": -0.13469238579273224, + "step": 520 + }, + { + "epoch": 0.04021854606161785, + "grad_norm": 1.539593920271273, + "learning_rate": 3.474977942104555e-06, + "log_odds_chosen": 0.5797973871231079, + "log_odds_ratio": -0.5848633050918579, + "logits/chosen": -0.947460949420929, + "logits/rejected": -0.8843749761581421, + "logps/chosen": -0.773632824420929, + "logps/rejected": -1.160742163658142, + "loss": 1.1431, + "nll_loss": 1.1005859375, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07738037407398224, + "rewards/margins": 0.03878479078412056, + "rewards/rejected": -0.11613769829273224, + "step": 530 + }, + { + "epoch": 0.040977386553346484, + "grad_norm": 1.4842674812940488, + "learning_rate": 3.442651863295481e-06, + "log_odds_chosen": 0.652294933795929, + "log_odds_ratio": -0.5572265386581421, + "logits/chosen": -0.9189453125, + "logits/rejected": -0.8451172113418579, + "logps/chosen": -0.749804675579071, + "logps/rejected": -1.199609398841858, + "loss": 1.1613, + "nll_loss": 1.100976586341858, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0750732421875, + "rewards/margins": 0.04497985914349556, + "rewards/rejected": -0.12001953274011612, + "step": 540 + }, + { + "epoch": 0.041736227045075125, + "grad_norm": 1.5239315494051218, + "learning_rate": 3.4112114616897665e-06, + "log_odds_chosen": 0.730712890625, + "log_odds_ratio": -0.532910168170929, + "logits/chosen": -0.9126952886581421, + "logits/rejected": -0.8607422113418579, + "logps/chosen": -0.740039050579071, + "logps/rejected": -1.240234375, + "loss": 1.1523, + "nll_loss": 1.071874976158142, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07404784858226776, + "rewards/margins": 0.04995117336511612, + "rewards/rejected": -0.12397460639476776, + "step": 550 + }, + { + "epoch": 0.042495067536803766, + "grad_norm": 1.4941924034303562, + "learning_rate": 3.3806170189140663e-06, + "log_odds_chosen": 0.6442626714706421, + "log_odds_ratio": -0.5541015863418579, + "logits/chosen": -0.9439452886581421, + "logits/rejected": -0.8671875, + "logps/chosen": -0.7408202886581421, + "logps/rejected": -1.167578101158142, + "loss": 1.1318, + "nll_loss": 1.0128905773162842, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07408447563648224, + "rewards/margins": 0.04265136644244194, + "rewards/rejected": -0.11665038764476776, + "step": 560 + }, + { + "epoch": 0.0432539080285324, + "grad_norm": 1.5068041723722188, + "learning_rate": 3.350831266333564e-06, + "log_odds_chosen": 0.572314441204071, + "log_odds_ratio": -0.5923827886581421, + "logits/chosen": -0.942187488079071, + "logits/rejected": -0.8658202886581421, + "logps/chosen": -0.798632800579071, + "logps/rejected": -1.197656273841858, + "loss": 1.1688, + "nll_loss": 1.111718773841858, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.079833984375, + "rewards/margins": 0.03993072360754013, + "rewards/rejected": -0.11982421576976776, + "step": 570 + }, + { + "epoch": 0.04401274852026104, + "grad_norm": 1.4143791261814025, + "learning_rate": 3.3218191941495984e-06, + "log_odds_chosen": 0.768115222454071, + "log_odds_ratio": -0.4981445372104645, + "logits/chosen": -0.9248046875, + "logits/rejected": -0.864453136920929, + "logps/chosen": -0.746874988079071, + "logps/rejected": -1.2615234851837158, + "loss": 1.1714, + "nll_loss": 1.0695312023162842, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07470703125, + "rewards/margins": 0.05145568773150444, + "rewards/rejected": -0.12619629502296448, + "step": 580 + }, + { + "epoch": 0.04477158901198968, + "grad_norm": 1.5758760245545085, + "learning_rate": 3.293547878370473e-06, + "log_odds_chosen": 0.5545409917831421, + "log_odds_ratio": -0.612597644329071, + "logits/chosen": -0.8935546875, + "logits/rejected": -0.856249988079071, + "logps/chosen": -0.765429675579071, + "logps/rejected": -1.1183593273162842, + "loss": 1.1337, + "nll_loss": 1.1160156726837158, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.07655028998851776, + "rewards/margins": 0.03520049899816513, + "rewards/rejected": -0.11186523735523224, + "step": 590 + }, + { + "epoch": 0.04553042950371832, + "grad_norm": 1.4028074979920369, + "learning_rate": 3.2659863237109044e-06, + "log_odds_chosen": 0.966503918170929, + "log_odds_ratio": -0.4481445252895355, + "logits/chosen": -0.9169921875, + "logits/rejected": -0.8544921875, + "logps/chosen": -0.7320312261581421, + "logps/rejected": -1.3779296875, + "loss": 1.1331, + "nll_loss": 1.105859398841858, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.07319335639476776, + "rewards/margins": 0.06447754055261612, + "rewards/rejected": -0.1376953125, + "step": 600 + }, + { + "epoch": 0.046289269995446955, + "grad_norm": 1.49643970687548, + "learning_rate": 3.239105320715664e-06, + "log_odds_chosen": 0.6585937738418579, + "log_odds_ratio": -0.5478515625, + "logits/chosen": -0.9150390625, + "logits/rejected": -0.8597656488418579, + "logps/chosen": -0.741015613079071, + "logps/rejected": -1.168554663658142, + "loss": 1.1114, + "nll_loss": 1.037109375, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07413329929113388, + "rewards/margins": 0.04278564453125, + "rewards/rejected": -0.11684570461511612, + "step": 610 + }, + { + "epoch": 0.047048110487175596, + "grad_norm": 1.4603778539467505, + "learning_rate": 3.2128773156099956e-06, + "log_odds_chosen": 0.7294921875, + "log_odds_ratio": -0.5185546875, + "logits/chosen": -0.9248046875, + "logits/rejected": -0.872265636920929, + "logps/chosen": -0.7431640625, + "logps/rejected": -1.24609375, + "loss": 1.1383, + "nll_loss": 1.102148413658142, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07432861626148224, + "rewards/margins": 0.05032043531537056, + "rewards/rejected": -0.12458495795726776, + "step": 620 + }, + { + "epoch": 0.047806950978904236, + "grad_norm": 1.4270703213144766, + "learning_rate": 3.187276291558383e-06, + "log_odds_chosen": 0.694042980670929, + "log_odds_ratio": -0.52392578125, + "logits/chosen": -0.9664062261581421, + "logits/rejected": -0.9027343988418579, + "logps/chosen": -0.7904297113418579, + "logps/rejected": -1.267578125, + "loss": 1.1549, + "nll_loss": 1.1359374523162842, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07899169623851776, + "rewards/margins": 0.047760009765625, + "rewards/rejected": -0.126708984375, + "step": 630 + }, + { + "epoch": 0.04856579147063287, + "grad_norm": 1.3640593722657062, + "learning_rate": 3.1622776601683788e-06, + "log_odds_chosen": 0.6767578125, + "log_odds_ratio": -0.5350586175918579, + "logits/chosen": -0.9214843511581421, + "logits/rejected": -0.876171886920929, + "logps/chosen": -0.781054675579071, + "logps/rejected": -1.224218726158142, + "loss": 1.1219, + "nll_loss": 1.0398437976837158, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.078125, + "rewards/margins": 0.04420776292681694, + "rewards/rejected": -0.12226562201976776, + "step": 640 + }, + { + "epoch": 0.04932463196236151, + "grad_norm": 1.39289555170604, + "learning_rate": 3.1378581622109444e-06, + "log_odds_chosen": 0.5647217035293579, + "log_odds_ratio": -0.6126953363418579, + "logits/chosen": -1.0076172351837158, + "logits/rejected": -0.9320312738418579, + "logps/chosen": -0.8109375238418579, + "logps/rejected": -1.2166016101837158, + "loss": 1.1219, + "nll_loss": 1.067773461341858, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -0.08111572265625, + "rewards/margins": 0.040557097643613815, + "rewards/rejected": -0.12163086235523224, + "step": 650 + }, + { + "epoch": 0.05008347245409015, + "grad_norm": 1.5575966456743369, + "learning_rate": 3.113995776646092e-06, + "log_odds_chosen": 0.9054199457168579, + "log_odds_ratio": -0.4915527403354645, + "logits/chosen": -0.9623047113418579, + "logits/rejected": -0.898242175579071, + "logps/chosen": -0.7110351324081421, + "logps/rejected": -1.308203101158142, + "loss": 1.1272, + "nll_loss": 1.0681641101837158, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07109375298023224, + "rewards/margins": 0.05971984937787056, + "rewards/rejected": -0.13078613579273224, + "step": 660 + }, + { + "epoch": 0.05084231294581879, + "grad_norm": 1.5820498982958486, + "learning_rate": 3.090669637145023e-06, + "log_odds_chosen": 0.920214831829071, + "log_odds_ratio": -0.48076170682907104, + "logits/chosen": -0.999218761920929, + "logits/rejected": -0.9287109375, + "logps/chosen": -0.7611328363418579, + "logps/rejected": -1.382421851158142, + "loss": 1.1265, + "nll_loss": 1.099023461341858, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07607422024011612, + "rewards/margins": 0.0622406005859375, + "rewards/rejected": -0.13835449516773224, + "step": 670 + }, + { + "epoch": 0.051601153437547426, + "grad_norm": 1.412160791122473, + "learning_rate": 3.0678599553894814e-06, + "log_odds_chosen": 0.636523425579071, + "log_odds_ratio": -0.5533202886581421, + "logits/chosen": -0.931640625, + "logits/rejected": -0.8912109136581421, + "logps/chosen": -0.754687488079071, + "logps/rejected": -1.1794922351837158, + "loss": 1.1396, + "nll_loss": 1.0632812976837158, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07546386867761612, + "rewards/margins": 0.042449951171875, + "rewards/rejected": -0.11796875298023224, + "step": 680 + }, + { + "epoch": 0.052359993929276066, + "grad_norm": 1.6469723394316966, + "learning_rate": 3.0455479505075235e-06, + "log_odds_chosen": 0.900097668170929, + "log_odds_ratio": -0.47685545682907104, + "logits/chosen": -0.951171875, + "logits/rejected": -0.8818359375, + "logps/chosen": -0.7466796636581421, + "logps/rejected": -1.344140648841858, + "loss": 1.1208, + "nll_loss": 1.0744140148162842, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07470703125, + "rewards/margins": 0.05967559665441513, + "rewards/rejected": -0.13437500596046448, + "step": 690 + }, + { + "epoch": 0.05311883442100471, + "grad_norm": 2.299409754878046, + "learning_rate": 3.0237157840738173e-06, + "log_odds_chosen": 0.818359375, + "log_odds_ratio": -0.502246081829071, + "logits/chosen": -0.93359375, + "logits/rejected": -0.8402343988418579, + "logps/chosen": -0.728515625, + "logps/rejected": -1.2646484375, + "loss": 1.1038, + "nll_loss": 1.045507788658142, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.07291259616613388, + "rewards/margins": 0.05359496921300888, + "rewards/rejected": -0.12663574516773224, + "step": 700 + }, + { + "epoch": 0.05387767491273334, + "grad_norm": 1.4366530156011492, + "learning_rate": 3.002346500163206e-06, + "log_odds_chosen": 0.676806628704071, + "log_odds_ratio": -0.568066418170929, + "logits/chosen": -0.892578125, + "logits/rejected": -0.8421875238418579, + "logps/chosen": -0.7298828363418579, + "logps/rejected": -1.1785156726837158, + "loss": 1.0838, + "nll_loss": 1.046875, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.072998046875, + "rewards/margins": 0.04483947902917862, + "rewards/rejected": -0.1177978515625, + "step": 710 + }, + { + "epoch": 0.05463651540446198, + "grad_norm": 1.4091415441420312, + "learning_rate": 2.9814239699997195e-06, + "log_odds_chosen": 0.730639636516571, + "log_odds_ratio": -0.5448242425918579, + "logits/chosen": -0.9156249761581421, + "logits/rejected": -0.869921863079071, + "logps/chosen": -0.7748047113418579, + "logps/rejected": -1.267968773841858, + "loss": 1.1281, + "nll_loss": 1.166601538658142, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07746581733226776, + "rewards/margins": 0.04938201978802681, + "rewards/rejected": -0.1268310546875, + "step": 720 + }, + { + "epoch": 0.05539535589619062, + "grad_norm": 1.5428528298733337, + "learning_rate": 2.9609328407904207e-06, + "log_odds_chosen": 0.861193835735321, + "log_odds_ratio": -0.473388671875, + "logits/chosen": -0.983203113079071, + "logits/rejected": -0.8863281011581421, + "logps/chosen": -0.708984375, + "logps/rejected": -1.2703125476837158, + "loss": 1.0983, + "nll_loss": 1.019921898841858, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.07081298530101776, + "rewards/margins": 0.05611877515912056, + "rewards/rejected": -0.12700195610523224, + "step": 730 + }, + { + "epoch": 0.05615419638791926, + "grad_norm": 1.3181361407151604, + "learning_rate": 2.940858488375231e-06, + "log_odds_chosen": 0.657470703125, + "log_odds_ratio": -0.5528320074081421, + "logits/chosen": -0.9462890625, + "logits/rejected": -0.882031261920929, + "logps/chosen": -0.7880859375, + "logps/rejected": -1.2468750476837158, + "loss": 1.1081, + "nll_loss": 1.021484375, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.07878418266773224, + "rewards/margins": 0.04585418850183487, + "rewards/rejected": -0.12468262016773224, + "step": 740 + }, + { + "epoch": 0.056913036879647896, + "grad_norm": 1.4129096893123931, + "learning_rate": 2.9211869733608857e-06, + "log_odds_chosen": 0.70599365234375, + "log_odds_ratio": -0.537890613079071, + "logits/chosen": -0.947265625, + "logits/rejected": -0.892773449420929, + "logps/chosen": -0.759960949420929, + "logps/rejected": -1.2421875, + "loss": 1.1035, + "nll_loss": 1.037695288658142, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07593993842601776, + "rewards/margins": 0.04822998121380806, + "rewards/rejected": -0.124267578125, + "step": 750 + }, + { + "epoch": 0.05767187737137654, + "grad_norm": 1.4910014683581294, + "learning_rate": 2.901905000440047e-06, + "log_odds_chosen": 0.8280273675918579, + "log_odds_ratio": -0.504443347454071, + "logits/chosen": -0.961132824420929, + "logits/rejected": -0.8509765863418579, + "logps/chosen": -0.71484375, + "logps/rejected": -1.2763671875, + "loss": 1.1171, + "nll_loss": 1.068750023841858, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07152099907398224, + "rewards/margins": 0.05616455152630806, + "rewards/rejected": -0.12775878608226776, + "step": 760 + }, + { + "epoch": 0.05843071786310518, + "grad_norm": 1.367660612325947, + "learning_rate": 2.8829998806257885e-06, + "log_odds_chosen": 0.779040515422821, + "log_odds_ratio": -0.5166991949081421, + "logits/chosen": -0.934374988079071, + "logits/rejected": -0.845507800579071, + "logps/chosen": -0.681347668170929, + "logps/rejected": -1.188085913658142, + "loss": 1.1298, + "nll_loss": 1.0185546875, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.068115234375, + "rewards/margins": 0.05065765231847763, + "rewards/rejected": -0.11887206882238388, + "step": 770 + }, + { + "epoch": 0.05918955835483381, + "grad_norm": 1.5187404836399663, + "learning_rate": 2.8644594961577314e-06, + "log_odds_chosen": 0.7642577886581421, + "log_odds_ratio": -0.5435546636581421, + "logits/chosen": -0.9417968988418579, + "logits/rejected": -0.8802734613418579, + "logps/chosen": -0.768750011920929, + "logps/rejected": -1.3263671398162842, + "loss": 1.1197, + "nll_loss": 1.089453101158142, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.07689209282398224, + "rewards/margins": 0.05573882907629013, + "rewards/rejected": -0.13266602158546448, + "step": 780 + }, + { + "epoch": 0.05994839884656245, + "grad_norm": 1.3998297985203276, + "learning_rate": 2.84627226785928e-06, + "log_odds_chosen": 0.68170166015625, + "log_odds_ratio": -0.550976574420929, + "logits/chosen": -0.9701172113418579, + "logits/rejected": -0.875, + "logps/chosen": -0.741992175579071, + "logps/rejected": -1.1867187023162842, + "loss": 1.1149, + "nll_loss": 1.048242211341858, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07418213039636612, + "rewards/margins": 0.04451598972082138, + "rewards/rejected": -0.11870117485523224, + "step": 790 + }, + { + "epoch": 0.06070723933829109, + "grad_norm": 1.5365658645460811, + "learning_rate": 2.82842712474619e-06, + "log_odds_chosen": 0.8926757574081421, + "log_odds_ratio": -0.5009765625, + "logits/chosen": -0.938281238079071, + "logits/rejected": -0.870312511920929, + "logps/chosen": -0.7412109375, + "logps/rejected": -1.378320336341858, + "loss": 1.1089, + "nll_loss": 1.032812476158142, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07413329929113388, + "rewards/margins": 0.06382445991039276, + "rewards/rejected": -0.13789062201976776, + "step": 800 + }, + { + "epoch": 0.061466079830019726, + "grad_norm": 1.553258112910012, + "learning_rate": 2.810913475705226e-06, + "log_odds_chosen": 0.827832043170929, + "log_odds_ratio": -0.4930664002895355, + "logits/chosen": -0.9130859375, + "logits/rejected": -0.861523449420929, + "logps/chosen": -0.7025390863418579, + "logps/rejected": -1.2615234851837158, + "loss": 1.1092, + "nll_loss": 1.0056641101837158, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07026366889476776, + "rewards/margins": 0.0557861328125, + "rewards/rejected": -0.1259765625, + "step": 810 + }, + { + "epoch": 0.06222492032174837, + "grad_norm": 1.6754050719010933, + "learning_rate": 2.7937211830783128e-06, + "log_odds_chosen": 0.825244128704071, + "log_odds_ratio": -0.513720691204071, + "logits/chosen": -0.9908202886581421, + "logits/rejected": -0.9544922113418579, + "logps/chosen": -0.761523425579071, + "logps/rejected": -1.33203125, + "loss": 1.1177, + "nll_loss": 1.129296898841858, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.07611083984375, + "rewards/margins": 0.05705719068646431, + "rewards/rejected": -0.13322754204273224, + "step": 820 + }, + { + "epoch": 0.06298376081347701, + "grad_norm": 1.4397189392753629, + "learning_rate": 2.776840538002493e-06, + "log_odds_chosen": 0.8416992425918579, + "log_odds_ratio": -0.51953125, + "logits/chosen": -0.950976550579071, + "logits/rejected": -0.9068359136581421, + "logps/chosen": -0.737109363079071, + "logps/rejected": -1.309960961341858, + "loss": 1.0814, + "nll_loss": 0.9839843511581421, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07368163764476776, + "rewards/margins": 0.0572509765625, + "rewards/rejected": -0.1309814453125, + "step": 830 + }, + { + "epoch": 0.06374260130520565, + "grad_norm": 1.481525111906686, + "learning_rate": 2.7602622373694163e-06, + "log_odds_chosen": 0.6253417730331421, + "log_odds_ratio": -0.569628894329071, + "logits/chosen": -0.931835949420929, + "logits/rejected": -0.8716796636581421, + "logps/chosen": -0.7300781011581421, + "logps/rejected": -1.1677734851837158, + "loss": 1.1057, + "nll_loss": 1.015234351158142, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07297363132238388, + "rewards/margins": 0.04375610500574112, + "rewards/rejected": -0.1168212890625, + "step": 840 + }, + { + "epoch": 0.06450144179693429, + "grad_norm": 1.4528723364813, + "learning_rate": 2.743977362280141e-06, + "log_odds_chosen": 0.7959960699081421, + "log_odds_ratio": -0.5220702886581421, + "logits/chosen": -0.975390613079071, + "logits/rejected": -0.915234386920929, + "logps/chosen": -0.7105468511581421, + "logps/rejected": -1.235937476158142, + "loss": 1.1185, + "nll_loss": 1.0314452648162842, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07106933742761612, + "rewards/margins": 0.0523834228515625, + "rewards/rejected": -0.12353515625, + "step": 850 + }, + { + "epoch": 0.06526028228866293, + "grad_norm": 1.4155260042724327, + "learning_rate": 2.7279773578818937e-06, + "log_odds_chosen": 0.774218738079071, + "log_odds_ratio": -0.54150390625, + "logits/chosen": -0.9326171875, + "logits/rejected": -0.8871093988418579, + "logps/chosen": -0.7484375238418579, + "logps/rejected": -1.286523461341858, + "loss": 1.1145, + "nll_loss": 0.9873046875, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.07486572116613388, + "rewards/margins": 0.05366821214556694, + "rewards/rejected": -0.12849120795726776, + "step": 860 + }, + { + "epoch": 0.06601912278039156, + "grad_norm": 1.4672576973291955, + "learning_rate": 2.7122540144832417e-06, + "log_odds_chosen": 0.6922851800918579, + "log_odds_ratio": -0.5411132574081421, + "logits/chosen": -0.9400390386581421, + "logits/rejected": -0.856249988079071, + "logps/chosen": -0.7162109613418579, + "logps/rejected": -1.152929663658142, + "loss": 1.1071, + "nll_loss": 1.0554687976837158, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07163085788488388, + "rewards/margins": 0.04363097995519638, + "rewards/rejected": -0.11533202975988388, + "step": 870 + }, + { + "epoch": 0.0667779632721202, + "grad_norm": 1.4377110735801357, + "learning_rate": 2.696799449852968e-06, + "log_odds_chosen": 0.7808837890625, + "log_odds_ratio": -0.517578125, + "logits/chosen": -0.8804687261581421, + "logits/rejected": -0.841796875, + "logps/chosen": -0.744335949420929, + "logps/rejected": -1.2470703125, + "loss": 1.1171, + "nll_loss": 1.073828101158142, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07441405951976776, + "rewards/margins": 0.05037841945886612, + "rewards/rejected": -0.12473144382238388, + "step": 880 + }, + { + "epoch": 0.06753680376384884, + "grad_norm": 1.5523580294418937, + "learning_rate": 2.6816060926159636e-06, + "log_odds_chosen": 0.8277221918106079, + "log_odds_ratio": -0.506103515625, + "logits/chosen": -0.948046863079071, + "logits/rejected": -0.8501952886581421, + "logps/chosen": -0.705078125, + "logps/rejected": -1.2453124523162842, + "loss": 1.0915, + "nll_loss": 1.0099608898162842, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07052002102136612, + "rewards/margins": 0.05402832105755806, + "rewards/rejected": -0.12456054985523224, + "step": 890 + }, + { + "epoch": 0.06829564425557748, + "grad_norm": 1.3981673710781848, + "learning_rate": 2.6666666666666664e-06, + "log_odds_chosen": 0.8017578125, + "log_odds_ratio": -0.513134777545929, + "logits/chosen": -0.940625011920929, + "logits/rejected": -0.8638671636581421, + "logps/chosen": -0.7030273675918579, + "logps/rejected": -1.211328148841858, + "loss": 1.0997, + "nll_loss": 1.0662109851837158, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07032470405101776, + "rewards/margins": 0.050933837890625, + "rewards/rejected": -0.12116698920726776, + "step": 900 + }, + { + "epoch": 0.06905448474730612, + "grad_norm": 1.4703260458527112, + "learning_rate": 2.6519741765271837e-06, + "log_odds_chosen": 0.8729492425918579, + "log_odds_ratio": -0.4708007872104645, + "logits/chosen": -0.9691406488418579, + "logits/rejected": -0.8560546636581421, + "logps/chosen": -0.6851562261581421, + "logps/rejected": -1.2585937976837158, + "loss": 1.1185, + "nll_loss": 1.0173828601837158, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06849364936351776, + "rewards/margins": 0.05747680738568306, + "rewards/rejected": -0.12590332329273224, + "step": 910 + }, + { + "epoch": 0.06981332523903476, + "grad_norm": 1.7240865708769755, + "learning_rate": 2.637521893583148e-06, + "log_odds_chosen": 0.833544909954071, + "log_odds_ratio": -0.522167980670929, + "logits/chosen": -0.9603515863418579, + "logits/rejected": -0.895703136920929, + "logps/chosen": -0.737500011920929, + "logps/rejected": -1.32421875, + "loss": 1.1254, + "nll_loss": 1.0656249523162842, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07379150390625, + "rewards/margins": 0.05867920070886612, + "rewards/rejected": -0.13249512016773224, + "step": 920 + }, + { + "epoch": 0.0705721657307634, + "grad_norm": 1.596536890919686, + "learning_rate": 2.6233033431358115e-06, + "log_odds_chosen": 0.688720703125, + "log_odds_ratio": -0.5291992425918579, + "logits/chosen": -0.9482421875, + "logits/rejected": -0.8775390386581421, + "logps/chosen": -0.698046863079071, + "logps/rejected": -1.146093726158142, + "loss": 1.1058, + "nll_loss": 1.027929663658142, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.06984863430261612, + "rewards/margins": 0.04474487155675888, + "rewards/rejected": -0.11459960788488388, + "step": 930 + }, + { + "epoch": 0.07133100622249203, + "grad_norm": 1.5205195253800634, + "learning_rate": 2.6093122922137685e-06, + "log_odds_chosen": 0.684814453125, + "log_odds_ratio": -0.5577148199081421, + "logits/chosen": -0.9986327886581421, + "logits/rejected": -0.963671863079071, + "logps/chosen": -0.7222656011581421, + "logps/rejected": -1.1583983898162842, + "loss": 1.1012, + "nll_loss": 0.95703125, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07225342094898224, + "rewards/margins": 0.04364623874425888, + "rewards/rejected": -0.11591796576976776, + "step": 940 + }, + { + "epoch": 0.07208984671422067, + "grad_norm": 1.5984518956278635, + "learning_rate": 2.5955427380922006e-06, + "log_odds_chosen": 0.7061523199081421, + "log_odds_ratio": -0.5171874761581421, + "logits/chosen": -0.9800781011581421, + "logits/rejected": -0.8882812261581421, + "logps/chosen": -0.6806640625, + "logps/rejected": -1.127539038658142, + "loss": 1.1089, + "nll_loss": 1.0857422351837158, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.06800536811351776, + "rewards/margins": 0.04460601881146431, + "rewards/rejected": -0.1126708984375, + "step": 950 + }, + { + "epoch": 0.07284868720594931, + "grad_norm": 1.5002616023423914, + "learning_rate": 2.5819888974716113e-06, + "log_odds_chosen": 0.765625, + "log_odds_ratio": -0.555371105670929, + "logits/chosen": -0.9507812261581421, + "logits/rejected": -0.8667968511581421, + "logps/chosen": -0.7378906011581421, + "logps/rejected": -1.265234351158142, + "loss": 1.1068, + "nll_loss": 1.115234375, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -0.07376708835363388, + "rewards/margins": 0.05278320237994194, + "rewards/rejected": -0.12646484375, + "step": 960 + }, + { + "epoch": 0.07360752769767795, + "grad_norm": 1.4651067791697183, + "learning_rate": 2.5686451962717425e-06, + "log_odds_chosen": 0.7742065191268921, + "log_odds_ratio": -0.534130871295929, + "logits/chosen": -0.9517577886581421, + "logits/rejected": -0.864453136920929, + "logps/chosen": -0.68896484375, + "logps/rejected": -1.2111327648162842, + "loss": 1.0792, + "nll_loss": 1.058984398841858, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.06888427585363388, + "rewards/margins": 0.0521976463496685, + "rewards/rejected": -0.12116698920726776, + "step": 970 + }, + { + "epoch": 0.07436636818940659, + "grad_norm": 1.3951266787145413, + "learning_rate": 2.5555062599997596e-06, + "log_odds_chosen": 0.83544921875, + "log_odds_ratio": -0.53076171875, + "logits/chosen": -1.0031249523162842, + "logits/rejected": -0.918164074420929, + "logps/chosen": -0.7308593988418579, + "logps/rejected": -1.3103516101837158, + "loss": 1.0967, + "nll_loss": 1.1240234375, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.07314453274011612, + "rewards/margins": 0.05808105319738388, + "rewards/rejected": -0.13111571967601776, + "step": 980 + }, + { + "epoch": 0.07512520868113523, + "grad_norm": 1.5995079707109392, + "learning_rate": 2.5425669046549126e-06, + "log_odds_chosen": 0.740466296672821, + "log_odds_ratio": -0.553906261920929, + "logits/chosen": -0.985156238079071, + "logits/rejected": -0.911914050579071, + "logps/chosen": -0.7710937261581421, + "logps/rejected": -1.286523461341858, + "loss": 1.1016, + "nll_loss": 1.021875023841858, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.07707519829273224, + "rewards/margins": 0.05153656005859375, + "rewards/rejected": -0.12871094048023224, + "step": 990 + }, + { + "epoch": 0.07588404917286386, + "grad_norm": 1.6759497570386803, + "learning_rate": 2.5298221281347034e-06, + "log_odds_chosen": 1.0483887195587158, + "log_odds_ratio": -0.4666992127895355, + "logits/chosen": -1.021875023841858, + "logits/rejected": -0.919140636920929, + "logps/chosen": -0.7113281488418579, + "logps/rejected": -1.450585961341858, + "loss": 1.0925, + "nll_loss": 1.025781273841858, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.071044921875, + "rewards/margins": 0.07399749755859375, + "rewards/rejected": -0.14519043266773224, + "step": 1000 + }, + { + "epoch": 0.0766428896645925, + "grad_norm": 1.4286639544950612, + "learning_rate": 2.5172671021102103e-06, + "log_odds_chosen": 0.787768542766571, + "log_odds_ratio": -0.5474609136581421, + "logits/chosen": -0.984179675579071, + "logits/rejected": -0.923632800579071, + "logps/chosen": -0.750195324420929, + "logps/rejected": -1.2976562976837158, + "loss": 1.0689, + "nll_loss": 1.0556640625, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07501220703125, + "rewards/margins": 0.05474243313074112, + "rewards/rejected": -0.12978515028953552, + "step": 1010 + }, + { + "epoch": 0.07740173015632114, + "grad_norm": 1.6143279270234796, + "learning_rate": 2.504897164340598e-06, + "log_odds_chosen": 0.7210693359375, + "log_odds_ratio": -0.561718761920929, + "logits/chosen": -0.991015613079071, + "logits/rejected": -0.899609386920929, + "logps/chosen": -0.723925769329071, + "logps/rejected": -1.206640601158142, + "loss": 1.0968, + "nll_loss": 1.0457031726837158, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07241211086511612, + "rewards/margins": 0.04823608323931694, + "rewards/rejected": -0.12058105319738388, + "step": 1020 + }, + { + "epoch": 0.07816057064804978, + "grad_norm": 1.5764818072728237, + "learning_rate": 2.492707811399023e-06, + "log_odds_chosen": 1.0046875476837158, + "log_odds_ratio": -0.4649414122104645, + "logits/chosen": -1.0078125, + "logits/rejected": -0.9419921636581421, + "logps/chosen": -0.6382812261581421, + "logps/rejected": -1.292578101158142, + "loss": 1.0769, + "nll_loss": 0.949999988079071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06381835788488388, + "rewards/margins": 0.06541137397289276, + "rewards/rejected": -0.12924805283546448, + "step": 1030 + }, + { + "epoch": 0.07891941113977842, + "grad_norm": 1.558748180429937, + "learning_rate": 2.480694691784169e-06, + "log_odds_chosen": 0.866259753704071, + "log_odds_ratio": -0.484375, + "logits/chosen": -0.9994140863418579, + "logits/rejected": -0.906445324420929, + "logps/chosen": -0.678906261920929, + "logps/rejected": -1.224218726158142, + "loss": 1.1036, + "nll_loss": 1.056640625, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06789550930261612, + "rewards/margins": 0.05458679050207138, + "rewards/rejected": -0.12250976264476776, + "step": 1040 + }, + { + "epoch": 0.07967825163150706, + "grad_norm": 1.458824951812319, + "learning_rate": 2.4688535993934706e-06, + "log_odds_chosen": 0.820385754108429, + "log_odds_ratio": -0.5077148675918579, + "logits/chosen": -1.006445288658142, + "logits/rejected": -0.925976574420929, + "logps/chosen": -0.7259765863418579, + "logps/rejected": -1.2607421875, + "loss": 1.072, + "nll_loss": 0.988476574420929, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07264403998851776, + "rewards/margins": 0.05347289890050888, + "rewards/rejected": -0.12607422471046448, + "step": 1050 + }, + { + "epoch": 0.0804370921232357, + "grad_norm": 1.6596203290820475, + "learning_rate": 2.457180467335805e-06, + "log_odds_chosen": 0.642260730266571, + "log_odds_ratio": -0.6029297113418579, + "logits/chosen": -0.9710937738418579, + "logits/rejected": -0.8902343511581421, + "logps/chosen": -0.794921875, + "logps/rejected": -1.2492187023162842, + "loss": 1.0757, + "nll_loss": 1.033203125, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -0.0794677734375, + "rewards/margins": 0.04537353664636612, + "rewards/rejected": -0.12482909858226776, + "step": 1060 + }, + { + "epoch": 0.08119593261496433, + "grad_norm": 2.2419464467091217, + "learning_rate": 2.4456713620629725e-06, + "log_odds_chosen": 0.9271484613418579, + "log_odds_ratio": -0.5121093988418579, + "logits/chosen": -0.953125, + "logits/rejected": -0.878125011920929, + "logps/chosen": -0.707812488079071, + "logps/rejected": -1.287109375, + "loss": 1.1071, + "nll_loss": 1.131250023841858, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07080078125, + "rewards/margins": 0.05796203762292862, + "rewards/rejected": -0.128662109375, + "step": 1070 + }, + { + "epoch": 0.08195477310669297, + "grad_norm": 1.5134269175876427, + "learning_rate": 2.4343224778007378e-06, + "log_odds_chosen": 0.7594238519668579, + "log_odds_ratio": -0.514941394329071, + "logits/chosen": -0.9462890625, + "logits/rejected": -0.8998047113418579, + "logps/chosen": -0.724609375, + "logps/rejected": -1.2138671875, + "loss": 1.0804, + "nll_loss": 1.0427734851837158, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.07246093451976776, + "rewards/margins": 0.04888305813074112, + "rewards/rejected": -0.12131347507238388, + "step": 1080 + }, + { + "epoch": 0.08271361359842161, + "grad_norm": 1.525216640704541, + "learning_rate": 2.4231301312615306e-06, + "log_odds_chosen": 0.975512683391571, + "log_odds_ratio": -0.5089355707168579, + "logits/chosen": -0.9750000238418579, + "logits/rejected": -0.8974609375, + "logps/chosen": -0.7203124761581421, + "logps/rejected": -1.395898461341858, + "loss": 1.0719, + "nll_loss": 0.993359386920929, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07199706882238388, + "rewards/margins": 0.067596435546875, + "rewards/rejected": -0.13955077528953552, + "step": 1090 + }, + { + "epoch": 0.08347245409015025, + "grad_norm": 1.4027404178012934, + "learning_rate": 2.412090756622109e-06, + "log_odds_chosen": 0.7673705816268921, + "log_odds_ratio": -0.5249999761581421, + "logits/chosen": -0.951367199420929, + "logits/rejected": -0.884960949420929, + "logps/chosen": -0.738574206829071, + "logps/rejected": -1.2646484375, + "loss": 1.0735, + "nll_loss": 0.994921863079071, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07377929985523224, + "rewards/margins": 0.05257873609662056, + "rewards/rejected": -0.12646484375, + "step": 1100 + }, + { + "epoch": 0.08423129458187889, + "grad_norm": 1.6015960818651562, + "learning_rate": 2.401200900750657e-06, + "log_odds_chosen": 0.680188000202179, + "log_odds_ratio": -0.5425781011581421, + "logits/chosen": -0.969531238079071, + "logits/rejected": -0.9166015386581421, + "logps/chosen": -0.766406238079071, + "logps/rejected": -1.244531273841858, + "loss": 1.0568, + "nll_loss": 0.9955078363418579, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07664795219898224, + "rewards/margins": 0.04779357835650444, + "rewards/rejected": -0.12448730319738388, + "step": 1110 + }, + { + "epoch": 0.08499013507360753, + "grad_norm": 1.4462554991694485, + "learning_rate": 2.390457218668787e-06, + "log_odds_chosen": 0.910449206829071, + "log_odds_ratio": -0.4771484434604645, + "logits/chosen": -0.9351562261581421, + "logits/rejected": -0.878710925579071, + "logps/chosen": -0.7552734613418579, + "logps/rejected": -1.367773413658142, + "loss": 1.0932, + "nll_loss": 1.1320312023162842, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07550048828125, + "rewards/margins": 0.06117553636431694, + "rewards/rejected": -0.1365966796875, + "step": 1120 + }, + { + "epoch": 0.08574897556533617, + "grad_norm": 1.3771017614094005, + "learning_rate": 2.379856469234918e-06, + "log_odds_chosen": 0.89666748046875, + "log_odds_ratio": -0.4830078184604645, + "logits/chosen": -0.9310547113418579, + "logits/rejected": -0.878710925579071, + "logps/chosen": -0.725781261920929, + "logps/rejected": -1.322656273841858, + "loss": 1.0739, + "nll_loss": 1.034765601158142, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07258300483226776, + "rewards/margins": 0.05972747877240181, + "rewards/rejected": -0.13229981064796448, + "step": 1130 + }, + { + "epoch": 0.0865078160570648, + "grad_norm": 1.5724289030509337, + "learning_rate": 2.369395511036369e-06, + "log_odds_chosen": 0.9185791015625, + "log_odds_ratio": -0.48344725370407104, + "logits/chosen": -0.9781249761581421, + "logits/rejected": -0.904101550579071, + "logps/chosen": -0.7250000238418579, + "logps/rejected": -1.345703125, + "loss": 1.0835, + "nll_loss": 1.1042969226837158, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.07246093451976776, + "rewards/margins": 0.0621337890625, + "rewards/rejected": -0.13469238579273224, + "step": 1140 + }, + { + "epoch": 0.08726665654879344, + "grad_norm": 1.5105580372447085, + "learning_rate": 2.359071298478354e-06, + "log_odds_chosen": 0.7783447504043579, + "log_odds_ratio": -0.5380859375, + "logits/chosen": -0.984375, + "logits/rejected": -0.8990234136581421, + "logps/chosen": -0.73828125, + "logps/rejected": -1.2673828601837158, + "loss": 1.0531, + "nll_loss": 1.0275390148162842, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.07381591945886612, + "rewards/margins": 0.05291900783777237, + "rewards/rejected": -0.12678222358226776, + "step": 1150 + }, + { + "epoch": 0.08802549704052208, + "grad_norm": 1.5063726205865329, + "learning_rate": 2.3488808780588137e-06, + "log_odds_chosen": 0.800000011920929, + "log_odds_ratio": -0.5308593511581421, + "logits/chosen": -0.9722656011581421, + "logits/rejected": -0.8876953125, + "logps/chosen": -0.73486328125, + "logps/rejected": -1.263671875, + "loss": 1.0574, + "nll_loss": 1.08984375, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07351074367761612, + "rewards/margins": 0.05285186693072319, + "rewards/rejected": -0.12639160454273224, + "step": 1160 + }, + { + "epoch": 0.08878433753225072, + "grad_norm": 1.640218129700523, + "learning_rate": 2.3388213848187446e-06, + "log_odds_chosen": 0.7334228754043579, + "log_odds_ratio": -0.54052734375, + "logits/chosen": -0.974414050579071, + "logits/rejected": -0.880078136920929, + "logps/chosen": -0.7416015863418579, + "logps/rejected": -1.2433593273162842, + "loss": 1.0795, + "nll_loss": 0.9974609613418579, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07415771484375, + "rewards/margins": 0.05020294338464737, + "rewards/rejected": -0.12434081733226776, + "step": 1170 + }, + { + "epoch": 0.08954317802397936, + "grad_norm": 1.4607546113706387, + "learning_rate": 2.328890038958328e-06, + "log_odds_chosen": 0.8072265386581421, + "log_odds_ratio": -0.529101550579071, + "logits/chosen": -0.927929699420929, + "logits/rejected": -0.8916015625, + "logps/chosen": -0.747265636920929, + "logps/rejected": -1.2888672351837158, + "loss": 1.084, + "nll_loss": 1.023828148841858, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07471923530101776, + "rewards/margins": 0.05414123460650444, + "rewards/rejected": -0.12890625, + "step": 1180 + }, + { + "epoch": 0.090302018515708, + "grad_norm": 1.590480942561905, + "learning_rate": 2.3190841426097937e-06, + "log_odds_chosen": 0.7764648199081421, + "log_odds_ratio": -0.4970703125, + "logits/chosen": -0.935546875, + "logits/rejected": -0.858593761920929, + "logps/chosen": -0.7076171636581421, + "logps/rejected": -1.2365233898162842, + "loss": 1.0711, + "nll_loss": 1.057031273841858, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07077636569738388, + "rewards/margins": 0.05285339429974556, + "rewards/rejected": -0.12360839545726776, + "step": 1190 + }, + { + "epoch": 0.09106085900743664, + "grad_norm": 1.5911920566602926, + "learning_rate": 2.309401076758503e-06, + "log_odds_chosen": 0.7508484125137329, + "log_odds_ratio": -0.55712890625, + "logits/chosen": -1.025781273841858, + "logits/rejected": -0.9849609136581421, + "logps/chosen": -0.7730468511581421, + "logps/rejected": -1.286718726158142, + "loss": 1.0467, + "nll_loss": 1.0251953601837158, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.07729492336511612, + "rewards/margins": 0.05129394680261612, + "rewards/rejected": -0.12856444716453552, + "step": 1200 + }, + { + "epoch": 0.09181969949916527, + "grad_norm": 1.6321906040324805, + "learning_rate": 2.299838298304276e-06, + "log_odds_chosen": 0.943359375, + "log_odds_ratio": -0.474853515625, + "logits/chosen": -0.9476562738418579, + "logits/rejected": -0.896679699420929, + "logps/chosen": -0.692578136920929, + "logps/rejected": -1.282812476158142, + "loss": 1.0844, + "nll_loss": 1.0271484851837158, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06927490234375, + "rewards/margins": 0.05902709811925888, + "rewards/rejected": -0.12834472954273224, + "step": 1210 + }, + { + "epoch": 0.09257853999089391, + "grad_norm": 1.7462695798120413, + "learning_rate": 2.2903933372554728e-06, + "log_odds_chosen": 0.7892090082168579, + "log_odds_ratio": -0.5526367425918579, + "logits/chosen": -1.009765625, + "logits/rejected": -0.9203125238418579, + "logps/chosen": -0.761425793170929, + "logps/rejected": -1.3087890148162842, + "loss": 1.0657, + "nll_loss": 1.0515625476837158, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.07618407905101776, + "rewards/margins": 0.05482025071978569, + "rewards/rejected": -0.1309814453125, + "step": 1220 + }, + { + "epoch": 0.09333738048262255, + "grad_norm": 1.5708827607202138, + "learning_rate": 2.281063794048804e-06, + "log_odds_chosen": 0.9153808355331421, + "log_odds_ratio": -0.5028320550918579, + "logits/chosen": -0.988476574420929, + "logits/rejected": -0.9078124761581421, + "logps/chosen": -0.7201172113418579, + "logps/rejected": -1.3582031726837158, + "loss": 1.0552, + "nll_loss": 0.9388672113418579, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07205810397863388, + "rewards/margins": 0.06379318237304688, + "rewards/rejected": -0.13588866591453552, + "step": 1230 + }, + { + "epoch": 0.09409622097435119, + "grad_norm": 1.4432294029947919, + "learning_rate": 2.271847336988259e-06, + "log_odds_chosen": 0.806347668170929, + "log_odds_ratio": -0.517871081829071, + "logits/chosen": -1.0128905773162842, + "logits/rejected": -0.901562511920929, + "logps/chosen": -0.7451171875, + "logps/rejected": -1.31640625, + "loss": 1.0815, + "nll_loss": 0.957226574420929, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07449951022863388, + "rewards/margins": 0.05713500827550888, + "rewards/rejected": -0.13168945908546448, + "step": 1240 + }, + { + "epoch": 0.09485506146607983, + "grad_norm": 1.5543647286170723, + "learning_rate": 2.262741699796952e-06, + "log_odds_chosen": 0.958984375, + "log_odds_ratio": -0.501269519329071, + "logits/chosen": -0.9544922113418579, + "logits/rejected": -0.9039062261581421, + "logps/chosen": -0.7734375, + "logps/rejected": -1.426171898841858, + "loss": 1.0468, + "nll_loss": 1.048437476158142, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07730712741613388, + "rewards/margins": 0.06527099758386612, + "rewards/rejected": -0.14252929389476776, + "step": 1250 + }, + { + "epoch": 0.09561390195780847, + "grad_norm": 1.674123203497309, + "learning_rate": 2.253744679276044e-06, + "log_odds_chosen": 0.7455810308456421, + "log_odds_ratio": -0.524609386920929, + "logits/chosen": -0.959179699420929, + "logits/rejected": -0.8783203363418579, + "logps/chosen": -0.7276366949081421, + "logps/rejected": -1.2091796398162842, + "loss": 1.0556, + "nll_loss": 1.010156273841858, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07276611030101776, + "rewards/margins": 0.04802246019244194, + "rewards/rejected": -0.12080077826976776, + "step": 1260 + }, + { + "epoch": 0.09637274244953711, + "grad_norm": 1.5730945328477453, + "learning_rate": 2.244854133065255e-06, + "log_odds_chosen": 0.91387939453125, + "log_odds_ratio": -0.529248058795929, + "logits/chosen": -0.990234375, + "logits/rejected": -0.9085937738418579, + "logps/chosen": -0.8076171875, + "logps/rejected": -1.453515648841858, + "loss": 1.0977, + "nll_loss": 1.0802733898162842, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.08072509616613388, + "rewards/margins": 0.06455688178539276, + "rewards/rejected": -0.14531250298023224, + "step": 1270 + }, + { + "epoch": 0.09713158294126574, + "grad_norm": 1.564022477389795, + "learning_rate": 2.2360679774997895e-06, + "log_odds_chosen": 0.7984374761581421, + "log_odds_ratio": -0.5282226800918579, + "logits/chosen": -0.9302734136581421, + "logits/rejected": -0.863476574420929, + "logps/chosen": -0.705859363079071, + "logps/rejected": -1.2312500476837158, + "loss": 1.0847, + "nll_loss": 1.0154297351837158, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07062987983226776, + "rewards/margins": 0.05250854417681694, + "rewards/rejected": -0.12302245944738388, + "step": 1280 + }, + { + "epoch": 0.09789042343299438, + "grad_norm": 1.498086269280391, + "learning_rate": 2.2273841855588183e-06, + "log_odds_chosen": 0.8680175542831421, + "log_odds_ratio": -0.501660168170929, + "logits/chosen": -0.9849609136581421, + "logits/rejected": -0.880078136920929, + "logps/chosen": -0.716601550579071, + "logps/rejected": -1.306640625, + "loss": 1.0609, + "nll_loss": 1.064453125, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07170410454273224, + "rewards/margins": 0.05905456468462944, + "rewards/rejected": -0.1307373046875, + "step": 1290 + }, + { + "epoch": 0.09864926392472302, + "grad_norm": 1.7728116233156204, + "learning_rate": 2.2188007849009167e-06, + "log_odds_chosen": 0.8995116949081421, + "log_odds_ratio": -0.5074707269668579, + "logits/chosen": -0.966992199420929, + "logits/rejected": -0.923046886920929, + "logps/chosen": -0.740039050579071, + "logps/rejected": -1.3748047351837158, + "loss": 1.0752, + "nll_loss": 1.068750023841858, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07395019382238388, + "rewards/margins": 0.06353454291820526, + "rewards/rejected": -0.13752441108226776, + "step": 1300 + }, + { + "epoch": 0.09940810441645166, + "grad_norm": 1.7158654014174184, + "learning_rate": 2.2103158559821502e-06, + "log_odds_chosen": 1.112890601158142, + "log_odds_ratio": -0.43603515625, + "logits/chosen": -0.960156261920929, + "logits/rejected": -0.853320300579071, + "logps/chosen": -0.6685546636581421, + "logps/rejected": -1.397070288658142, + "loss": 1.03, + "nll_loss": 0.949999988079071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06690673530101776, + "rewards/margins": 0.072845458984375, + "rewards/rejected": -0.13969726860523224, + "step": 1310 + }, + { + "epoch": 0.1001669449081803, + "grad_norm": 1.7372509904613194, + "learning_rate": 2.2019275302527213e-06, + "log_odds_chosen": 1.05224609375, + "log_odds_ratio": -0.4795898497104645, + "logits/chosen": -0.982421875, + "logits/rejected": -0.8744140863418579, + "logps/chosen": -0.6839843988418579, + "logps/rejected": -1.416406273841858, + "loss": 1.0835, + "nll_loss": 0.9994140863418579, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06842041015625, + "rewards/margins": 0.07326050102710724, + "rewards/rejected": -0.1417236328125, + "step": 1320 + }, + { + "epoch": 0.10092578539990894, + "grad_norm": 1.5946406192046245, + "learning_rate": 2.193633988428327e-06, + "log_odds_chosen": 0.7843261957168579, + "log_odds_ratio": -0.521679699420929, + "logits/chosen": -0.9453125, + "logits/rejected": -0.89453125, + "logps/chosen": -0.7162109613418579, + "logps/rejected": -1.2314453125, + "loss": 1.0827, + "nll_loss": 1.0789062976837158, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07159423828125, + "rewards/margins": 0.05157775804400444, + "rewards/rejected": -0.12312011420726776, + "step": 1330 + }, + { + "epoch": 0.10168462589163758, + "grad_norm": 1.4243961328613546, + "learning_rate": 2.185433458832612e-06, + "log_odds_chosen": 0.6023925542831421, + "log_odds_ratio": -0.574999988079071, + "logits/chosen": -0.9908202886581421, + "logits/rejected": -0.9302734136581421, + "logps/chosen": -0.713085949420929, + "logps/rejected": -1.095703125, + "loss": 1.0959, + "nll_loss": 1.042382836341858, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.0712890625, + "rewards/margins": 0.03832244873046875, + "rewards/rejected": -0.10966797173023224, + "step": 1340 + }, + { + "epoch": 0.10244346638336621, + "grad_norm": 1.495406382498458, + "learning_rate": 2.177324215807269e-06, + "log_odds_chosen": 1.001367211341858, + "log_odds_ratio": -0.441650390625, + "logits/chosen": -0.953125, + "logits/rejected": -0.881640613079071, + "logps/chosen": -0.700488269329071, + "logps/rejected": -1.345312476158142, + "loss": 1.0643, + "nll_loss": 1.0226562023162842, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07004394382238388, + "rewards/margins": 0.064483642578125, + "rewards/rejected": -0.13447265326976776, + "step": 1350 + }, + { + "epoch": 0.10320230687509485, + "grad_norm": 1.4865252925204064, + "learning_rate": 2.1693045781865616e-06, + "log_odds_chosen": 1.048974633216858, + "log_odds_ratio": -0.4736328125, + "logits/chosen": -0.910937488079071, + "logits/rejected": -0.861328125, + "logps/chosen": -0.7164062261581421, + "logps/rejected": -1.437109351158142, + "loss": 1.0463, + "nll_loss": 1.000390648841858, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07167968899011612, + "rewards/margins": 0.07203521579504013, + "rewards/rejected": -0.14365234971046448, + "step": 1360 + }, + { + "epoch": 0.10396114736682349, + "grad_norm": 2.887947200974786, + "learning_rate": 2.1613729078331965e-06, + "log_odds_chosen": 0.974902331829071, + "log_odds_ratio": -0.46269530057907104, + "logits/chosen": -0.9169921875, + "logits/rejected": -0.8324218988418579, + "logps/chosen": -0.7001953125, + "logps/rejected": -1.377343773841858, + "loss": 1.0644, + "nll_loss": 0.994140625, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07004394382238388, + "rewards/margins": 0.06776733696460724, + "rewards/rejected": -0.13779297471046448, + "step": 1370 + }, + { + "epoch": 0.10471998785855213, + "grad_norm": 1.5824770881296906, + "learning_rate": 2.1535276082326617e-06, + "log_odds_chosen": 0.932446300983429, + "log_odds_ratio": -0.5478515625, + "logits/chosen": -0.9814453125, + "logits/rejected": -0.9380859136581421, + "logps/chosen": -0.7708984613418579, + "logps/rejected": -1.4343750476837158, + "loss": 1.0571, + "nll_loss": 1.02734375, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07706298679113388, + "rewards/margins": 0.06638336181640625, + "rewards/rejected": -0.14340820908546448, + "step": 1380 + }, + { + "epoch": 0.10547882835028077, + "grad_norm": 2.06581191948831, + "learning_rate": 2.14576712314328e-06, + "log_odds_chosen": 0.946362316608429, + "log_odds_ratio": -0.5375000238418579, + "logits/chosen": -0.9765625, + "logits/rejected": -0.904101550579071, + "logps/chosen": -0.7408202886581421, + "logps/rejected": -1.4210937023162842, + "loss": 1.0305, + "nll_loss": 0.9906250238418579, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.0740966796875, + "rewards/margins": 0.06792907416820526, + "rewards/rejected": -0.14211425185203552, + "step": 1390 + }, + { + "epoch": 0.10623766884200941, + "grad_norm": 1.503598713934619, + "learning_rate": 2.138089935299395e-06, + "log_odds_chosen": 1.009765625, + "log_odds_ratio": -0.4737792909145355, + "logits/chosen": -0.95703125, + "logits/rejected": -0.8873046636581421, + "logps/chosen": -0.7177734375, + "logps/rejected": -1.396875023841858, + "loss": 1.0736, + "nll_loss": 1.0187499523162842, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.07183837890625, + "rewards/margins": 0.06788787990808487, + "rewards/rejected": -0.13967284560203552, + "step": 1400 + }, + { + "epoch": 0.10699650933373805, + "grad_norm": 1.492803973103788, + "learning_rate": 2.1304945651652297e-06, + "log_odds_chosen": 0.8885742425918579, + "log_odds_ratio": -0.538867175579071, + "logits/chosen": -0.9580078125, + "logits/rejected": -0.864062488079071, + "logps/chosen": -0.7298828363418579, + "logps/rejected": -1.323632836341858, + "loss": 1.0644, + "nll_loss": 1.1648437976837158, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07294921576976776, + "rewards/margins": 0.05940551683306694, + "rewards/rejected": -0.13237304985523224, + "step": 1410 + }, + { + "epoch": 0.10775534982546668, + "grad_norm": 1.665221406365988, + "learning_rate": 2.122979569737101e-06, + "log_odds_chosen": 0.9991210699081421, + "log_odds_ratio": -0.47651368379592896, + "logits/chosen": -1.0068359375, + "logits/rejected": -0.9341796636581421, + "logps/chosen": -0.6558593511581421, + "logps/rejected": -1.295507788658142, + "loss": 1.0489, + "nll_loss": 0.9662109613418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0655517578125, + "rewards/margins": 0.06397704780101776, + "rewards/rejected": -0.12954100966453552, + "step": 1420 + }, + { + "epoch": 0.10851419031719532, + "grad_norm": 2.0746542535178603, + "learning_rate": 2.11554354139178e-06, + "log_odds_chosen": 0.9710448980331421, + "log_odds_ratio": -0.5282226800918579, + "logits/chosen": -0.9888671636581421, + "logits/rejected": -0.9351562261581421, + "logps/chosen": -0.8203125, + "logps/rejected": -1.509765625, + "loss": 1.0624, + "nll_loss": 1.026953101158142, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.08211670070886612, + "rewards/margins": 0.06894836574792862, + "rewards/rejected": -0.15102538466453552, + "step": 1430 + }, + { + "epoch": 0.10927303080892396, + "grad_norm": 1.592392453133004, + "learning_rate": 2.1081851067789196e-06, + "log_odds_chosen": 0.970703125, + "log_odds_ratio": -0.47978514432907104, + "logits/chosen": -0.912109375, + "logits/rejected": -0.851367175579071, + "logps/chosen": -0.677734375, + "logps/rejected": -1.3093750476837158, + "loss": 1.0409, + "nll_loss": 0.966796875, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.0677490234375, + "rewards/margins": 0.06310882419347763, + "rewards/rejected": -0.13095703721046448, + "step": 1440 + }, + { + "epoch": 0.1100318713006526, + "grad_norm": 1.622693092846365, + "learning_rate": 2.1009029257555606e-06, + "log_odds_chosen": 0.923657238483429, + "log_odds_ratio": -0.487548828125, + "logits/chosen": -0.943164050579071, + "logits/rejected": -0.868359386920929, + "logps/chosen": -0.6958984136581421, + "logps/rejected": -1.3054687976837158, + "loss": 1.0425, + "nll_loss": 1.00390625, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06960449367761612, + "rewards/margins": 0.06094970554113388, + "rewards/rejected": -0.13059082627296448, + "step": 1450 + }, + { + "epoch": 0.11079071179238124, + "grad_norm": 5.009691330274977, + "learning_rate": 2.0936956903608545e-06, + "log_odds_chosen": 0.996289074420929, + "log_odds_ratio": -0.47612303495407104, + "logits/chosen": -0.931835949420929, + "logits/rejected": -0.8363281488418579, + "logps/chosen": -0.7001953125, + "logps/rejected": -1.349218726158142, + "loss": 1.0934, + "nll_loss": 1.058007836341858, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07009277492761612, + "rewards/margins": 0.06481323391199112, + "rewards/rejected": -0.13486328721046448, + "step": 1460 + }, + { + "epoch": 0.11154955228410988, + "grad_norm": 1.5133282112647446, + "learning_rate": 2.0865621238292046e-06, + "log_odds_chosen": 0.929492175579071, + "log_odds_ratio": -0.4931640625, + "logits/chosen": -0.900390625, + "logits/rejected": -0.8248046636581421, + "logps/chosen": -0.71484375, + "logps/rejected": -1.3298828601837158, + "loss": 1.0446, + "nll_loss": 1.064062476158142, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.071533203125, + "rewards/margins": 0.06144409254193306, + "rewards/rejected": -0.13300780951976776, + "step": 1470 + }, + { + "epoch": 0.11230839277583853, + "grad_norm": 1.6448887742190683, + "learning_rate": 2.079500979640145e-06, + "log_odds_chosen": 0.8451293706893921, + "log_odds_ratio": -0.503125011920929, + "logits/chosen": -0.9248046875, + "logits/rejected": -0.856249988079071, + "logps/chosen": -0.701855480670929, + "logps/rejected": -1.2658202648162842, + "loss": 1.0617, + "nll_loss": 0.9662109613418579, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.0701904296875, + "rewards/margins": 0.05644073337316513, + "rewards/rejected": -0.12651367485523224, + "step": 1480 + }, + { + "epoch": 0.11306723326756715, + "grad_norm": 1.6079772952545273, + "learning_rate": 2.072511040603359e-06, + "log_odds_chosen": 0.8741699457168579, + "log_odds_ratio": -0.5518554449081421, + "logits/chosen": -0.915820300579071, + "logits/rejected": -0.830859363079071, + "logps/chosen": -0.749804675579071, + "logps/rejected": -1.3640625476837158, + "loss": 1.0472, + "nll_loss": 0.9970703125, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07498779147863388, + "rewards/margins": 0.061373136937618256, + "rewards/rejected": -0.13645020127296448, + "step": 1490 + }, + { + "epoch": 0.11382607375929579, + "grad_norm": 1.5317978336165587, + "learning_rate": 2.065591117977289e-06, + "log_odds_chosen": 0.8371948003768921, + "log_odds_ratio": -0.5206054449081421, + "logits/chosen": -0.9642578363418579, + "logits/rejected": -0.899218738079071, + "logps/chosen": -0.7547851800918579, + "logps/rejected": -1.3203125, + "loss": 1.0712, + "nll_loss": 1.0656249523162842, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.07547607272863388, + "rewards/margins": 0.05657196044921875, + "rewards/rejected": -0.13210448622703552, + "step": 1500 + }, + { + "epoch": 0.11458491425102443, + "grad_norm": 2.3627141683478374, + "learning_rate": 2.058740050619915e-06, + "log_odds_chosen": 1.017968773841858, + "log_odds_ratio": -0.46826171875, + "logits/chosen": -1.0009765625, + "logits/rejected": -0.8857421875, + "logps/chosen": -0.7376953363418579, + "logps/rejected": -1.4552733898162842, + "loss": 1.0386, + "nll_loss": 0.9716796875, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.07375488430261612, + "rewards/margins": 0.07177734375, + "rewards/rejected": -0.1455078125, + "step": 1510 + }, + { + "epoch": 0.11534375474275307, + "grad_norm": 1.8229312547459466, + "learning_rate": 2.0519567041703083e-06, + "log_odds_chosen": 0.9739745855331421, + "log_odds_ratio": -0.4912109375, + "logits/chosen": -0.9917968511581421, + "logits/rejected": -0.9136718511581421, + "logps/chosen": -0.746874988079071, + "logps/rejected": -1.441796898841858, + "loss": 1.0527, + "nll_loss": 0.9945312738418579, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.07463379204273224, + "rewards/margins": 0.069488525390625, + "rewards/rejected": -0.14421387016773224, + "step": 1520 + }, + { + "epoch": 0.11610259523448171, + "grad_norm": 1.781377185920502, + "learning_rate": 2.0452399702596544e-06, + "log_odds_chosen": 0.925097644329071, + "log_odds_ratio": -0.48115235567092896, + "logits/chosen": -0.938281238079071, + "logits/rejected": -0.8638671636581421, + "logps/chosen": -0.7284179925918579, + "logps/rejected": -1.318750023841858, + "loss": 1.033, + "nll_loss": 0.90234375, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07282714545726776, + "rewards/margins": 0.05898742750287056, + "rewards/rejected": -0.1317138671875, + "step": 1530 + }, + { + "epoch": 0.11686143572621036, + "grad_norm": 1.7487600964578844, + "learning_rate": 2.0385887657505017e-06, + "log_odds_chosen": 1.035742163658142, + "log_odds_ratio": -0.4712890684604645, + "logits/chosen": -0.9691406488418579, + "logits/rejected": -0.9072265625, + "logps/chosen": -0.739453136920929, + "logps/rejected": -1.449804663658142, + "loss": 1.0245, + "nll_loss": 0.939453125, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.07387695461511612, + "rewards/margins": 0.071044921875, + "rewards/rejected": -0.14497070014476776, + "step": 1540 + }, + { + "epoch": 0.117620276217939, + "grad_norm": 1.454469728477322, + "learning_rate": 2.032002032003048e-06, + "log_odds_chosen": 0.838940441608429, + "log_odds_ratio": -0.51806640625, + "logits/chosen": -0.9722656011581421, + "logits/rejected": -0.8675781488418579, + "logps/chosen": -0.70751953125, + "logps/rejected": -1.2646484375, + "loss": 1.0354, + "nll_loss": 1.0078125, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.07076416164636612, + "rewards/margins": 0.05572357028722763, + "rewards/rejected": -0.12663574516773224, + "step": 1550 + }, + { + "epoch": 0.11837911670966762, + "grad_norm": 1.6474342799970592, + "learning_rate": 2.025478734167333e-06, + "log_odds_chosen": 0.848681628704071, + "log_odds_ratio": -0.5018554925918579, + "logits/chosen": -0.921875, + "logits/rejected": -0.810351550579071, + "logps/chosen": -0.7069336175918579, + "logps/rejected": -1.231054663658142, + "loss": 1.0514, + "nll_loss": 0.983203113079071, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07071533054113388, + "rewards/margins": 0.05249633640050888, + "rewards/rejected": -0.12314452975988388, + "step": 1560 + }, + { + "epoch": 0.11913795720139626, + "grad_norm": 1.6449397249587745, + "learning_rate": 2.0190178605002747e-06, + "log_odds_chosen": 0.708850085735321, + "log_odds_ratio": -0.52001953125, + "logits/chosen": -0.9468749761581421, + "logits/rejected": -0.879101574420929, + "logps/chosen": -0.728320300579071, + "logps/rejected": -1.185156226158142, + "loss": 1.0435, + "nll_loss": 0.990429699420929, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07283935695886612, + "rewards/margins": 0.04575958102941513, + "rewards/rejected": -0.11857910454273224, + "step": 1570 + }, + { + "epoch": 0.1198967976931249, + "grad_norm": 1.5827948770408402, + "learning_rate": 2.0126184217065104e-06, + "log_odds_chosen": 1.0593750476837158, + "log_odds_ratio": -0.46733397245407104, + "logits/chosen": -1.004296898841858, + "logits/rejected": -0.8949218988418579, + "logps/chosen": -0.705078125, + "logps/rejected": -1.451562523841858, + "loss": 1.0502, + "nll_loss": 0.982226550579071, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.07044677436351776, + "rewards/margins": 0.07470397651195526, + "rewards/rejected": -0.1451416015625, + "step": 1580 + }, + { + "epoch": 0.12065563818485354, + "grad_norm": 1.6904017282178356, + "learning_rate": 2.0062794503020765e-06, + "log_odds_chosen": 1.045166015625, + "log_odds_ratio": -0.44599610567092896, + "logits/chosen": -0.9761718511581421, + "logits/rejected": -0.8818359375, + "logps/chosen": -0.6561523675918579, + "logps/rejected": -1.330078125, + "loss": 1.0226, + "nll_loss": 0.9820312261581421, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06564941257238388, + "rewards/margins": 0.06737060844898224, + "rewards/rejected": -0.13310547173023224, + "step": 1590 + }, + { + "epoch": 0.12141447867658219, + "grad_norm": 1.668297709829456, + "learning_rate": 2e-06, + "log_odds_chosen": 1.0166747570037842, + "log_odds_ratio": -0.4783691465854645, + "logits/chosen": -0.9869140386581421, + "logits/rejected": -0.8951171636581421, + "logps/chosen": -0.7251952886581421, + "logps/rejected": -1.4386718273162842, + "loss": 1.0305, + "nll_loss": 1.0388672351837158, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07248535007238388, + "rewards/margins": 0.07139892876148224, + "rewards/rejected": -0.14394530653953552, + "step": 1600 + }, + { + "epoch": 0.12217331916831083, + "grad_norm": 1.6682675310046382, + "learning_rate": 1.993779145116907e-06, + "log_odds_chosen": 1.00390625, + "log_odds_ratio": -0.4740234315395355, + "logits/chosen": -0.9330078363418579, + "logits/rejected": -0.853710949420929, + "logps/chosen": -0.6610351800918579, + "logps/rejected": -1.3292968273162842, + "loss": 1.0441, + "nll_loss": 1.013281226158142, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06612548977136612, + "rewards/margins": 0.0667724609375, + "rewards/rejected": -0.13295897841453552, + "step": 1610 + }, + { + "epoch": 0.12293215966003945, + "grad_norm": 1.7474719643333534, + "learning_rate": 1.987615979999813e-06, + "log_odds_chosen": 0.926440417766571, + "log_odds_ratio": -0.503369152545929, + "logits/chosen": -0.912890613079071, + "logits/rejected": -0.834179699420929, + "logps/chosen": -0.751953125, + "logps/rejected": -1.3935546875, + "loss": 1.0461, + "nll_loss": 1.0597655773162842, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07512207329273224, + "rewards/margins": 0.06424407660961151, + "rewards/rejected": -0.13930663466453552, + "step": 1620 + }, + { + "epoch": 0.1236910001517681, + "grad_norm": 1.6993682853829732, + "learning_rate": 1.9815096184722797e-06, + "log_odds_chosen": 0.7530761957168579, + "log_odds_ratio": -0.5435546636581421, + "logits/chosen": -0.955273449420929, + "logits/rejected": -0.853320300579071, + "logps/chosen": -0.714062511920929, + "logps/rejected": -1.2175781726837158, + "loss": 1.0468, + "nll_loss": 1.0089843273162842, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.07138671725988388, + "rewards/margins": 0.05036468431353569, + "rewards/rejected": -0.12172851711511612, + "step": 1630 + }, + { + "epoch": 0.12444984064349673, + "grad_norm": 1.6683755597518508, + "learning_rate": 1.9754591932991793e-06, + "log_odds_chosen": 1.045678734779358, + "log_odds_ratio": -0.462646484375, + "logits/chosen": -1.0167968273162842, + "logits/rejected": -0.935546875, + "logps/chosen": -0.717480480670929, + "logps/rejected": -1.446874976158142, + "loss": 1.0296, + "nll_loss": 1.0029296875, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07177734375, + "rewards/margins": 0.0728912353515625, + "rewards/rejected": -0.1446533203125, + "step": 1640 + }, + { + "epoch": 0.12520868113522537, + "grad_norm": 1.8852024815139483, + "learning_rate": 1.9694638556693235e-06, + "log_odds_chosen": 0.8907715082168579, + "log_odds_ratio": -0.5005859136581421, + "logits/chosen": -0.937304675579071, + "logits/rejected": -0.86328125, + "logps/chosen": -0.686328113079071, + "logps/rejected": -1.248437523841858, + "loss": 1.0571, + "nll_loss": 0.938671886920929, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06870117038488388, + "rewards/margins": 0.056203462183475494, + "rewards/rejected": -0.12495116889476776, + "step": 1650 + }, + { + "epoch": 0.12596752162695402, + "grad_norm": 1.6508004109020493, + "learning_rate": 1.963522774695264e-06, + "log_odds_chosen": 1.0453612804412842, + "log_odds_ratio": -0.4659667909145355, + "logits/chosen": -0.986132800579071, + "logits/rejected": -0.87109375, + "logps/chosen": -0.696972668170929, + "logps/rejected": -1.4103515148162842, + "loss": 1.0341, + "nll_loss": 0.9345703125, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06975097954273224, + "rewards/margins": 0.0713348388671875, + "rewards/rejected": -0.14116211235523224, + "step": 1660 + }, + { + "epoch": 0.12672636211868266, + "grad_norm": 1.521239804311641, + "learning_rate": 1.9576351369295853e-06, + "log_odds_chosen": 1.26806640625, + "log_odds_ratio": -0.4283203184604645, + "logits/chosen": -0.953125, + "logits/rejected": -0.876757800579071, + "logps/chosen": -0.6666015386581421, + "logps/rejected": -1.5470702648162842, + "loss": 1.0098, + "nll_loss": 0.956835925579071, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06671142578125, + "rewards/margins": 0.08796997368335724, + "rewards/rejected": -0.15471191704273224, + "step": 1670 + }, + { + "epoch": 0.1274852026104113, + "grad_norm": 1.5577811905592742, + "learning_rate": 1.951800145897066e-06, + "log_odds_chosen": 0.88671875, + "log_odds_ratio": -0.5093749761581421, + "logits/chosen": -0.9527343511581421, + "logits/rejected": -0.8958984613418579, + "logps/chosen": -0.727734386920929, + "logps/rejected": -1.3302733898162842, + "loss": 1.0335, + "nll_loss": 1.004296898841858, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07281494140625, + "rewards/margins": 0.06018371507525444, + "rewards/rejected": -0.13300780951976776, + "step": 1680 + }, + { + "epoch": 0.12824404310213994, + "grad_norm": 1.7603246411817988, + "learning_rate": 1.9460170216420797e-06, + "log_odds_chosen": 0.936328113079071, + "log_odds_ratio": -0.500732421875, + "logits/chosen": -0.977343738079071, + "logits/rejected": -0.8462890386581421, + "logps/chosen": -0.71875, + "logps/rejected": -1.3875000476837158, + "loss": 1.0302, + "nll_loss": 0.973828136920929, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07187499850988388, + "rewards/margins": 0.06686095893383026, + "rewards/rejected": -0.13859863579273224, + "step": 1690 + }, + { + "epoch": 0.12900288359386858, + "grad_norm": 1.8509322922788338, + "learning_rate": 1.9402850002906637e-06, + "log_odds_chosen": 0.94775390625, + "log_odds_ratio": -0.5142577886581421, + "logits/chosen": -0.9898437261581421, + "logits/rejected": -0.9117187261581421, + "logps/chosen": -0.7020508050918579, + "logps/rejected": -1.3449218273162842, + "loss": 1.0089, + "nll_loss": 0.9169921875, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.0701904296875, + "rewards/margins": 0.06424560397863388, + "rewards/rejected": -0.13444824516773224, + "step": 1700 + }, + { + "epoch": 0.12976172408559722, + "grad_norm": 1.6167006582441545, + "learning_rate": 1.9346033336266974e-06, + "log_odds_chosen": 0.915332019329071, + "log_odds_ratio": -0.5011230707168579, + "logits/chosen": -0.997265636920929, + "logits/rejected": -0.8988281488418579, + "logps/chosen": -0.695117175579071, + "logps/rejected": -1.2619140148162842, + "loss": 1.0415, + "nll_loss": 0.981640636920929, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.06951904296875, + "rewards/margins": 0.05672607570886612, + "rewards/rejected": -0.12624511122703552, + "step": 1710 + }, + { + "epoch": 0.13052056457732586, + "grad_norm": 1.6591686774932126, + "learning_rate": 1.9289712886816486e-06, + "log_odds_chosen": 0.6905517578125, + "log_odds_ratio": -0.588671863079071, + "logits/chosen": -0.9921875, + "logits/rejected": -0.923632800579071, + "logps/chosen": -0.75, + "logps/rejected": -1.218359351158142, + "loss": 1.0406, + "nll_loss": 0.9466797113418579, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.07501220703125, + "rewards/margins": 0.04682159423828125, + "rewards/rejected": -0.12175293266773224, + "step": 1720 + }, + { + "epoch": 0.13127940506905447, + "grad_norm": 1.585308028498714, + "learning_rate": 1.92338814733738e-06, + "log_odds_chosen": 1.0014159679412842, + "log_odds_ratio": -0.4735351502895355, + "logits/chosen": -0.9554687738418579, + "logits/rejected": -0.865039050579071, + "logps/chosen": -0.671679675579071, + "logps/rejected": -1.2966797351837158, + "loss": 1.0322, + "nll_loss": 0.9839843511581421, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06717529147863388, + "rewards/margins": 0.06257476657629013, + "rewards/rejected": -0.12978515028953552, + "step": 1730 + }, + { + "epoch": 0.1320382455607831, + "grad_norm": 1.7984044160498958, + "learning_rate": 1.9178532059415367e-06, + "log_odds_chosen": 0.91259765625, + "log_odds_ratio": -0.5243164300918579, + "logits/chosen": -0.9722656011581421, + "logits/rejected": -0.883593738079071, + "logps/chosen": -0.701953113079071, + "logps/rejected": -1.354882836341858, + "loss": 1.0332, + "nll_loss": 0.9810546636581421, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.0701904296875, + "rewards/margins": 0.0653533935546875, + "rewards/rejected": -0.13559570908546448, + "step": 1740 + }, + { + "epoch": 0.13279708605251175, + "grad_norm": 1.6269570980620953, + "learning_rate": 1.9123657749350298e-06, + "log_odds_chosen": 0.9396728277206421, + "log_odds_ratio": -0.529589831829071, + "logits/chosen": -0.9664062261581421, + "logits/rejected": -0.855273425579071, + "logps/chosen": -0.6953125, + "logps/rejected": -1.3576171398162842, + "loss": 1.041, + "nll_loss": 1.004296898841858, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06959228217601776, + "rewards/margins": 0.0662078857421875, + "rewards/rejected": -0.13571777939796448, + "step": 1750 + }, + { + "epoch": 0.1335559265442404, + "grad_norm": 1.6259186946124007, + "learning_rate": 1.9069251784911844e-06, + "log_odds_chosen": 0.8407348394393921, + "log_odds_ratio": -0.5379883050918579, + "logits/chosen": -0.993945300579071, + "logits/rejected": -0.9056640863418579, + "logps/chosen": -0.70849609375, + "logps/rejected": -1.2921874523162842, + "loss": 1.0583, + "nll_loss": 0.951953113079071, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07082519680261612, + "rewards/margins": 0.05832824856042862, + "rewards/rejected": -0.12924805283546448, + "step": 1760 + }, + { + "epoch": 0.13431476703596903, + "grad_norm": 1.7328381362199747, + "learning_rate": 1.9015307541661132e-06, + "log_odds_chosen": 0.80621337890625, + "log_odds_ratio": -0.551562488079071, + "logits/chosen": -0.9144531488418579, + "logits/rejected": -0.888867199420929, + "logps/chosen": -0.720703125, + "logps/rejected": -1.255273461341858, + "loss": 1.0261, + "nll_loss": 0.962109386920929, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07207031548023224, + "rewards/margins": 0.053497314453125, + "rewards/rejected": -0.1256103515625, + "step": 1770 + }, + { + "epoch": 0.13507360752769768, + "grad_norm": 1.6704809725658407, + "learning_rate": 1.8961818525599089e-06, + "log_odds_chosen": 0.929638683795929, + "log_odds_ratio": -0.5340820550918579, + "logits/chosen": -1.002343773841858, + "logits/rejected": -0.9302734136581421, + "logps/chosen": -0.705859363079071, + "logps/rejected": -1.334375023841858, + "loss": 1.0152, + "nll_loss": 0.9947265386581421, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.07058105617761612, + "rewards/margins": 0.06288299709558487, + "rewards/rejected": -0.13349609076976776, + "step": 1780 + }, + { + "epoch": 0.13583244801942632, + "grad_norm": 1.5345342962965025, + "learning_rate": 1.890877836988262e-06, + "log_odds_chosen": 0.905834972858429, + "log_odds_ratio": -0.513378918170929, + "logits/chosen": -0.994921863079071, + "logits/rejected": -0.876171886920929, + "logps/chosen": -0.7416015863418579, + "logps/rejected": -1.358984351158142, + "loss": 1.019, + "nll_loss": 0.966015636920929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07421875, + "rewards/margins": 0.06169281154870987, + "rewards/rejected": -0.13581542670726776, + "step": 1790 + }, + { + "epoch": 0.13659128851115496, + "grad_norm": 1.6843206698869806, + "learning_rate": 1.8856180831641269e-06, + "log_odds_chosen": 1.1140625476837158, + "log_odds_ratio": -0.4765625, + "logits/chosen": -0.9654296636581421, + "logits/rejected": -0.8818359375, + "logps/chosen": -0.7650390863418579, + "logps/rejected": -1.559960961341858, + "loss": 1.0252, + "nll_loss": 1.0236327648162842, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.07652588188648224, + "rewards/margins": 0.07954101264476776, + "rewards/rejected": -0.1561279296875, + "step": 1800 + }, + { + "epoch": 0.1373501290028836, + "grad_norm": 1.5515481390991603, + "learning_rate": 1.8804019788890737e-06, + "log_odds_chosen": 1.1181640625, + "log_odds_ratio": -0.4608398377895355, + "logits/chosen": -0.9326171875, + "logits/rejected": -0.841015636920929, + "logps/chosen": -0.7177734375, + "logps/rejected": -1.4988281726837158, + "loss": 1.022, + "nll_loss": 1.052734375, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.07178954780101776, + "rewards/margins": 0.07809142768383026, + "rewards/rejected": -0.14985351264476776, + "step": 1810 + }, + { + "epoch": 0.13810896949461224, + "grad_norm": 1.6376595591782022, + "learning_rate": 1.8752289237539816e-06, + "log_odds_chosen": 1.2386963367462158, + "log_odds_ratio": -0.43408203125, + "logits/chosen": -0.9908202886581421, + "logits/rejected": -0.8837890625, + "logps/chosen": -0.698925793170929, + "logps/rejected": -1.574609398841858, + "loss": 1.0241, + "nll_loss": 0.9544922113418579, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06990966945886612, + "rewards/margins": 0.08756866306066513, + "rewards/rejected": -0.15737304091453552, + "step": 1820 + }, + { + "epoch": 0.13886780998634088, + "grad_norm": 1.7008481895449235, + "learning_rate": 1.8700983288487376e-06, + "log_odds_chosen": 1.112207055091858, + "log_odds_ratio": -0.4563964903354645, + "logits/chosen": -1.0037109851837158, + "logits/rejected": -0.9115234613418579, + "logps/chosen": -0.701953113079071, + "logps/rejected": -1.4744141101837158, + "loss": 1.0151, + "nll_loss": 0.9326171875, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07020263373851776, + "rewards/margins": 0.0773773193359375, + "rewards/rejected": -0.14755859971046448, + "step": 1830 + }, + { + "epoch": 0.13962665047806952, + "grad_norm": 1.9175312329784642, + "learning_rate": 1.8650096164806275e-06, + "log_odds_chosen": 1.190039038658142, + "log_odds_ratio": -0.4319824278354645, + "logits/chosen": -0.9283202886581421, + "logits/rejected": -0.8541015386581421, + "logps/chosen": -0.649707019329071, + "logps/rejected": -1.4619140625, + "loss": 1.0337, + "nll_loss": 0.9732421636581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06492920219898224, + "rewards/margins": 0.08115844428539276, + "rewards/rejected": -0.14606933295726776, + "step": 1840 + }, + { + "epoch": 0.14038549096979816, + "grad_norm": 2.443144967546165, + "learning_rate": 1.8599622199011084e-06, + "log_odds_chosen": 1.020605444908142, + "log_odds_ratio": -0.4903808534145355, + "logits/chosen": -0.9642578363418579, + "logits/rejected": -0.889843761920929, + "logps/chosen": -0.6878906488418579, + "logps/rejected": -1.416406273841858, + "loss": 1.0489, + "nll_loss": 0.9966796636581421, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.06878662109375, + "rewards/margins": 0.07291259616613388, + "rewards/rejected": -0.1417236328125, + "step": 1850 + }, + { + "epoch": 0.1411443314615268, + "grad_norm": 1.663086728989741, + "learning_rate": 1.854955583040673e-06, + "log_odds_chosen": 1.0613281726837158, + "log_odds_ratio": -0.4869140684604645, + "logits/chosen": -0.9642578363418579, + "logits/rejected": -0.901171863079071, + "logps/chosen": -0.7315429449081421, + "logps/rejected": -1.4523437023162842, + "loss": 0.9994, + "nll_loss": 0.9876953363418579, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07315673679113388, + "rewards/margins": 0.07206115871667862, + "rewards/rejected": -0.14519043266773224, + "step": 1860 + }, + { + "epoch": 0.1419031719532554, + "grad_norm": 1.9117980136195871, + "learning_rate": 1.849989160251521e-06, + "log_odds_chosen": 0.9891723394393921, + "log_odds_ratio": -0.5174804925918579, + "logits/chosen": -0.968554675579071, + "logits/rejected": -0.898242175579071, + "logps/chosen": -0.693554699420929, + "logps/rejected": -1.387304663658142, + "loss": 1.0329, + "nll_loss": 0.9984375238418579, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.06934814155101776, + "rewards/margins": 0.06931991875171661, + "rewards/rejected": -0.13864746689796448, + "step": 1870 + }, + { + "epoch": 0.14266201244498405, + "grad_norm": 2.0856627792753577, + "learning_rate": 1.8450624160577701e-06, + "log_odds_chosen": 1.063867211341858, + "log_odds_ratio": -0.49394530057907104, + "logits/chosen": -1.012304663658142, + "logits/rejected": -0.9169921875, + "logps/chosen": -0.740234375, + "logps/rejected": -1.5066406726837158, + "loss": 1.007, + "nll_loss": 1.001367211341858, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07404784858226776, + "rewards/margins": 0.07670287787914276, + "rewards/rejected": -0.15078124403953552, + "step": 1880 + }, + { + "epoch": 0.1434208529367127, + "grad_norm": 1.576304279008401, + "learning_rate": 1.8401748249129445e-06, + "log_odds_chosen": 1.0471680164337158, + "log_odds_ratio": -0.4671386778354645, + "logits/chosen": -1.000390648841858, + "logits/rejected": -0.910351574420929, + "logps/chosen": -0.6888672113418579, + "logps/rejected": -1.420507788658142, + "loss": 1.015, + "nll_loss": 0.985546886920929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06890869140625, + "rewards/margins": 0.0731201171875, + "rewards/rejected": -0.14199218153953552, + "step": 1890 + }, + { + "epoch": 0.14417969342844134, + "grad_norm": 2.2943205801406075, + "learning_rate": 1.8353258709644938e-06, + "log_odds_chosen": 1.0422852039337158, + "log_odds_ratio": -0.4927734434604645, + "logits/chosen": -0.955859363079071, + "logits/rejected": -0.901562511920929, + "logps/chosen": -0.7401367425918579, + "logps/rejected": -1.451562523841858, + "loss": 1.0349, + "nll_loss": 1.0314452648162842, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07406006008386612, + "rewards/margins": 0.07120971381664276, + "rewards/rejected": -0.14523926377296448, + "step": 1900 + }, + { + "epoch": 0.14493853392016998, + "grad_norm": 2.0490705861246323, + "learning_rate": 1.830515047825102e-06, + "log_odds_chosen": 0.9310058355331421, + "log_odds_ratio": -0.512890636920929, + "logits/chosen": -0.9560546875, + "logits/rejected": -0.9115234613418579, + "logps/chosen": -0.7147461175918579, + "logps/rejected": -1.365625023841858, + "loss": 1.0137, + "nll_loss": 0.9736328125, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07149658352136612, + "rewards/margins": 0.06497955322265625, + "rewards/rejected": -0.13642577826976776, + "step": 1910 + }, + { + "epoch": 0.14569737441189862, + "grad_norm": 1.7749586371627204, + "learning_rate": 1.8257418583505536e-06, + "log_odds_chosen": 1.212499976158142, + "log_odds_ratio": -0.42607420682907104, + "logits/chosen": -0.954882800579071, + "logits/rejected": -0.861132800579071, + "logps/chosen": -0.6689453125, + "logps/rejected": -1.5017578601837158, + "loss": 1.0019, + "nll_loss": 0.922656238079071, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06691894680261612, + "rewards/margins": 0.08331298828125, + "rewards/rejected": -0.15012207627296448, + "step": 1920 + }, + { + "epoch": 0.14645621490362726, + "grad_norm": 1.6902217952090752, + "learning_rate": 1.8210058144239416e-06, + "log_odds_chosen": 0.8382812738418579, + "log_odds_ratio": -0.5379883050918579, + "logits/chosen": -0.9527343511581421, + "logits/rejected": -0.8731445074081421, + "logps/chosen": -0.73974609375, + "logps/rejected": -1.356054663658142, + "loss": 1.0088, + "nll_loss": 1.009179711341858, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07390137016773224, + "rewards/margins": 0.06163330003619194, + "rewards/rejected": -0.13544921576976776, + "step": 1930 + }, + { + "epoch": 0.1472150553953559, + "grad_norm": 1.8073168305465488, + "learning_rate": 1.816306436745999e-06, + "log_odds_chosen": 0.97314453125, + "log_odds_ratio": -0.520800769329071, + "logits/chosen": -0.963085949420929, + "logits/rejected": -0.865429699420929, + "logps/chosen": -0.755078136920929, + "logps/rejected": -1.433984398841858, + "loss": 1.0157, + "nll_loss": 0.952929675579071, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.07548828423023224, + "rewards/margins": 0.06783447414636612, + "rewards/rejected": -0.143310546875, + "step": 1940 + }, + { + "epoch": 0.14797389588708454, + "grad_norm": 2.1030165974817656, + "learning_rate": 1.8116432546313529e-06, + "log_odds_chosen": 0.807910144329071, + "log_odds_ratio": -0.547119140625, + "logits/chosen": -0.967968761920929, + "logits/rejected": -0.854296863079071, + "logps/chosen": -0.7210937738418579, + "logps/rejected": -1.2625000476837158, + "loss": 1.0078, + "nll_loss": 0.974804699420929, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07205810397863388, + "rewards/margins": 0.054046630859375, + "rewards/rejected": -0.12607422471046448, + "step": 1950 + }, + { + "epoch": 0.14873273637881318, + "grad_norm": 1.7734373496612974, + "learning_rate": 1.8070158058105026e-06, + "log_odds_chosen": 1.068139672279358, + "log_odds_ratio": -0.47832030057907104, + "logits/chosen": -0.9847656488418579, + "logits/rejected": -0.9007812738418579, + "logps/chosen": -0.6900390386581421, + "logps/rejected": -1.4294922351837158, + "loss": 1.0195, + "nll_loss": 0.934374988079071, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06900634616613388, + "rewards/margins": 0.0738983154296875, + "rewards/rejected": -0.14304199814796448, + "step": 1960 + }, + { + "epoch": 0.14949157687054182, + "grad_norm": 1.5424244301559844, + "learning_rate": 1.8024236362373315e-06, + "log_odds_chosen": 0.9842773675918579, + "log_odds_ratio": -0.46113282442092896, + "logits/chosen": -0.9634765386581421, + "logits/rejected": -0.8519531488418579, + "logps/chosen": -0.681835949420929, + "logps/rejected": -1.3474609851837158, + "loss": 1.0132, + "nll_loss": 0.969921886920929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06818847358226776, + "rewards/margins": 0.0665283203125, + "rewards/rejected": -0.13474121689796448, + "step": 1970 + }, + { + "epoch": 0.15025041736227046, + "grad_norm": 4.294685050248751, + "learning_rate": 1.7978662999019787e-06, + "log_odds_chosen": 1.1591796875, + "log_odds_ratio": -0.4561523497104645, + "logits/chosen": -0.9873046875, + "logits/rejected": -0.898632824420929, + "logps/chosen": -0.675000011920929, + "logps/rejected": -1.481835961341858, + "loss": 1.018, + "nll_loss": 0.981640636920929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06749267876148224, + "rewards/margins": 0.080718994140625, + "rewards/rejected": -0.14836426079273224, + "step": 1980 + }, + { + "epoch": 0.1510092578539991, + "grad_norm": 1.8285917174807669, + "learning_rate": 1.793343358648881e-06, + "log_odds_chosen": 1.26025390625, + "log_odds_ratio": -0.43413084745407104, + "logits/chosen": -0.942187488079071, + "logits/rejected": -0.839062511920929, + "logps/chosen": -0.688281238079071, + "logps/rejected": -1.578710913658142, + "loss": 0.9978, + "nll_loss": 0.955273449420929, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06879882514476776, + "rewards/margins": 0.0889892578125, + "rewards/rejected": -0.15781250596046448, + "step": 1990 + }, + { + "epoch": 0.15176809834572771, + "grad_norm": 1.7246984869527726, + "learning_rate": 1.7888543819998317e-06, + "log_odds_chosen": 1.23291015625, + "log_odds_ratio": -0.46337890625, + "logits/chosen": -0.979296863079071, + "logits/rejected": -0.8853515386581421, + "logps/chosen": -0.7085937261581421, + "logps/rejected": -1.5851562023162842, + "loss": 1.0057, + "nll_loss": 0.959765613079071, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07087402045726776, + "rewards/margins": 0.08778686821460724, + "rewards/rejected": -0.1585693359375, + "step": 2000 + }, + { + "epoch": 0.15252693883745636, + "grad_norm": 1.8078525016188747, + "learning_rate": 1.7843989469818819e-06, + "log_odds_chosen": 0.66796875, + "log_odds_ratio": -0.581250011920929, + "logits/chosen": -0.9097656011581421, + "logits/rejected": -0.8248046636581421, + "logps/chosen": -0.721484363079071, + "logps/rejected": -1.169531226158142, + "loss": 1.0303, + "nll_loss": 0.930859386920929, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.07215575873851776, + "rewards/margins": 0.044759370386600494, + "rewards/rejected": -0.11696777492761612, + "step": 2010 + }, + { + "epoch": 0.153285779329185, + "grad_norm": 1.750955590287212, + "learning_rate": 1.779976637959939e-06, + "log_odds_chosen": 1.244970679283142, + "log_odds_ratio": -0.44819337129592896, + "logits/chosen": -0.9283202886581421, + "logits/rejected": -0.814648449420929, + "logps/chosen": -0.7040039300918579, + "logps/rejected": -1.57421875, + "loss": 1.0132, + "nll_loss": 0.975781261920929, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.07039795070886612, + "rewards/margins": 0.0870361328125, + "rewards/rejected": -0.15751953423023224, + "step": 2020 + }, + { + "epoch": 0.15404461982091364, + "grad_norm": 1.8459833504202043, + "learning_rate": 1.7755870464739012e-06, + "log_odds_chosen": 0.91015625, + "log_odds_ratio": -0.5287109613418579, + "logits/chosen": -0.950976550579071, + "logits/rejected": -0.8560546636581421, + "logps/chosen": -0.6919921636581421, + "logps/rejected": -1.326562523841858, + "loss": 1.0335, + "nll_loss": 0.981249988079071, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.06920166313648224, + "rewards/margins": 0.06353607028722763, + "rewards/rejected": -0.13271483778953552, + "step": 2030 + }, + { + "epoch": 0.15480346031264228, + "grad_norm": 1.7056677979498798, + "learning_rate": 1.7712297710801907e-06, + "log_odds_chosen": 1.076074242591858, + "log_odds_ratio": -0.47509765625, + "logits/chosen": -0.9117187261581421, + "logits/rejected": -0.833789050579071, + "logps/chosen": -0.6922851800918579, + "logps/rejected": -1.421484351158142, + "loss": 1.0171, + "nll_loss": 0.967968761920929, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.0692138671875, + "rewards/margins": 0.07306518405675888, + "rewards/rejected": -0.14218750596046448, + "step": 2040 + }, + { + "epoch": 0.15556230080437092, + "grad_norm": 1.605351728289974, + "learning_rate": 1.7669044171975444e-06, + "log_odds_chosen": 1.0661499500274658, + "log_odds_ratio": -0.4999023377895355, + "logits/chosen": -0.9091796875, + "logits/rejected": -0.8255859613418579, + "logps/chosen": -0.7171875238418579, + "logps/rejected": -1.4519531726837158, + "loss": 1.0172, + "nll_loss": 0.9818359613418579, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.07167968899011612, + "rewards/margins": 0.07357482612133026, + "rewards/rejected": -0.14523926377296448, + "step": 2050 + }, + { + "epoch": 0.15632114129609956, + "grad_norm": 1.6960342772381833, + "learning_rate": 1.7626105969569268e-06, + "log_odds_chosen": 1.058984398841858, + "log_odds_ratio": -0.460693359375, + "logits/chosen": -0.9281250238418579, + "logits/rejected": -0.8433593511581421, + "logps/chosen": -0.7040039300918579, + "logps/rejected": -1.4226562976837158, + "loss": 1.0291, + "nll_loss": 1.0041015148162842, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.07033691555261612, + "rewards/margins": 0.07182617485523224, + "rewards/rejected": -0.14223632216453552, + "step": 2060 + }, + { + "epoch": 0.1570799817878282, + "grad_norm": 2.0842118121138205, + "learning_rate": 1.758347929055432e-06, + "log_odds_chosen": 1.00048828125, + "log_odds_ratio": -0.455810546875, + "logits/chosen": -0.9208984375, + "logits/rejected": -0.8349609375, + "logps/chosen": -0.6654297113418579, + "logps/rejected": -1.3132812976837158, + "loss": 0.9886, + "nll_loss": 0.919140636920929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06655273586511612, + "rewards/margins": 0.06477661430835724, + "rewards/rejected": -0.13137206435203552, + "step": 2070 + }, + { + "epoch": 0.15783882227955684, + "grad_norm": 2.120569477558702, + "learning_rate": 1.7541160386140582e-06, + "log_odds_chosen": 1.3015625476837158, + "log_odds_ratio": -0.4002929627895355, + "logits/chosen": -0.9339843988418579, + "logits/rejected": -0.81640625, + "logps/chosen": -0.648242175579071, + "logps/rejected": -1.5167968273162842, + "loss": 0.9989, + "nll_loss": 0.984375, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06483153998851776, + "rewards/margins": 0.08698730170726776, + "rewards/rejected": -0.15175780653953552, + "step": 2080 + }, + { + "epoch": 0.15859766277128548, + "grad_norm": 1.6539249012178663, + "learning_rate": 1.7499145570392284e-06, + "log_odds_chosen": 1.0309326648712158, + "log_odds_ratio": -0.49125975370407104, + "logits/chosen": -1.0031249523162842, + "logits/rejected": -0.8828125, + "logps/chosen": -0.7064453363418579, + "logps/rejected": -1.4093749523162842, + "loss": 1.002, + "nll_loss": 0.9527343511581421, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07062987983226776, + "rewards/margins": 0.07036437839269638, + "rewards/rejected": -0.140869140625, + "step": 2090 + }, + { + "epoch": 0.15935650326301412, + "grad_norm": 1.6299919394171718, + "learning_rate": 1.745743121887939e-06, + "log_odds_chosen": 0.9908691644668579, + "log_odds_ratio": -0.49169921875, + "logits/chosen": -0.9585937261581421, + "logits/rejected": -0.833984375, + "logps/chosen": -0.6826171875, + "logps/rejected": -1.361914038658142, + "loss": 1.0307, + "nll_loss": 0.9857422113418579, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.06827392429113388, + "rewards/margins": 0.06794128566980362, + "rewards/rejected": -0.13613280653953552, + "step": 2100 + }, + { + "epoch": 0.16011534375474276, + "grad_norm": 1.5610258992867714, + "learning_rate": 1.7416013767364324e-06, + "log_odds_chosen": 0.9908202886581421, + "log_odds_ratio": -0.4573730528354645, + "logits/chosen": -0.960156261920929, + "logits/rejected": -0.8716796636581421, + "logps/chosen": -0.6415039300918579, + "logps/rejected": -1.2478516101837158, + "loss": 0.9743, + "nll_loss": 0.8705078363418579, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06418456882238388, + "rewards/margins": 0.06059875339269638, + "rewards/rejected": -0.12473144382238388, + "step": 2110 + }, + { + "epoch": 0.1608741842464714, + "grad_norm": 1.7782745410513578, + "learning_rate": 1.7374889710522776e-06, + "log_odds_chosen": 1.322363257408142, + "log_odds_ratio": -0.4240478575229645, + "logits/chosen": -0.943554699420929, + "logits/rejected": -0.85546875, + "logps/chosen": -0.656542956829071, + "logps/rejected": -1.5681641101837158, + "loss": 0.9845, + "nll_loss": 0.860156238079071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06564941257238388, + "rewards/margins": 0.09116820991039276, + "rewards/rejected": -0.15683594346046448, + "step": 2120 + }, + { + "epoch": 0.16163302473820004, + "grad_norm": 1.6843050386022687, + "learning_rate": 1.7334055600697579e-06, + "log_odds_chosen": 1.218359351158142, + "log_odds_ratio": -0.44462889432907104, + "logits/chosen": -0.924023449420929, + "logits/rejected": -0.867382824420929, + "logps/chosen": -0.6788085699081421, + "logps/rejected": -1.494726538658142, + "loss": 1.0162, + "nll_loss": 0.98046875, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06787109375, + "rewards/margins": 0.08159790188074112, + "rewards/rejected": -0.14948730170726776, + "step": 2130 + }, + { + "epoch": 0.16239186522992866, + "grad_norm": 1.8290613554640052, + "learning_rate": 1.7293508046684678e-06, + "log_odds_chosen": 0.8995605707168579, + "log_odds_ratio": -0.5249999761581421, + "logits/chosen": -0.9195312261581421, + "logits/rejected": -0.853320300579071, + "logps/chosen": -0.704296886920929, + "logps/rejected": -1.318359375, + "loss": 0.9991, + "nll_loss": 0.9419921636581421, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07045898586511612, + "rewards/margins": 0.06148681789636612, + "rewards/rejected": -0.13188476860523224, + "step": 2140 + }, + { + "epoch": 0.1631507057216573, + "grad_norm": 1.7340261251194247, + "learning_rate": 1.7253243712550145e-06, + "log_odds_chosen": 0.962109386920929, + "log_odds_ratio": -0.5, + "logits/chosen": -0.900390625, + "logits/rejected": -0.802441418170929, + "logps/chosen": -0.6602538824081421, + "logps/rejected": -1.2755858898162842, + "loss": 1.0306, + "nll_loss": 1.052148461341858, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.06602783501148224, + "rewards/margins": 0.06157531589269638, + "rewards/rejected": -0.12758788466453552, + "step": 2150 + }, + { + "epoch": 0.16390954621338594, + "grad_norm": 1.6987763744425957, + "learning_rate": 1.7213259316477406e-06, + "log_odds_chosen": 1.0940430164337158, + "log_odds_ratio": -0.5050293207168579, + "logits/chosen": -0.9619140625, + "logits/rejected": -0.8824218511581421, + "logps/chosen": -0.757031261920929, + "logps/rejected": -1.5478515625, + "loss": 1.0029, + "nll_loss": 0.9525390863418579, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07576904445886612, + "rewards/margins": 0.07902832329273224, + "rewards/rejected": -0.15480956435203552, + "step": 2160 + }, + { + "epoch": 0.16466838670511458, + "grad_norm": 1.9183213512351749, + "learning_rate": 1.7173551629643674e-06, + "log_odds_chosen": 1.0694091320037842, + "log_odds_ratio": -0.48066407442092896, + "logits/chosen": -0.904101550579071, + "logits/rejected": -0.8335937261581421, + "logps/chosen": -0.738085925579071, + "logps/rejected": -1.4951171875, + "loss": 1.0171, + "nll_loss": 1.003515601158142, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07376708835363388, + "rewards/margins": 0.07562103122472763, + "rewards/rejected": -0.14951172471046448, + "step": 2170 + }, + { + "epoch": 0.16542722719684322, + "grad_norm": 1.7591486737480413, + "learning_rate": 1.713411747512477e-06, + "log_odds_chosen": 1.2626953125, + "log_odds_ratio": -0.4422851502895355, + "logits/chosen": -0.9052734375, + "logits/rejected": -0.803906261920929, + "logps/chosen": -0.6700195074081421, + "logps/rejected": -1.5408203601837158, + "loss": 1.0147, + "nll_loss": 0.922656238079071, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06697998195886612, + "rewards/margins": 0.08709716796875, + "rewards/rejected": -0.15410156548023224, + "step": 2180 + }, + { + "epoch": 0.16618606768857186, + "grad_norm": 2.2730392023754424, + "learning_rate": 1.709495372682753e-06, + "log_odds_chosen": 1.2048828601837158, + "log_odds_ratio": -0.47001951932907104, + "logits/chosen": -0.890429675579071, + "logits/rejected": -0.804492175579071, + "logps/chosen": -0.738085925579071, + "logps/rejected": -1.5867187976837158, + "loss": 0.9889, + "nll_loss": 0.924023449420929, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07381591945886612, + "rewards/margins": 0.08494262397289276, + "rewards/rejected": -0.15876464545726776, + "step": 2190 + }, + { + "epoch": 0.1669449081803005, + "grad_norm": 2.037458706640104, + "learning_rate": 1.7056057308448832e-06, + "log_odds_chosen": 1.121679663658142, + "log_odds_ratio": -0.43730467557907104, + "logits/chosen": -0.916210949420929, + "logits/rejected": -0.8021484613418579, + "logps/chosen": -0.645312488079071, + "logps/rejected": -1.4080078601837158, + "loss": 0.9718, + "nll_loss": 0.9302734136581421, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06459961086511612, + "rewards/margins": 0.07626952975988388, + "rewards/rejected": -0.14077147841453552, + "step": 2200 + }, + { + "epoch": 0.16770374867202914, + "grad_norm": 2.046099172625031, + "learning_rate": 1.701742519246068e-06, + "log_odds_chosen": 0.951708972454071, + "log_odds_ratio": -0.4844726622104645, + "logits/chosen": -0.9341796636581421, + "logits/rejected": -0.8324218988418579, + "logps/chosen": -0.7157226800918579, + "logps/rejected": -1.3728516101837158, + "loss": 1.0105, + "nll_loss": 0.9869140386581421, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07159423828125, + "rewards/margins": 0.065643310546875, + "rewards/rejected": -0.13715820014476776, + "step": 2210 + }, + { + "epoch": 0.16846258916375778, + "grad_norm": 1.6540454740677957, + "learning_rate": 1.6979054399120355e-06, + "log_odds_chosen": 1.15185546875, + "log_odds_ratio": -0.47382813692092896, + "logits/chosen": -0.906054675579071, + "logits/rejected": -0.828320324420929, + "logps/chosen": -0.669628918170929, + "logps/rejected": -1.450781226158142, + "loss": 0.9966, + "nll_loss": 0.908984363079071, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06693115085363388, + "rewards/margins": 0.07802734524011612, + "rewards/rejected": -0.14506836235523224, + "step": 2220 + }, + { + "epoch": 0.16922142965548642, + "grad_norm": 1.6756053571033922, + "learning_rate": 1.6940941995505069e-06, + "log_odds_chosen": 0.881176769733429, + "log_odds_ratio": -0.536328136920929, + "logits/chosen": -0.912890613079071, + "logits/rejected": -0.8296874761581421, + "logps/chosen": -0.773242175579071, + "logps/rejected": -1.3869140148162842, + "loss": 1.0266, + "nll_loss": 1.0226562023162842, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07731933891773224, + "rewards/margins": 0.06138915941119194, + "rewards/rejected": -0.13857421278953552, + "step": 2230 + }, + { + "epoch": 0.16998027014721506, + "grad_norm": 1.555734920343979, + "learning_rate": 1.6903085094570331e-06, + "log_odds_chosen": 0.941113293170929, + "log_odds_ratio": -0.49560546875, + "logits/chosen": -0.943359375, + "logits/rejected": -0.8056640625, + "logps/chosen": -0.69580078125, + "logps/rejected": -1.3240234851837158, + "loss": 0.9995, + "nll_loss": 0.9052734375, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06951904296875, + "rewards/margins": 0.06293945014476776, + "rewards/rejected": -0.13237304985523224, + "step": 2240 + }, + { + "epoch": 0.1707391106389437, + "grad_norm": 1.7021231359836484, + "learning_rate": 1.6865480854231356e-06, + "log_odds_chosen": 0.9696289300918579, + "log_odds_ratio": -0.51904296875, + "logits/chosen": -0.891796886920929, + "logits/rejected": -0.807421863079071, + "logps/chosen": -0.720898449420929, + "logps/rejected": -1.3908202648162842, + "loss": 1.0088, + "nll_loss": 0.9750000238418579, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07203368842601776, + "rewards/margins": 0.06697998195886612, + "rewards/rejected": -0.13901367783546448, + "step": 2250 + }, + { + "epoch": 0.17149795113067234, + "grad_norm": 1.657905024680801, + "learning_rate": 1.682812647646685e-06, + "log_odds_chosen": 1.2488281726837158, + "log_odds_ratio": -0.39140623807907104, + "logits/chosen": -0.9117187261581421, + "logits/rejected": -0.821484386920929, + "logps/chosen": -0.67041015625, + "logps/rejected": -1.491796851158142, + "loss": 0.9972, + "nll_loss": 0.909375011920929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06705322116613388, + "rewards/margins": 0.08208008110523224, + "rewards/rejected": -0.14914551377296448, + "step": 2260 + }, + { + "epoch": 0.17225679162240098, + "grad_norm": 1.6187004410213444, + "learning_rate": 1.6791019206444541e-06, + "log_odds_chosen": 0.9906250238418579, + "log_odds_ratio": -0.4784179627895355, + "logits/chosen": -0.935351550579071, + "logits/rejected": -0.831250011920929, + "logps/chosen": -0.707324206829071, + "logps/rejected": -1.4052734375, + "loss": 1.0238, + "nll_loss": 0.9925781488418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07073974609375, + "rewards/margins": 0.06994018703699112, + "rewards/rejected": -0.140625, + "step": 2270 + }, + { + "epoch": 0.1730156321141296, + "grad_norm": 2.3583378920089197, + "learning_rate": 1.675415633166782e-06, + "log_odds_chosen": 1.3103516101837158, + "log_odds_ratio": -0.42724609375, + "logits/chosen": -0.949023425579071, + "logits/rejected": -0.8480468988418579, + "logps/chosen": -0.6993163824081421, + "logps/rejected": -1.608984351158142, + "loss": 0.9873, + "nll_loss": 0.891406238079071, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06988525390625, + "rewards/margins": 0.09085693210363388, + "rewards/rejected": -0.160888671875, + "step": 2280 + }, + { + "epoch": 0.17377447260585824, + "grad_norm": 1.8684296943890162, + "learning_rate": 1.6717535181142914e-06, + "log_odds_chosen": 0.8514159917831421, + "log_odds_ratio": -0.5181640386581421, + "logits/chosen": -0.957812488079071, + "logits/rejected": -0.8353515863418579, + "logps/chosen": -0.724609375, + "logps/rejected": -1.302734375, + "loss": 1.0093, + "nll_loss": 0.948437511920929, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07241211086511612, + "rewards/margins": 0.05777130275964737, + "rewards/rejected": -0.13020019233226776, + "step": 2290 + }, + { + "epoch": 0.17453331309758688, + "grad_norm": 1.7443057982305137, + "learning_rate": 1.668115312456598e-06, + "log_odds_chosen": 0.9159179925918579, + "log_odds_ratio": -0.482666015625, + "logits/chosen": -0.918749988079071, + "logits/rejected": -0.8291015625, + "logps/chosen": -0.676953136920929, + "logps/rejected": -1.273828148841858, + "loss": 0.9937, + "nll_loss": 0.9429687261581421, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06765136867761612, + "rewards/margins": 0.05970611423254013, + "rewards/rejected": -0.12736816704273224, + "step": 2300 + }, + { + "epoch": 0.17529215358931552, + "grad_norm": 1.7495476536042223, + "learning_rate": 1.6645007571529578e-06, + "log_odds_chosen": 1.193359375, + "log_odds_ratio": -0.4368896484375, + "logits/chosen": -0.9228515625, + "logits/rejected": -0.783984363079071, + "logps/chosen": -0.710156261920929, + "logps/rejected": -1.5369141101837158, + "loss": 0.9818, + "nll_loss": 0.991406261920929, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07098388671875, + "rewards/margins": 0.082611083984375, + "rewards/rejected": -0.15371093153953552, + "step": 2310 + }, + { + "epoch": 0.17605099408104416, + "grad_norm": 2.061978268838713, + "learning_rate": 1.6609095970747992e-06, + "log_odds_chosen": 0.933422863483429, + "log_odds_ratio": -0.5176757574081421, + "logits/chosen": -0.9085937738418579, + "logits/rejected": -0.8267577886581421, + "logps/chosen": -0.770312488079071, + "logps/rejected": -1.447265625, + "loss": 1.0063, + "nll_loss": 1.0353515148162842, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07706298679113388, + "rewards/margins": 0.06778564304113388, + "rewards/rejected": -0.14484862983226776, + "step": 2320 + }, + { + "epoch": 0.1768098345727728, + "grad_norm": 1.9864693037076206, + "learning_rate": 1.6573415809300833e-06, + "log_odds_chosen": 0.85400390625, + "log_odds_ratio": -0.526562511920929, + "logits/chosen": -0.8955078125, + "logits/rejected": -0.846484363079071, + "logps/chosen": -0.7152343988418579, + "logps/rejected": -1.2712891101837158, + "loss": 0.999, + "nll_loss": 1.0095703601837158, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07152099907398224, + "rewards/margins": 0.05556793138384819, + "rewards/rejected": -0.12712402641773224, + "step": 2330 + }, + { + "epoch": 0.17756867506450144, + "grad_norm": 1.5250036352910814, + "learning_rate": 1.6537964611894462e-06, + "log_odds_chosen": 0.8726806640625, + "log_odds_ratio": -0.4984374940395355, + "logits/chosen": -0.9457031488418579, + "logits/rejected": -0.881054699420929, + "logps/chosen": -0.6864258050918579, + "logps/rejected": -1.264257788658142, + "loss": 0.9864, + "nll_loss": 0.995898425579071, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06871338188648224, + "rewards/margins": 0.05774535983800888, + "rewards/rejected": -0.12641601264476776, + "step": 2340 + }, + { + "epoch": 0.17832751555623008, + "grad_norm": 1.7843935011591716, + "learning_rate": 1.6502739940140692e-06, + "log_odds_chosen": 0.805798351764679, + "log_odds_ratio": -0.510449230670929, + "logits/chosen": -0.935546875, + "logits/rejected": -0.809765636920929, + "logps/chosen": -0.7056640386581421, + "logps/rejected": -1.2316405773162842, + "loss": 1.0055, + "nll_loss": 1.05859375, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07052002102136612, + "rewards/margins": 0.05268402025103569, + "rewards/rejected": -0.12321777641773224, + "step": 2350 + }, + { + "epoch": 0.17908635604795872, + "grad_norm": 1.7329387674163914, + "learning_rate": 1.6467739391852364e-06, + "log_odds_chosen": 0.932861328125, + "log_odds_ratio": -0.492919921875, + "logits/chosen": -0.903124988079071, + "logits/rejected": -0.8291015625, + "logps/chosen": -0.696582019329071, + "logps/rejected": -1.316015601158142, + "loss": 0.972, + "nll_loss": 0.9710937738418579, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.06962890923023224, + "rewards/margins": 0.06194610521197319, + "rewards/rejected": -0.13166503608226776, + "step": 2360 + }, + { + "epoch": 0.17984519653968736, + "grad_norm": 1.6461497139628067, + "learning_rate": 1.6432960600355221e-06, + "log_odds_chosen": 1.0046875476837158, + "log_odds_ratio": -0.46044921875, + "logits/chosen": -0.933398425579071, + "logits/rejected": -0.8314453363418579, + "logps/chosen": -0.6783202886581421, + "logps/rejected": -1.338281273841858, + "loss": 0.9891, + "nll_loss": 0.9117187261581421, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06779785454273224, + "rewards/margins": 0.0659637451171875, + "rewards/rejected": -0.13381347060203552, + "step": 2370 + }, + { + "epoch": 0.180604037031416, + "grad_norm": 1.925964568167815, + "learning_rate": 1.6398401233815756e-06, + "log_odds_chosen": 1.0750000476837158, + "log_odds_ratio": -0.47919923067092896, + "logits/chosen": -0.955078125, + "logits/rejected": -0.859375, + "logps/chosen": -0.7408202886581421, + "logps/rejected": -1.4873046875, + "loss": 0.9939, + "nll_loss": 0.900585949420929, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07413329929113388, + "rewards/margins": 0.07451782375574112, + "rewards/rejected": -0.14860840141773224, + "step": 2380 + }, + { + "epoch": 0.18136287752314464, + "grad_norm": 1.8137519236514643, + "learning_rate": 1.6364058994584524e-06, + "log_odds_chosen": 0.9222656488418579, + "log_odds_ratio": -0.5028320550918579, + "logits/chosen": -0.8755859136581421, + "logits/rejected": -0.799023449420929, + "logps/chosen": -0.7115234136581421, + "logps/rejected": -1.3359375, + "loss": 0.9888, + "nll_loss": 0.9359375238418579, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07113037258386612, + "rewards/margins": 0.06235351413488388, + "rewards/rejected": -0.13352051377296448, + "step": 2390 + }, + { + "epoch": 0.18212171801487329, + "grad_norm": 1.688614794458292, + "learning_rate": 1.6329931618554522e-06, + "log_odds_chosen": 1.150390625, + "log_odds_ratio": -0.453125, + "logits/chosen": -0.930468738079071, + "logits/rejected": -0.8667968511581421, + "logps/chosen": -0.686230480670929, + "logps/rejected": -1.476171851158142, + "loss": 0.9823, + "nll_loss": 0.946484386920929, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06862793117761612, + "rewards/margins": 0.079010009765625, + "rewards/rejected": -0.14760741591453552, + "step": 2400 + }, + { + "epoch": 0.18288055850660193, + "grad_norm": 1.8014616595320827, + "learning_rate": 1.6296016874534209e-06, + "log_odds_chosen": 1.108154296875, + "log_odds_ratio": -0.458984375, + "logits/chosen": -0.911328136920929, + "logits/rejected": -0.850390613079071, + "logps/chosen": -0.693554699420929, + "logps/rejected": -1.4207031726837158, + "loss": 0.9831, + "nll_loss": 0.981640636920929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06934814155101776, + "rewards/margins": 0.07284469902515411, + "rewards/rejected": -0.14213867485523224, + "step": 2410 + }, + { + "epoch": 0.18363939899833054, + "grad_norm": 1.7304693115361636, + "learning_rate": 1.6262312563634835e-06, + "log_odds_chosen": 1.104882836341858, + "log_odds_ratio": -0.4971679747104645, + "logits/chosen": -0.998828113079071, + "logits/rejected": -0.899218738079071, + "logps/chosen": -0.7134765386581421, + "logps/rejected": -1.490820288658142, + "loss": 0.9875, + "nll_loss": 0.934765636920929, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.07139892876148224, + "rewards/margins": 0.07766113430261612, + "rewards/rejected": -0.14907225966453552, + "step": 2420 + }, + { + "epoch": 0.18439823949005918, + "grad_norm": 2.032034928793834, + "learning_rate": 1.6228816518671587e-06, + "log_odds_chosen": 1.02294921875, + "log_odds_ratio": -0.4745117127895355, + "logits/chosen": -0.957812488079071, + "logits/rejected": -0.842968761920929, + "logps/chosen": -0.710156261920929, + "logps/rejected": -1.451171875, + "loss": 0.9879, + "nll_loss": 0.9244140386581421, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07102050632238388, + "rewards/margins": 0.07415161281824112, + "rewards/rejected": -0.14511719346046448, + "step": 2430 + }, + { + "epoch": 0.18515707998178782, + "grad_norm": 2.0892961715952696, + "learning_rate": 1.619552660357832e-06, + "log_odds_chosen": 1.128198266029358, + "log_odds_ratio": -0.49091798067092896, + "logits/chosen": -0.934765636920929, + "logits/rejected": -0.8541015386581421, + "logps/chosen": -0.7372070550918579, + "logps/rejected": -1.5187499523162842, + "loss": 0.99, + "nll_loss": 0.9781249761581421, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.07374267280101776, + "rewards/margins": 0.07811889797449112, + "rewards/rejected": -0.15183106064796448, + "step": 2440 + }, + { + "epoch": 0.18591592047351646, + "grad_norm": 1.66781243666752, + "learning_rate": 1.616244071283537e-06, + "log_odds_chosen": 1.009863257408142, + "log_odds_ratio": -0.4766601622104645, + "logits/chosen": -0.9652343988418579, + "logits/rejected": -0.8421875238418579, + "logps/chosen": -0.7496093511581421, + "logps/rejected": -1.4240233898162842, + "loss": 0.9889, + "nll_loss": 0.971484363079071, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07501220703125, + "rewards/margins": 0.06739501655101776, + "rewards/rejected": -0.14240722358226776, + "step": 2450 + }, + { + "epoch": 0.1866747609652451, + "grad_norm": 1.8462614488666282, + "learning_rate": 1.6129556770910235e-06, + "log_odds_chosen": 1.1233398914337158, + "log_odds_ratio": -0.4259277284145355, + "logits/chosen": -0.985546886920929, + "logits/rejected": -0.8519531488418579, + "logps/chosen": -0.646679699420929, + "logps/rejected": -1.3787109851837158, + "loss": 0.9742, + "nll_loss": 0.9052734375, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06461181491613388, + "rewards/margins": 0.07322387397289276, + "rewards/rejected": -0.13774414360523224, + "step": 2460 + }, + { + "epoch": 0.18743360145697374, + "grad_norm": 2.077714916187083, + "learning_rate": 1.6096872731710673e-06, + "log_odds_chosen": 1.1902344226837158, + "log_odds_ratio": -0.42304688692092896, + "logits/chosen": -0.9332031011581421, + "logits/rejected": -0.84375, + "logps/chosen": -0.743457019329071, + "logps/rejected": -1.571679711341858, + "loss": 0.9835, + "nll_loss": 0.9898437261581421, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07436523586511612, + "rewards/margins": 0.0826416015625, + "rewards/rejected": -0.15708008408546448, + "step": 2470 + }, + { + "epoch": 0.18819244194870238, + "grad_norm": 1.8616129591738162, + "learning_rate": 1.6064386578049978e-06, + "log_odds_chosen": 1.16064453125, + "log_odds_ratio": -0.47441405057907104, + "logits/chosen": -0.904296875, + "logits/rejected": -0.820507824420929, + "logps/chosen": -0.693164050579071, + "logps/rejected": -1.5470702648162842, + "loss": 0.9871, + "nll_loss": 0.9693359136581421, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.06928710639476776, + "rewards/margins": 0.08541259914636612, + "rewards/rejected": -0.1546630859375, + "step": 2480 + }, + { + "epoch": 0.18895128244043102, + "grad_norm": 2.1166882079269342, + "learning_rate": 1.6032096321124046e-06, + "log_odds_chosen": 0.8385375738143921, + "log_odds_ratio": -0.5474609136581421, + "logits/chosen": -0.9267578125, + "logits/rejected": -0.8587890863418579, + "logps/chosen": -0.755175769329071, + "logps/rejected": -1.3542969226837158, + "loss": 0.9952, + "nll_loss": 0.971484363079071, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07552490383386612, + "rewards/margins": 0.05993194505572319, + "rewards/rejected": -0.13544921576976776, + "step": 2490 + }, + { + "epoch": 0.18971012293215966, + "grad_norm": 2.110336575040382, + "learning_rate": 1.6e-06, + "log_odds_chosen": 1.1823241710662842, + "log_odds_ratio": -0.46098631620407104, + "logits/chosen": -0.9212890863418579, + "logits/rejected": -0.8544921875, + "logps/chosen": -0.652636706829071, + "logps/rejected": -1.503320336341858, + "loss": 0.9806, + "nll_loss": 0.9033203125, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.06525878608226776, + "rewards/margins": 0.0849609375, + "rewards/rejected": -0.15021972358226776, + "step": 2500 + }, + { + "epoch": 0.1904689634238883, + "grad_norm": 1.6348076610742528, + "learning_rate": 1.5968095681115984e-06, + "log_odds_chosen": 1.142578125, + "log_odds_ratio": -0.486328125, + "logits/chosen": -0.9554687738418579, + "logits/rejected": -0.8511718511581421, + "logps/chosen": -0.7681640386581421, + "logps/rejected": -1.591796875, + "loss": 0.9933, + "nll_loss": 0.9615234136581421, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07675781100988388, + "rewards/margins": 0.08242187649011612, + "rewards/rejected": -0.15913085639476776, + "step": 2510 + }, + { + "epoch": 0.19122780391561695, + "grad_norm": 2.136414388223389, + "learning_rate": 1.5936381457791914e-06, + "log_odds_chosen": 1.12158203125, + "log_odds_ratio": -0.4488281309604645, + "logits/chosen": -1.003515601158142, + "logits/rejected": -0.9029296636581421, + "logps/chosen": -0.730761706829071, + "logps/rejected": -1.494726538658142, + "loss": 0.9646, + "nll_loss": 0.960156261920929, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07314453274011612, + "rewards/margins": 0.076385498046875, + "rewards/rejected": -0.14958496391773224, + "step": 2520 + }, + { + "epoch": 0.19198664440734559, + "grad_norm": 1.709686144800437, + "learning_rate": 1.590485544975088e-06, + "log_odds_chosen": 1.1706054210662842, + "log_odds_ratio": -0.43974608182907104, + "logits/chosen": -0.9906250238418579, + "logits/rejected": -0.8759765625, + "logps/chosen": -0.724804699420929, + "logps/rejected": -1.545507788658142, + "loss": 0.9767, + "nll_loss": 1.0037109851837158, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07244873046875, + "rewards/margins": 0.08211059868335724, + "rewards/rejected": -0.15458984673023224, + "step": 2530 + }, + { + "epoch": 0.19274548489907423, + "grad_norm": 1.8255597130329309, + "learning_rate": 1.5873515802650901e-06, + "log_odds_chosen": 0.9172118902206421, + "log_odds_ratio": -0.507275402545929, + "logits/chosen": -0.990429699420929, + "logits/rejected": -0.904101550579071, + "logps/chosen": -0.6786133050918579, + "logps/rejected": -1.294531226158142, + "loss": 0.9685, + "nll_loss": 0.9468749761581421, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.06785888969898224, + "rewards/margins": 0.06157226487994194, + "rewards/rejected": -0.12946777045726776, + "step": 2540 + }, + { + "epoch": 0.19350432539080287, + "grad_norm": 1.789326725731532, + "learning_rate": 1.584236068762679e-06, + "log_odds_chosen": 1.466210961341858, + "log_odds_ratio": -0.3741455078125, + "logits/chosen": -0.9769531488418579, + "logits/rejected": -0.878710925579071, + "logps/chosen": -0.6670898199081421, + "logps/rejected": -1.676367163658142, + "loss": 1.0203, + "nll_loss": 0.9556640386581421, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06667480617761612, + "rewards/margins": 0.10093994438648224, + "rewards/rejected": -0.16752929985523224, + "step": 2550 + }, + { + "epoch": 0.19426316588253148, + "grad_norm": 1.8900962347223054, + "learning_rate": 1.5811388300841894e-06, + "log_odds_chosen": 1.261132836341858, + "log_odds_ratio": -0.4341796934604645, + "logits/chosen": -0.9541015625, + "logits/rejected": -0.8617187738418579, + "logps/chosen": -0.699023425579071, + "logps/rejected": -1.5593750476837158, + "loss": 0.9719, + "nll_loss": 1.002539038658142, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06990966945886612, + "rewards/margins": 0.08603973686695099, + "rewards/rejected": -0.15598145127296448, + "step": 2560 + }, + { + "epoch": 0.19502200637426012, + "grad_norm": 1.6750647053286092, + "learning_rate": 1.5780596863049431e-06, + "log_odds_chosen": 1.4158203601837158, + "log_odds_ratio": -0.38447266817092896, + "logits/chosen": -0.9541015625, + "logits/rejected": -0.8203125, + "logps/chosen": -0.653613269329071, + "logps/rejected": -1.607031226158142, + "loss": 0.9651, + "nll_loss": 0.8833984136581421, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06544189155101776, + "rewards/margins": 0.09538574516773224, + "rewards/rejected": -0.16069336235523224, + "step": 2570 + }, + { + "epoch": 0.19578084686598876, + "grad_norm": 1.9326493464721346, + "learning_rate": 1.5749984619163156e-06, + "log_odds_chosen": 1.1027343273162842, + "log_odds_ratio": -0.4658203125, + "logits/chosen": -0.9818359613418579, + "logits/rejected": -0.8500000238418579, + "logps/chosen": -0.693066418170929, + "logps/rejected": -1.4611327648162842, + "loss": 0.9866, + "nll_loss": 0.9554687738418579, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06929931789636612, + "rewards/margins": 0.07676391303539276, + "rewards/rejected": -0.14606933295726776, + "step": 2580 + }, + { + "epoch": 0.1965396873577174, + "grad_norm": 1.67777232703704, + "learning_rate": 1.5719549837837187e-06, + "log_odds_chosen": 0.979296863079071, + "log_odds_ratio": -0.48750001192092896, + "logits/chosen": -1.03125, + "logits/rejected": -0.916210949420929, + "logps/chosen": -0.693652331829071, + "logps/rejected": -1.3572266101837158, + "loss": 0.9592, + "nll_loss": 0.8955078125, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.06942138820886612, + "rewards/margins": 0.06621094048023224, + "rewards/rejected": -0.13564452528953552, + "step": 2590 + }, + { + "epoch": 0.19729852784944604, + "grad_norm": 1.847138545524677, + "learning_rate": 1.5689290811054722e-06, + "log_odds_chosen": 1.246191382408142, + "log_odds_ratio": -0.446044921875, + "logits/chosen": -0.973437488079071, + "logits/rejected": -0.8705078363418579, + "logps/chosen": -0.675000011920929, + "logps/rejected": -1.5144531726837158, + "loss": 0.9621, + "nll_loss": 0.9115234613418579, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06757812201976776, + "rewards/margins": 0.08388672024011612, + "rewards/rejected": -0.1513671875, + "step": 2600 + }, + { + "epoch": 0.19805736834117468, + "grad_norm": 1.8218694874948793, + "learning_rate": 1.5659205853725426e-06, + "log_odds_chosen": 1.093359351158142, + "log_odds_ratio": -0.47246092557907104, + "logits/chosen": -0.990234375, + "logits/rejected": -0.8970702886581421, + "logps/chosen": -0.67236328125, + "logps/rejected": -1.4619140625, + "loss": 0.9777, + "nll_loss": 0.9501953125, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06721191108226776, + "rewards/margins": 0.07899780571460724, + "rewards/rejected": -0.14621582627296448, + "step": 2610 + }, + { + "epoch": 0.19881620883290332, + "grad_norm": 1.837412074872585, + "learning_rate": 1.562929330329127e-06, + "log_odds_chosen": 0.895214855670929, + "log_odds_ratio": -0.4990234375, + "logits/chosen": -0.9296875, + "logits/rejected": -0.865234375, + "logps/chosen": -0.706835925579071, + "logps/rejected": -1.3162109851837158, + "loss": 0.9985, + "nll_loss": 1.008398413658142, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.07066650688648224, + "rewards/margins": 0.06090698391199112, + "rewards/rejected": -0.13166503608226776, + "step": 2620 + }, + { + "epoch": 0.19957504932463196, + "grad_norm": 1.7043281951830767, + "learning_rate": 1.5599551519340636e-06, + "log_odds_chosen": 1.339453101158142, + "log_odds_ratio": -0.38862305879592896, + "logits/chosen": -1.014257788658142, + "logits/rejected": -0.9058593511581421, + "logps/chosen": -0.6318359375, + "logps/rejected": -1.546289086341858, + "loss": 0.9407, + "nll_loss": 0.896679699420929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06322021782398224, + "rewards/margins": 0.091400146484375, + "rewards/rejected": -0.15456542372703552, + "step": 2630 + }, + { + "epoch": 0.2003338898163606, + "grad_norm": 1.7931003101605318, + "learning_rate": 1.556997888323046e-06, + "log_odds_chosen": 1.1256835460662842, + "log_odds_ratio": -0.44111329317092896, + "logits/chosen": -0.958984375, + "logits/rejected": -0.8802734613418579, + "logps/chosen": -0.689257800579071, + "logps/rejected": -1.437890648841858, + "loss": 0.9724, + "nll_loss": 0.929492175579071, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06889648735523224, + "rewards/margins": 0.0749664306640625, + "rewards/rejected": -0.14387206733226776, + "step": 2640 + }, + { + "epoch": 0.20109273030808925, + "grad_norm": 1.571974698546549, + "learning_rate": 1.5540573797716226e-06, + "log_odds_chosen": 1.1706054210662842, + "log_odds_ratio": -0.45585936307907104, + "logits/chosen": -0.959765613079071, + "logits/rejected": -0.8968750238418579, + "logps/chosen": -0.676953136920929, + "logps/rejected": -1.4599609375, + "loss": 0.9565, + "nll_loss": 0.900585949420929, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06772460788488388, + "rewards/margins": 0.078369140625, + "rewards/rejected": -0.14604492485523224, + "step": 2650 + }, + { + "epoch": 0.2018515707998179, + "grad_norm": 1.8202591383722864, + "learning_rate": 1.5511334686589623e-06, + "log_odds_chosen": 1.417089819908142, + "log_odds_ratio": -0.3907714784145355, + "logits/chosen": -0.9935547113418579, + "logits/rejected": -0.895312488079071, + "logps/chosen": -0.65087890625, + "logps/rejected": -1.615625023841858, + "loss": 0.9577, + "nll_loss": 0.876171886920929, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06508789211511612, + "rewards/margins": 0.09650878608226776, + "rewards/rejected": -0.16157226264476776, + "step": 2660 + }, + { + "epoch": 0.20261041129154653, + "grad_norm": 1.8834458505225085, + "learning_rate": 1.548225999432367e-06, + "log_odds_chosen": 1.3351562023162842, + "log_odds_ratio": -0.41259765625, + "logits/chosen": -0.982617199420929, + "logits/rejected": -0.847851574420929, + "logps/chosen": -0.672167956829071, + "logps/rejected": -1.5802733898162842, + "loss": 0.9765, + "nll_loss": 0.915820300579071, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06721191108226776, + "rewards/margins": 0.09076537936925888, + "rewards/rejected": -0.15788574516773224, + "step": 2670 + }, + { + "epoch": 0.20336925178327517, + "grad_norm": 2.0250133124028173, + "learning_rate": 1.5453348185725114e-06, + "log_odds_chosen": 0.9356933832168579, + "log_odds_ratio": -0.510546863079071, + "logits/chosen": -0.9546874761581421, + "logits/rejected": -0.8519531488418579, + "logps/chosen": -0.7032226324081421, + "logps/rejected": -1.329687476158142, + "loss": 1.0076, + "nll_loss": 0.931445300579071, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07032470405101776, + "rewards/margins": 0.06263580173254013, + "rewards/rejected": -0.13298340141773224, + "step": 2680 + }, + { + "epoch": 0.20412809227500378, + "grad_norm": 2.386913601696167, + "learning_rate": 1.542459774559398e-06, + "log_odds_chosen": 1.2929198741912842, + "log_odds_ratio": -0.4048828184604645, + "logits/chosen": -0.96484375, + "logits/rejected": -0.892578125, + "logps/chosen": -0.708789050579071, + "logps/rejected": -1.5808594226837158, + "loss": 0.9777, + "nll_loss": 0.9580078125, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.07087402045726776, + "rewards/margins": 0.08717040717601776, + "rewards/rejected": -0.15803222358226776, + "step": 2690 + }, + { + "epoch": 0.20488693276673242, + "grad_norm": 1.8473571912546065, + "learning_rate": 1.539600717839002e-06, + "log_odds_chosen": 1.0650513172149658, + "log_odds_ratio": -0.501953125, + "logits/chosen": -0.962695300579071, + "logits/rejected": -0.890429675579071, + "logps/chosen": -0.742968738079071, + "logps/rejected": -1.5226562023162842, + "loss": 0.9632, + "nll_loss": 0.9105468988418579, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07431640475988388, + "rewards/margins": 0.07787322998046875, + "rewards/rejected": -0.15224608778953552, + "step": 2700 + }, + { + "epoch": 0.20564577325846106, + "grad_norm": 1.872727304303063, + "learning_rate": 1.536757500790597e-06, + "log_odds_chosen": 1.247460961341858, + "log_odds_ratio": -0.4300781190395355, + "logits/chosen": -0.9798828363418579, + "logits/rejected": -0.8876953125, + "logps/chosen": -0.7367187738418579, + "logps/rejected": -1.5886719226837158, + "loss": 0.9677, + "nll_loss": 0.928906261920929, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07359619438648224, + "rewards/margins": 0.08521118015050888, + "rewards/rejected": -0.15895995497703552, + "step": 2710 + }, + { + "epoch": 0.2064046137501897, + "grad_norm": 1.7627286806989997, + "learning_rate": 1.5339299776947407e-06, + "log_odds_chosen": 1.3813965320587158, + "log_odds_ratio": -0.44189453125, + "logits/chosen": -0.9408203363418579, + "logits/rejected": -0.8451172113418579, + "logps/chosen": -0.6982421875, + "logps/rejected": -1.683203101158142, + "loss": 0.967, + "nll_loss": 0.920703113079071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06981201469898224, + "rewards/margins": 0.09839019924402237, + "rewards/rejected": -0.16828612983226776, + "step": 2720 + }, + { + "epoch": 0.20716345424191834, + "grad_norm": 2.225315838235998, + "learning_rate": 1.5311180047019054e-06, + "log_odds_chosen": 0.9217284917831421, + "log_odds_ratio": -0.5398925542831421, + "logits/chosen": -0.8974609375, + "logits/rejected": -0.8558593988418579, + "logps/chosen": -0.7349609136581421, + "logps/rejected": -1.3703124523162842, + "loss": 0.9745, + "nll_loss": 0.9449218511581421, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07352294772863388, + "rewards/margins": 0.06348266452550888, + "rewards/rejected": -0.136962890625, + "step": 2730 + }, + { + "epoch": 0.20792229473364698, + "grad_norm": 1.7123621724820337, + "learning_rate": 1.5283214398017402e-06, + "log_odds_chosen": 1.113378882408142, + "log_odds_ratio": -0.4542480409145355, + "logits/chosen": -1.009765625, + "logits/rejected": -0.9136718511581421, + "logps/chosen": -0.660937488079071, + "logps/rejected": -1.4210937023162842, + "loss": 0.9636, + "nll_loss": 0.865429699420929, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.0660400390625, + "rewards/margins": 0.07600097358226776, + "rewards/rejected": -0.14206543564796448, + "step": 2740 + }, + { + "epoch": 0.20868113522537562, + "grad_norm": 1.8912075942460569, + "learning_rate": 1.5255401427929477e-06, + "log_odds_chosen": 1.3098633289337158, + "log_odds_ratio": -0.4039062559604645, + "logits/chosen": -0.9701172113418579, + "logits/rejected": -0.884570300579071, + "logps/chosen": -0.669628918170929, + "logps/rejected": -1.5916016101837158, + "loss": 0.9523, + "nll_loss": 0.807910144329071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06696777045726776, + "rewards/margins": 0.09216918796300888, + "rewards/rejected": -0.15913085639476776, + "step": 2750 + }, + { + "epoch": 0.20943997571710427, + "grad_norm": 1.9111616410727177, + "learning_rate": 1.5227739752537617e-06, + "log_odds_chosen": 1.12451171875, + "log_odds_ratio": -0.4161621034145355, + "logits/chosen": -0.992382824420929, + "logits/rejected": -0.8779296875, + "logps/chosen": -0.6781250238418579, + "logps/rejected": -1.424414038658142, + "loss": 0.9826, + "nll_loss": 0.9384765625, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06783447414636612, + "rewards/margins": 0.07457275688648224, + "rewards/rejected": -0.1424560546875, + "step": 2760 + }, + { + "epoch": 0.2101988162088329, + "grad_norm": 2.054694787863413, + "learning_rate": 1.5200228005130127e-06, + "log_odds_chosen": 1.0719726085662842, + "log_odds_ratio": -0.48759764432907104, + "logits/chosen": -0.9869140386581421, + "logits/rejected": -0.9039062261581421, + "logps/chosen": -0.7115234136581421, + "logps/rejected": -1.458398461341858, + "loss": 0.9641, + "nll_loss": 0.932421863079071, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07111816108226776, + "rewards/margins": 0.07462158054113388, + "rewards/rejected": -0.145751953125, + "step": 2770 + }, + { + "epoch": 0.21095765670056155, + "grad_norm": 1.9682083058062396, + "learning_rate": 1.5172864836217631e-06, + "log_odds_chosen": 1.0216796398162842, + "log_odds_ratio": -0.513427734375, + "logits/chosen": -0.9623047113418579, + "logits/rejected": -0.8841797113418579, + "logps/chosen": -0.7049804925918579, + "logps/rejected": -1.436914086341858, + "loss": 0.9801, + "nll_loss": 0.90283203125, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.07052002102136612, + "rewards/margins": 0.07314453274011612, + "rewards/rejected": -0.14362792670726776, + "step": 2780 + }, + { + "epoch": 0.2117164971922902, + "grad_norm": 1.7076823298104469, + "learning_rate": 1.514564891325506e-06, + "log_odds_chosen": 1.11669921875, + "log_odds_ratio": -0.45429688692092896, + "logits/chosen": -0.9652343988418579, + "logits/rejected": -0.8560546636581421, + "logps/chosen": -0.624707043170929, + "logps/rejected": -1.333984375, + "loss": 0.9698, + "nll_loss": 0.918749988079071, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06241454929113388, + "rewards/margins": 0.07102050632238388, + "rewards/rejected": -0.13339844346046448, + "step": 2790 + }, + { + "epoch": 0.21247533768401883, + "grad_norm": 1.8454933653782706, + "learning_rate": 1.5118578920369086e-06, + "log_odds_chosen": 1.041650414466858, + "log_odds_ratio": -0.47929686307907104, + "logits/chosen": -0.971484363079071, + "logits/rejected": -0.8687499761581421, + "logps/chosen": -0.73779296875, + "logps/rejected": -1.4578125476837158, + "loss": 0.9761, + "nll_loss": 0.9339843988418579, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07375488430261612, + "rewards/margins": 0.07195434719324112, + "rewards/rejected": -0.14582519233226776, + "step": 2800 + }, + { + "epoch": 0.21323417817574747, + "grad_norm": 2.3111648063323242, + "learning_rate": 1.5091653558090898e-06, + "log_odds_chosen": 1.108007788658142, + "log_odds_ratio": -0.448486328125, + "logits/chosen": -0.935351550579071, + "logits/rejected": -0.8687499761581421, + "logps/chosen": -0.7041015625, + "logps/rejected": -1.473046898841858, + "loss": 0.9538, + "nll_loss": 0.876757800579071, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.07039795070886612, + "rewards/margins": 0.076904296875, + "rewards/rejected": -0.14731445908546448, + "step": 2810 + }, + { + "epoch": 0.2139930186674761, + "grad_norm": 4.148395754218525, + "learning_rate": 1.506487154309419e-06, + "log_odds_chosen": 1.150976538658142, + "log_odds_ratio": -0.45585936307907104, + "logits/chosen": -0.928515613079071, + "logits/rejected": -0.889453113079071, + "logps/chosen": -0.7118164300918579, + "logps/rejected": -1.466406226158142, + "loss": 0.9455, + "nll_loss": 0.878710925579071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.07124023139476776, + "rewards/margins": 0.07537536323070526, + "rewards/rejected": -0.14663085341453552, + "step": 2820 + }, + { + "epoch": 0.21475185915920472, + "grad_norm": 1.9726707897843883, + "learning_rate": 1.5038231607938247e-06, + "log_odds_chosen": 1.0893065929412842, + "log_odds_ratio": -0.48725587129592896, + "logits/chosen": -0.9554687738418579, + "logits/rejected": -0.862109363079071, + "logps/chosen": -0.7093750238418579, + "logps/rejected": -1.4777343273162842, + "loss": 0.9787, + "nll_loss": 0.935742199420929, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07087402045726776, + "rewards/margins": 0.07680968940258026, + "rewards/rejected": -0.1475830078125, + "step": 2830 + }, + { + "epoch": 0.21551069965093336, + "grad_norm": 1.8387982254575808, + "learning_rate": 1.501173250081603e-06, + "log_odds_chosen": 1.326562523841858, + "log_odds_ratio": -0.43168944120407104, + "logits/chosen": -0.973828136920929, + "logits/rejected": -0.841015636920929, + "logps/chosen": -0.6865234375, + "logps/rejected": -1.63671875, + "loss": 0.9605, + "nll_loss": 0.9320312738418579, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.068603515625, + "rewards/margins": 0.09507445991039276, + "rewards/rejected": -0.16372069716453552, + "step": 2840 + }, + { + "epoch": 0.216269540142662, + "grad_norm": 2.2021903588634215, + "learning_rate": 1.4985372985307103e-06, + "log_odds_chosen": 1.35150146484375, + "log_odds_ratio": -0.40996092557907104, + "logits/chosen": -0.947265625, + "logits/rejected": -0.8636718988418579, + "logps/chosen": -0.686328113079071, + "logps/rejected": -1.6365234851837158, + "loss": 0.9485, + "nll_loss": 0.9105468988418579, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06868896633386612, + "rewards/margins": 0.09504852443933487, + "rewards/rejected": -0.16367188096046448, + "step": 2850 + }, + { + "epoch": 0.21702838063439064, + "grad_norm": 1.7415261017566501, + "learning_rate": 1.4959151840135313e-06, + "log_odds_chosen": 1.178320288658142, + "log_odds_ratio": -0.4439453184604645, + "logits/chosen": -0.990429699420929, + "logits/rejected": -0.90234375, + "logps/chosen": -0.7320312261581421, + "logps/rejected": -1.5597655773162842, + "loss": 0.9675, + "nll_loss": 0.963085949420929, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07321777194738388, + "rewards/margins": 0.08275756984949112, + "rewards/rejected": -0.15598145127296448, + "step": 2860 + }, + { + "epoch": 0.21778722112611928, + "grad_norm": 1.7962173674694326, + "learning_rate": 1.4933067858931148e-06, + "log_odds_chosen": 1.2593505382537842, + "log_odds_ratio": -0.41767579317092896, + "logits/chosen": -0.9828125238418579, + "logits/rejected": -0.885546863079071, + "logps/chosen": -0.66357421875, + "logps/rejected": -1.4919922351837158, + "loss": 0.9735, + "nll_loss": 0.9029296636581421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06641845405101776, + "rewards/margins": 0.08284759521484375, + "rewards/rejected": -0.14924316108226776, + "step": 2870 + }, + { + "epoch": 0.21854606161784793, + "grad_norm": 1.8444965000372955, + "learning_rate": 1.4907119849998597e-06, + "log_odds_chosen": 1.254296898841858, + "log_odds_ratio": -0.45166015625, + "logits/chosen": -0.9599609375, + "logits/rejected": -0.887890636920929, + "logps/chosen": -0.6917968988418579, + "logps/rejected": -1.537109375, + "loss": 0.9678, + "nll_loss": 0.9164062738418579, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.069091796875, + "rewards/margins": 0.08457031100988388, + "rewards/rejected": -0.15373535454273224, + "step": 2880 + }, + { + "epoch": 0.21930490210957657, + "grad_norm": 1.7432037993290854, + "learning_rate": 1.488130663608649e-06, + "log_odds_chosen": 1.1090819835662842, + "log_odds_ratio": -0.43535155057907104, + "logits/chosen": -0.935546875, + "logits/rejected": -0.8447265625, + "logps/chosen": -0.64599609375, + "logps/rejected": -1.378515601158142, + "loss": 0.9707, + "nll_loss": 0.8880859613418579, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.0645751953125, + "rewards/margins": 0.07330322265625, + "rewards/rejected": -0.13786621391773224, + "step": 2890 + }, + { + "epoch": 0.2200637426013052, + "grad_norm": 1.8270299370282952, + "learning_rate": 1.4855627054164149e-06, + "log_odds_chosen": 1.2333984375, + "log_odds_ratio": -0.42988282442092896, + "logits/chosen": -0.9619140625, + "logits/rejected": -0.8291015625, + "logps/chosen": -0.6923828125, + "logps/rejected": -1.574609398841858, + "loss": 0.9413, + "nll_loss": 0.9408203363418579, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06927490234375, + "rewards/margins": 0.08817748725414276, + "rewards/rejected": -0.157470703125, + "step": 2900 + }, + { + "epoch": 0.22082258309303385, + "grad_norm": 1.9292240039959558, + "learning_rate": 1.4830079955201294e-06, + "log_odds_chosen": 1.2151367664337158, + "log_odds_ratio": -0.44414061307907104, + "logits/chosen": -0.9921875, + "logits/rejected": -0.8570312261581421, + "logps/chosen": -0.715039074420929, + "logps/rejected": -1.593164086341858, + "loss": 0.9494, + "nll_loss": 0.9208984375, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07150878757238388, + "rewards/margins": 0.08788452297449112, + "rewards/rejected": -0.15939942002296448, + "step": 2910 + }, + { + "epoch": 0.2215814235847625, + "grad_norm": 2.581943848053353, + "learning_rate": 1.4804664203952103e-06, + "log_odds_chosen": 1.3484375476837158, + "log_odds_ratio": -0.41411131620407104, + "logits/chosen": -0.9193359613418579, + "logits/rejected": -0.8314453363418579, + "logps/chosen": -0.697460949420929, + "logps/rejected": -1.636328101158142, + "loss": 0.9562, + "nll_loss": 0.967968761920929, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06971435248851776, + "rewards/margins": 0.09394530951976776, + "rewards/rejected": -0.16376952826976776, + "step": 2920 + }, + { + "epoch": 0.22234026407649113, + "grad_norm": 1.7988904180171552, + "learning_rate": 1.4779378678743327e-06, + "log_odds_chosen": 1.0883057117462158, + "log_odds_ratio": -0.501660168170929, + "logits/chosen": -0.9693359136581421, + "logits/rejected": -0.90234375, + "logps/chosen": -0.715136706829071, + "logps/rejected": -1.501953125, + "loss": 0.9604, + "nll_loss": 0.933789074420929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07152099907398224, + "rewards/margins": 0.07857055962085724, + "rewards/rejected": -0.15004882216453552, + "step": 2930 + }, + { + "epoch": 0.22309910456821977, + "grad_norm": 1.9160010373129888, + "learning_rate": 1.4754222271266348e-06, + "log_odds_chosen": 1.13330078125, + "log_odds_ratio": -0.47773438692092896, + "logits/chosen": -0.9613281488418579, + "logits/rejected": -0.8382812738418579, + "logps/chosen": -0.697070300579071, + "logps/rejected": -1.5056641101837158, + "loss": 0.9634, + "nll_loss": 0.8658202886581421, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.06971435248851776, + "rewards/margins": 0.08093567192554474, + "rewards/rejected": -0.15053710341453552, + "step": 2940 + }, + { + "epoch": 0.2238579450599484, + "grad_norm": 1.861090283579612, + "learning_rate": 1.4729193886373175e-06, + "log_odds_chosen": 1.147375464439392, + "log_odds_ratio": -0.4544921815395355, + "logits/chosen": -0.9873046875, + "logits/rejected": -0.8873046636581421, + "logps/chosen": -0.658203125, + "logps/rejected": -1.4572265148162842, + "loss": 0.9554, + "nll_loss": 0.822460949420929, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06580810248851776, + "rewards/margins": 0.07994689792394638, + "rewards/rejected": -0.14572754502296448, + "step": 2950 + }, + { + "epoch": 0.22461678555167705, + "grad_norm": 2.8696331367440693, + "learning_rate": 1.4704292441876156e-06, + "log_odds_chosen": 1.3240234851837158, + "log_odds_ratio": -0.398193359375, + "logits/chosen": -0.9740234613418579, + "logits/rejected": -0.857226550579071, + "logps/chosen": -0.65380859375, + "logps/rejected": -1.5812499523162842, + "loss": 0.949, + "nll_loss": 0.836718738079071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.0654296875, + "rewards/margins": 0.09274597465991974, + "rewards/rejected": -0.15810546278953552, + "step": 2960 + }, + { + "epoch": 0.22537562604340566, + "grad_norm": 1.6682423889188396, + "learning_rate": 1.4679516868351474e-06, + "log_odds_chosen": 1.2998046875, + "log_odds_ratio": -0.4522460997104645, + "logits/chosen": -0.9380859136581421, + "logits/rejected": -0.828320324420929, + "logps/chosen": -0.75, + "logps/rejected": -1.726953148841858, + "loss": 0.9733, + "nll_loss": 0.953320324420929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07496337592601776, + "rewards/margins": 0.09773864597082138, + "rewards/rejected": -0.1727294921875, + "step": 2970 + }, + { + "epoch": 0.2261344665351343, + "grad_norm": 1.8546074916340258, + "learning_rate": 1.4654866108946234e-06, + "log_odds_chosen": 0.9668945074081421, + "log_odds_ratio": -0.508056640625, + "logits/chosen": -0.953125, + "logits/rejected": -0.862109363079071, + "logps/chosen": -0.647753894329071, + "logps/rejected": -1.292382836341858, + "loss": 0.9674, + "nll_loss": 0.953320324420929, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.06480713188648224, + "rewards/margins": 0.06438598781824112, + "rewards/rejected": -0.12919922173023224, + "step": 2980 + }, + { + "epoch": 0.22689330702686294, + "grad_norm": 1.7596453202428828, + "learning_rate": 1.4630339119189101e-06, + "log_odds_chosen": 0.931640625, + "log_odds_ratio": -0.5057617425918579, + "logits/chosen": -0.935546875, + "logits/rejected": -0.845898449420929, + "logps/chosen": -0.7242187261581421, + "logps/rejected": -1.37890625, + "loss": 0.9486, + "nll_loss": 0.899609386920929, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07239989936351776, + "rewards/margins": 0.06554565578699112, + "rewards/rejected": -0.13789062201976776, + "step": 2990 + }, + { + "epoch": 0.22765214751859159, + "grad_norm": 1.847450689305971, + "learning_rate": 1.4605934866804429e-06, + "log_odds_chosen": 1.257592797279358, + "log_odds_ratio": -0.400146484375, + "logits/chosen": -0.987500011920929, + "logits/rejected": -0.8763672113418579, + "logps/chosen": -0.6636718511581421, + "logps/rejected": -1.506250023841858, + "loss": 0.9456, + "nll_loss": 0.8394531011581421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06638183444738388, + "rewards/margins": 0.08422088623046875, + "rewards/rejected": -0.15065917372703552, + "step": 3000 + }, + { + "epoch": 0.22841098801032023, + "grad_norm": 1.7257030329384566, + "learning_rate": 1.4581652331529784e-06, + "log_odds_chosen": 1.3411133289337158, + "log_odds_ratio": -0.4209960997104645, + "logits/chosen": -0.9423828125, + "logits/rejected": -0.8199218511581421, + "logps/chosen": -0.6571289300918579, + "logps/rejected": -1.5955078601837158, + "loss": 0.9403, + "nll_loss": 0.8746093511581421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06572265923023224, + "rewards/margins": 0.09383697807788849, + "rewards/rejected": -0.15966796875, + "step": 3010 + }, + { + "epoch": 0.22916982850204887, + "grad_norm": 1.6543337971685594, + "learning_rate": 1.4557490504936778e-06, + "log_odds_chosen": 0.958984375, + "log_odds_ratio": -0.4776367247104645, + "logits/chosen": -0.913867175579071, + "logits/rejected": -0.82421875, + "logps/chosen": -0.692187488079071, + "logps/rejected": -1.3244140148162842, + "loss": 0.9727, + "nll_loss": 0.958203136920929, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06922607123851776, + "rewards/margins": 0.06323852390050888, + "rewards/rejected": -0.132568359375, + "step": 3020 + }, + { + "epoch": 0.2299286689937775, + "grad_norm": 1.9978653383725091, + "learning_rate": 1.453344839025519e-06, + "log_odds_chosen": 1.125830054283142, + "log_odds_ratio": -0.46538084745407104, + "logits/chosen": -0.9478515386581421, + "logits/rejected": -0.8636718988418579, + "logps/chosen": -0.657910168170929, + "logps/rejected": -1.415624976158142, + "loss": 0.943, + "nll_loss": 0.846484363079071, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06577148288488388, + "rewards/margins": 0.07586517184972763, + "rewards/rejected": -0.1416015625, + "step": 3030 + }, + { + "epoch": 0.23068750948550615, + "grad_norm": 1.9001759526352406, + "learning_rate": 1.4509525002200234e-06, + "log_odds_chosen": 1.4861938953399658, + "log_odds_ratio": -0.40898436307907104, + "logits/chosen": -0.9076172113418579, + "logits/rejected": -0.833789050579071, + "logps/chosen": -0.7176758050918579, + "logps/rejected": -1.7810547351837158, + "loss": 0.9498, + "nll_loss": 0.9462890625, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.07175292819738388, + "rewards/margins": 0.10638885200023651, + "rewards/rejected": -0.17822265625, + "step": 3040 + }, + { + "epoch": 0.2314463499772348, + "grad_norm": 1.812933320589776, + "learning_rate": 1.4485719366802965e-06, + "log_odds_chosen": 1.245507836341858, + "log_odds_ratio": -0.4149414002895355, + "logits/chosen": -0.9673827886581421, + "logits/rejected": -0.8486328125, + "logps/chosen": -0.669140636920929, + "logps/rejected": -1.4578125476837158, + "loss": 0.9417, + "nll_loss": 0.8720703125, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06687011569738388, + "rewards/margins": 0.07879638671875, + "rewards/rejected": -0.14580078423023224, + "step": 3050 + }, + { + "epoch": 0.23220519046896343, + "grad_norm": 1.787462020163273, + "learning_rate": 1.4462030521243742e-06, + "log_odds_chosen": 1.0899536609649658, + "log_odds_ratio": -0.4447265565395355, + "logits/chosen": -0.9195312261581421, + "logits/rejected": -0.7939453125, + "logps/chosen": -0.6776367425918579, + "logps/rejected": -1.380859375, + "loss": 0.9589, + "nll_loss": 0.9677734375, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06778564304113388, + "rewards/margins": 0.07031555473804474, + "rewards/rejected": -0.13803711533546448, + "step": 3060 + }, + { + "epoch": 0.23296403096069207, + "grad_norm": 1.6858354000752622, + "learning_rate": 1.443845751368867e-06, + "log_odds_chosen": 1.0482909679412842, + "log_odds_ratio": -0.48193359375, + "logits/chosen": -0.9501953125, + "logits/rejected": -0.8919922113418579, + "logps/chosen": -0.6900390386581421, + "logps/rejected": -1.3957030773162842, + "loss": 0.9766, + "nll_loss": 0.935546875, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06901855766773224, + "rewards/margins": 0.07039795070886612, + "rewards/rejected": -0.13947753608226776, + "step": 3070 + }, + { + "epoch": 0.2337228714524207, + "grad_norm": 1.8339758263485157, + "learning_rate": 1.4414999403128943e-06, + "log_odds_chosen": 1.2374999523162842, + "log_odds_ratio": -0.44316405057907104, + "logits/chosen": -0.887890636920929, + "logits/rejected": -0.8041015863418579, + "logps/chosen": -0.6859375238418579, + "logps/rejected": -1.5460937023162842, + "loss": 0.9417, + "nll_loss": 0.9515625238418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06850586086511612, + "rewards/margins": 0.08607177436351776, + "rewards/rejected": -0.15461425483226776, + "step": 3080 + }, + { + "epoch": 0.23448171194414935, + "grad_norm": 2.068493502798679, + "learning_rate": 1.439165525922309e-06, + "log_odds_chosen": 1.157812476158142, + "log_odds_ratio": -0.4925781190395355, + "logits/chosen": -0.904492199420929, + "logits/rejected": -0.8500000238418579, + "logps/chosen": -0.6498047113418579, + "logps/rejected": -1.4328124523162842, + "loss": 0.9524, + "nll_loss": 0.868359386920929, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.06501464545726776, + "rewards/margins": 0.07831726223230362, + "rewards/rejected": -0.1434326171875, + "step": 3090 + }, + { + "epoch": 0.235240552435878, + "grad_norm": 1.8901274723008876, + "learning_rate": 1.4368424162141992e-06, + "log_odds_chosen": 1.166015625, + "log_odds_ratio": -0.482177734375, + "logits/chosen": -0.915820300579071, + "logits/rejected": -0.81640625, + "logps/chosen": -0.7708984613418579, + "logps/rejected": -1.6150391101837158, + "loss": 0.948, + "nll_loss": 0.9623047113418579, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07711181789636612, + "rewards/margins": 0.08429565280675888, + "rewards/rejected": -0.1614990234375, + "step": 3100 + }, + { + "epoch": 0.2359993929276066, + "grad_norm": 1.9627846293262432, + "learning_rate": 1.434530520241665e-06, + "log_odds_chosen": 1.162011742591858, + "log_odds_ratio": -0.47773438692092896, + "logits/chosen": -0.9302734136581421, + "logits/rejected": -0.835742175579071, + "logps/chosen": -0.658496081829071, + "logps/rejected": -1.464453101158142, + "loss": 0.9522, + "nll_loss": 0.9443359375, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06583251804113388, + "rewards/margins": 0.08067016303539276, + "rewards/rejected": -0.14645996689796448, + "step": 3110 + }, + { + "epoch": 0.23675823341933525, + "grad_norm": 1.8470239258734957, + "learning_rate": 1.4322297480788657e-06, + "log_odds_chosen": 1.0634765625, + "log_odds_ratio": -0.49458009004592896, + "logits/chosen": -0.911914050579071, + "logits/rejected": -0.8257812261581421, + "logps/chosen": -0.753125011920929, + "logps/rejected": -1.4982421398162842, + "loss": 0.9471, + "nll_loss": 0.931835949420929, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07525634765625, + "rewards/margins": 0.074554443359375, + "rewards/rejected": -0.14982910454273224, + "step": 3120 + }, + { + "epoch": 0.2375170739110639, + "grad_norm": 1.9227094808395646, + "learning_rate": 1.4299400108063247e-06, + "log_odds_chosen": 1.065527319908142, + "log_odds_ratio": -0.4964843690395355, + "logits/chosen": -0.9326171875, + "logits/rejected": -0.841796875, + "logps/chosen": -0.6968749761581421, + "logps/rejected": -1.4296875, + "loss": 0.92, + "nll_loss": 0.9214843511581421, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.0697021484375, + "rewards/margins": 0.07349243015050888, + "rewards/rejected": -0.14306640625, + "step": 3130 + }, + { + "epoch": 0.23827591440279253, + "grad_norm": 1.8725285434199292, + "learning_rate": 1.4276612204964992e-06, + "log_odds_chosen": 1.26513671875, + "log_odds_ratio": -0.45307618379592896, + "logits/chosen": -0.918164074420929, + "logits/rejected": -0.8291015625, + "logps/chosen": -0.675585925579071, + "logps/rejected": -1.580078125, + "loss": 0.9348, + "nll_loss": 0.960742175579071, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06754150241613388, + "rewards/margins": 0.09055785834789276, + "rewards/rejected": -0.15803222358226776, + "step": 3140 + }, + { + "epoch": 0.23903475489452117, + "grad_norm": 1.7486311411327649, + "learning_rate": 1.4253932901995967e-06, + "log_odds_chosen": 1.062890648841858, + "log_odds_ratio": -0.4705566465854645, + "logits/chosen": -0.893359363079071, + "logits/rejected": -0.814257800579071, + "logps/chosen": -0.67822265625, + "logps/rejected": -1.414648413658142, + "loss": 0.9437, + "nll_loss": 0.8882812261581421, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.06781005859375, + "rewards/margins": 0.07365112006664276, + "rewards/rejected": -0.14145508408546448, + "step": 3150 + }, + { + "epoch": 0.2397935953862498, + "grad_norm": 1.871562780745004, + "learning_rate": 1.42313613392964e-06, + "log_odds_chosen": 1.2360718250274658, + "log_odds_ratio": -0.41899412870407104, + "logits/chosen": -0.942187488079071, + "logits/rejected": -0.836718738079071, + "logps/chosen": -0.6473633050918579, + "logps/rejected": -1.494140625, + "loss": 0.927, + "nll_loss": 0.8384765386581421, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06468506157398224, + "rewards/margins": 0.08463592827320099, + "rewards/rejected": -0.14943847060203552, + "step": 3160 + }, + { + "epoch": 0.24055243587797845, + "grad_norm": 1.9319998881984413, + "learning_rate": 1.4208896666507756e-06, + "log_odds_chosen": 1.1755859851837158, + "log_odds_ratio": -0.47407227754592896, + "logits/chosen": -0.883984386920929, + "logits/rejected": -0.807421863079071, + "logps/chosen": -0.6717773675918579, + "logps/rejected": -1.48046875, + "loss": 0.9449, + "nll_loss": 0.9410156011581421, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06711425632238388, + "rewards/margins": 0.08091125637292862, + "rewards/rejected": -0.1480712890625, + "step": 3170 + }, + { + "epoch": 0.2413112763697071, + "grad_norm": 2.812202784971474, + "learning_rate": 1.4186538042638173e-06, + "log_odds_chosen": 1.2048828601837158, + "log_odds_ratio": -0.439453125, + "logits/chosen": -0.9654296636581421, + "logits/rejected": -0.8451172113418579, + "logps/chosen": -0.736328125, + "logps/rejected": -1.6017577648162842, + "loss": 0.9318, + "nll_loss": 0.9462890625, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07366943359375, + "rewards/margins": 0.08660583198070526, + "rewards/rejected": -0.16021728515625, + "step": 3180 + }, + { + "epoch": 0.24207011686143573, + "grad_norm": 1.8365782277502412, + "learning_rate": 1.416428463593022e-06, + "log_odds_chosen": 1.343164086341858, + "log_odds_ratio": -0.4187988340854645, + "logits/chosen": -0.939648449420929, + "logits/rejected": -0.803906261920929, + "logps/chosen": -0.6578124761581421, + "logps/rejected": -1.591796875, + "loss": 0.9503, + "nll_loss": 0.8871093988418579, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06582031399011612, + "rewards/margins": 0.09349365532398224, + "rewards/rejected": -0.15922851860523224, + "step": 3190 + }, + { + "epoch": 0.24282895735316437, + "grad_norm": 1.8323563524467135, + "learning_rate": 1.414213562373095e-06, + "log_odds_chosen": 1.247460961341858, + "log_odds_ratio": -0.44145506620407104, + "logits/chosen": -0.9507812261581421, + "logits/rejected": -0.8531249761581421, + "logps/chosen": -0.6361328363418579, + "logps/rejected": -1.495507836341858, + "loss": 0.9085, + "nll_loss": 0.9072265625, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.063720703125, + "rewards/margins": 0.0858154296875, + "rewards/rejected": -0.14948730170726776, + "step": 3200 + }, + { + "epoch": 0.243587797844893, + "grad_norm": 1.9099153823357262, + "learning_rate": 1.4120090192364154e-06, + "log_odds_chosen": 1.0369141101837158, + "log_odds_ratio": -0.4970703125, + "logits/chosen": -0.969531238079071, + "logits/rejected": -0.8443359136581421, + "logps/chosen": -0.691601574420929, + "logps/rejected": -1.372656226158142, + "loss": 0.9456, + "nll_loss": 0.9193359613418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06915283203125, + "rewards/margins": 0.06818237155675888, + "rewards/rejected": -0.1373291015625, + "step": 3210 + }, + { + "epoch": 0.24434663833662165, + "grad_norm": 2.0375256450793087, + "learning_rate": 1.4098147537004828e-06, + "log_odds_chosen": 1.133935570716858, + "log_odds_ratio": -0.44086915254592896, + "logits/chosen": -0.9365234375, + "logits/rejected": -0.8529297113418579, + "logps/chosen": -0.646679699420929, + "logps/rejected": -1.3826172351837158, + "loss": 0.9461, + "nll_loss": 0.9072265625, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06463623046875, + "rewards/margins": 0.07357482612133026, + "rewards/rejected": -0.13820800185203552, + "step": 3220 + }, + { + "epoch": 0.2451054788283503, + "grad_norm": 1.9912908730465613, + "learning_rate": 1.4076306861555735e-06, + "log_odds_chosen": 1.2411620616912842, + "log_odds_ratio": -0.41997069120407104, + "logits/chosen": -0.9458984136581421, + "logits/rejected": -0.8486328125, + "logps/chosen": -0.67578125, + "logps/rejected": -1.5251953601837158, + "loss": 0.9424, + "nll_loss": 0.9007812738418579, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06752929836511612, + "rewards/margins": 0.08501434326171875, + "rewards/rejected": -0.15249022841453552, + "step": 3230 + }, + { + "epoch": 0.2458643193200789, + "grad_norm": 2.274586805113359, + "learning_rate": 1.405456737852613e-06, + "log_odds_chosen": 1.1833984851837158, + "log_odds_ratio": -0.4684081971645355, + "logits/chosen": -0.9150390625, + "logits/rejected": -0.835156261920929, + "logps/chosen": -0.70703125, + "logps/rejected": -1.558007836341858, + "loss": 0.9503, + "nll_loss": 0.878125011920929, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.0706787109375, + "rewards/margins": 0.08520660549402237, + "rewards/rejected": -0.15581054985523224, + "step": 3240 + }, + { + "epoch": 0.24662315981180755, + "grad_norm": 1.931992941128289, + "learning_rate": 1.4032928308912468e-06, + "log_odds_chosen": 1.2144043445587158, + "log_odds_ratio": -0.4669433534145355, + "logits/chosen": -0.9580078125, + "logits/rejected": -0.8726562261581421, + "logps/chosen": -0.6917968988418579, + "logps/rejected": -1.5554687976837158, + "loss": 0.9286, + "nll_loss": 0.8759765625, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.06925048679113388, + "rewards/margins": 0.08652496337890625, + "rewards/rejected": -0.15571288764476776, + "step": 3250 + }, + { + "epoch": 0.2473820003035362, + "grad_norm": 1.889574173524338, + "learning_rate": 1.4011388882081175e-06, + "log_odds_chosen": 1.2414062023162842, + "log_odds_ratio": -0.44091796875, + "logits/chosen": -0.9189453125, + "logits/rejected": -0.818164050579071, + "logps/chosen": -0.604687511920929, + "logps/rejected": -1.4462890625, + "loss": 0.9378, + "nll_loss": 0.8216797113418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06051025539636612, + "rewards/margins": 0.08405761420726776, + "rewards/rejected": -0.14462891221046448, + "step": 3260 + }, + { + "epoch": 0.24814084079526483, + "grad_norm": 1.9404559201119644, + "learning_rate": 1.3989948335653378e-06, + "log_odds_chosen": 1.404687523841858, + "log_odds_ratio": -0.427978515625, + "logits/chosen": -1.0126953125, + "logits/rejected": -0.861523449420929, + "logps/chosen": -0.62841796875, + "logps/rejected": -1.622460961341858, + "loss": 0.9268, + "nll_loss": 0.807910144329071, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06280517578125, + "rewards/margins": 0.09949950873851776, + "rewards/rejected": -0.1622314453125, + "step": 3270 + }, + { + "epoch": 0.24889968128699347, + "grad_norm": 1.7452378668385211, + "learning_rate": 1.3968605915391564e-06, + "log_odds_chosen": 1.380468726158142, + "log_odds_ratio": -0.4466796815395355, + "logits/chosen": -0.944531261920929, + "logits/rejected": -0.848437488079071, + "logps/chosen": -0.7064453363418579, + "logps/rejected": -1.7205078601837158, + "loss": 0.9524, + "nll_loss": 0.9287109375, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.07064209133386612, + "rewards/margins": 0.10134277492761612, + "rewards/rejected": -0.17197266221046448, + "step": 3280 + }, + { + "epoch": 0.2496585217787221, + "grad_norm": 1.9903896388016717, + "learning_rate": 1.3947360875088132e-06, + "log_odds_chosen": 1.1735351085662842, + "log_odds_ratio": -0.46064454317092896, + "logits/chosen": -0.9371093511581421, + "logits/rejected": -0.8333984613418579, + "logps/chosen": -0.70166015625, + "logps/rejected": -1.510156273841858, + "loss": 0.9344, + "nll_loss": 0.8818359375, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07014159858226776, + "rewards/margins": 0.08088989555835724, + "rewards/rejected": -0.15109863877296448, + "step": 3290 + }, + { + "epoch": 0.25041736227045075, + "grad_norm": 1.825929681479405, + "learning_rate": 1.3926212476455828e-06, + "log_odds_chosen": 1.066686987876892, + "log_odds_ratio": -0.496337890625, + "logits/chosen": -0.9189453125, + "logits/rejected": -0.8451172113418579, + "logps/chosen": -0.681640625, + "logps/rejected": -1.4054687023162842, + "loss": 0.9379, + "nll_loss": 0.9091796875, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06812743842601776, + "rewards/margins": 0.07235412299633026, + "rewards/rejected": -0.140625, + "step": 3300 + }, + { + "epoch": 0.2511762027621794, + "grad_norm": 2.1511451858131787, + "learning_rate": 1.3905159989019964e-06, + "log_odds_chosen": 1.366601586341858, + "log_odds_ratio": -0.42021483182907104, + "logits/chosen": -0.9537109136581421, + "logits/rejected": -0.8388671875, + "logps/chosen": -0.6988281011581421, + "logps/rejected": -1.6652343273162842, + "loss": 0.9458, + "nll_loss": 0.897265613079071, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06990966945886612, + "rewards/margins": 0.09661865234375, + "rewards/rejected": -0.16650390625, + "step": 3310 + }, + { + "epoch": 0.25193504325390803, + "grad_norm": 2.4080145026266675, + "learning_rate": 1.3884202690012465e-06, + "log_odds_chosen": 1.415136694908142, + "log_odds_ratio": -0.40556639432907104, + "logits/chosen": -0.946484386920929, + "logits/rejected": -0.8427734375, + "logps/chosen": -0.737011730670929, + "logps/rejected": -1.7296874523162842, + "loss": 0.9546, + "nll_loss": 0.9208984375, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.07371826469898224, + "rewards/margins": 0.099334716796875, + "rewards/rejected": -0.17304687201976776, + "step": 3320 + }, + { + "epoch": 0.25269388374563667, + "grad_norm": 2.136961368070725, + "learning_rate": 1.3863339864267636e-06, + "log_odds_chosen": 0.906054675579071, + "log_odds_ratio": -0.49775391817092896, + "logits/chosen": -0.9507812261581421, + "logits/rejected": -0.841601550579071, + "logps/chosen": -0.6709960699081421, + "logps/rejected": -1.2195312976837158, + "loss": 0.9513, + "nll_loss": 0.88671875, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06708984076976776, + "rewards/margins": 0.05487060546875, + "rewards/rejected": -0.12202148139476776, + "step": 3330 + }, + { + "epoch": 0.2534527242373653, + "grad_norm": 1.9884853306075043, + "learning_rate": 1.3842570804119655e-06, + "log_odds_chosen": 1.0947265625, + "log_odds_ratio": -0.44355469942092896, + "logits/chosen": -0.9673827886581421, + "logits/rejected": -0.832226574420929, + "logps/chosen": -0.6509765386581421, + "logps/rejected": -1.3839843273162842, + "loss": 0.9472, + "nll_loss": 0.8648437261581421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06513671576976776, + "rewards/margins": 0.07327880710363388, + "rewards/rejected": -0.13840332627296448, + "step": 3340 + }, + { + "epoch": 0.25421156472909395, + "grad_norm": 3.7716690402223643, + "learning_rate": 1.3821894809301763e-06, + "log_odds_chosen": 1.4836914539337158, + "log_odds_ratio": -0.3805175721645355, + "logits/chosen": -0.8980468511581421, + "logits/rejected": -0.7972656488418579, + "logps/chosen": -0.64111328125, + "logps/rejected": -1.6281249523162842, + "loss": 0.9458, + "nll_loss": 0.8804687261581421, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06407471001148224, + "rewards/margins": 0.09869384765625, + "rewards/rejected": -0.16264648735523224, + "step": 3350 + }, + { + "epoch": 0.2549704052208226, + "grad_norm": 1.9417352866678919, + "learning_rate": 1.3801311186847081e-06, + "log_odds_chosen": 1.279882788658142, + "log_odds_ratio": -0.42607420682907104, + "logits/chosen": -0.94921875, + "logits/rejected": -0.835156261920929, + "logps/chosen": -0.6883789300918579, + "logps/rejected": -1.540624976158142, + "loss": 0.9421, + "nll_loss": 0.9169921875, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06882324069738388, + "rewards/margins": 0.08518066257238388, + "rewards/rejected": -0.15407714247703552, + "step": 3360 + }, + { + "epoch": 0.25572924571255123, + "grad_norm": 1.9677525270182894, + "learning_rate": 1.378081925099109e-06, + "log_odds_chosen": 1.1848633289337158, + "log_odds_ratio": -0.41083985567092896, + "logits/chosen": -0.86328125, + "logits/rejected": -0.802734375, + "logps/chosen": -0.6630859375, + "logps/rejected": -1.443945288658142, + "loss": 0.9459, + "nll_loss": 0.881640613079071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06630859524011612, + "rewards/margins": 0.07810668647289276, + "rewards/rejected": -0.1444091796875, + "step": 3370 + }, + { + "epoch": 0.2564880862042799, + "grad_norm": 1.7058936097763349, + "learning_rate": 1.376041832307563e-06, + "log_odds_chosen": 1.240234375, + "log_odds_ratio": -0.4556640684604645, + "logits/chosen": -0.948437511920929, + "logits/rejected": -0.8101562261581421, + "logps/chosen": -0.694042980670929, + "logps/rejected": -1.537109375, + "loss": 0.9528, + "nll_loss": 0.9375, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06947021186351776, + "rewards/margins": 0.08425293117761612, + "rewards/rejected": -0.15361328423023224, + "step": 3380 + }, + { + "epoch": 0.2572469266960085, + "grad_norm": 2.2082044702359465, + "learning_rate": 1.3740107731454524e-06, + "log_odds_chosen": 1.232031226158142, + "log_odds_ratio": -0.4321533143520355, + "logits/chosen": -0.9410156011581421, + "logits/rejected": -0.8345702886581421, + "logps/chosen": -0.6962890625, + "logps/rejected": -1.5615234375, + "loss": 0.9379, + "nll_loss": 0.9957031011581421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06964111328125, + "rewards/margins": 0.08657226711511612, + "rewards/rejected": -0.15615233778953552, + "step": 3390 + }, + { + "epoch": 0.25800576718773716, + "grad_norm": 2.1820637460198142, + "learning_rate": 1.3719886811400705e-06, + "log_odds_chosen": 1.3191406726837158, + "log_odds_ratio": -0.4033203125, + "logits/chosen": -0.981640636920929, + "logits/rejected": -0.876953125, + "logps/chosen": -0.6343749761581421, + "logps/rejected": -1.5291016101837158, + "loss": 0.922, + "nll_loss": 0.8228515386581421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06345214694738388, + "rewards/margins": 0.0894775390625, + "rewards/rejected": -0.15297850966453552, + "step": 3400 + }, + { + "epoch": 0.2587646076794658, + "grad_norm": 1.8723787448291065, + "learning_rate": 1.3699754905014834e-06, + "log_odds_chosen": 1.3383300304412842, + "log_odds_ratio": -0.427001953125, + "logits/chosen": -0.9251953363418579, + "logits/rejected": -0.8294922113418579, + "logps/chosen": -0.6695312261581421, + "logps/rejected": -1.616796851158142, + "loss": 0.9542, + "nll_loss": 0.871874988079071, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06691894680261612, + "rewards/margins": 0.09471359103918076, + "rewards/rejected": -0.16169433295726776, + "step": 3410 + }, + { + "epoch": 0.25952344817119444, + "grad_norm": 1.909954286522631, + "learning_rate": 1.3679711361135388e-06, + "log_odds_chosen": 1.097021460533142, + "log_odds_ratio": -0.4974609315395355, + "logits/chosen": -0.9791015386581421, + "logits/rejected": -0.898632824420929, + "logps/chosen": -0.7193359136581421, + "logps/rejected": -1.520898461341858, + "loss": 0.9201, + "nll_loss": 0.8841797113418579, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07192382961511612, + "rewards/margins": 0.08006133884191513, + "rewards/rejected": -0.152099609375, + "step": 3420 + }, + { + "epoch": 0.2602822886629231, + "grad_norm": 1.8424699803048412, + "learning_rate": 1.3659755535250212e-06, + "log_odds_chosen": 1.4440429210662842, + "log_odds_ratio": -0.39545899629592896, + "logits/chosen": -0.941601574420929, + "logits/rejected": -0.8285156488418579, + "logps/chosen": -0.63427734375, + "logps/rejected": -1.6384766101837158, + "loss": 0.9418, + "nll_loss": 0.841015636920929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06340332329273224, + "rewards/margins": 0.10048218071460724, + "rewards/rejected": -0.16384276747703552, + "step": 3430 + }, + { + "epoch": 0.2610411291546517, + "grad_norm": 1.9198743792335182, + "learning_rate": 1.3639886789409469e-06, + "log_odds_chosen": 1.4453125, + "log_odds_ratio": -0.38801270723342896, + "logits/chosen": -0.9693359136581421, + "logits/rejected": -0.830859363079071, + "logps/chosen": -0.6615234613418579, + "logps/rejected": -1.659765601158142, + "loss": 0.9337, + "nll_loss": 0.9193359613418579, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06612548977136612, + "rewards/margins": 0.09989013522863388, + "rewards/rejected": -0.16608886420726776, + "step": 3440 + }, + { + "epoch": 0.2617999696463803, + "grad_norm": 1.8882359433159133, + "learning_rate": 1.3620104492139977e-06, + "log_odds_chosen": 1.552832007408142, + "log_odds_ratio": -0.38276368379592896, + "logits/chosen": -0.944531261920929, + "logits/rejected": -0.8271484375, + "logps/chosen": -0.650195300579071, + "logps/rejected": -1.726953148841858, + "loss": 0.9616, + "nll_loss": 0.879687488079071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06500244140625, + "rewards/margins": 0.10766907036304474, + "rewards/rejected": -0.17258301377296448, + "step": 3450 + }, + { + "epoch": 0.26255881013810894, + "grad_norm": 1.9301623087868962, + "learning_rate": 1.3600408018360918e-06, + "log_odds_chosen": 1.2575194835662842, + "log_odds_ratio": -0.4151855409145355, + "logits/chosen": -0.938671886920929, + "logits/rejected": -0.852343738079071, + "logps/chosen": -0.65234375, + "logps/rejected": -1.4910156726837158, + "loss": 0.9257, + "nll_loss": 0.842968761920929, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06523437798023224, + "rewards/margins": 0.08380126953125, + "rewards/rejected": -0.14899902045726776, + "step": 3460 + }, + { + "epoch": 0.2633176506298376, + "grad_norm": 1.9454646415269345, + "learning_rate": 1.3580796749300878e-06, + "log_odds_chosen": 1.2312500476837158, + "log_odds_ratio": -0.4684081971645355, + "logits/chosen": -0.9351562261581421, + "logits/rejected": -0.8656250238418579, + "logps/chosen": -0.7134765386581421, + "logps/rejected": -1.5994141101837158, + "loss": 0.9468, + "nll_loss": 0.8939453363418579, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07143554836511612, + "rewards/margins": 0.08856506645679474, + "rewards/rejected": -0.15996094048023224, + "step": 3470 + }, + { + "epoch": 0.2640764911215662, + "grad_norm": 1.8682090773735338, + "learning_rate": 1.3561270072416209e-06, + "log_odds_chosen": 1.100976586341858, + "log_odds_ratio": -0.4833984375, + "logits/chosen": -0.9535156488418579, + "logits/rejected": -0.8607422113418579, + "logps/chosen": -0.6826171875, + "logps/rejected": -1.4677734375, + "loss": 0.9192, + "nll_loss": 0.895703136920929, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06829833984375, + "rewards/margins": 0.07857055962085724, + "rewards/rejected": -0.14677734673023224, + "step": 3480 + }, + { + "epoch": 0.26483533161329487, + "grad_norm": 2.031528833234933, + "learning_rate": 1.3541827381310652e-06, + "log_odds_chosen": 1.175878882408142, + "log_odds_ratio": -0.46772462129592896, + "logits/chosen": -0.931445300579071, + "logits/rejected": -0.8638671636581421, + "logps/chosen": -0.707324206829071, + "logps/rejected": -1.5478515625, + "loss": 0.9382, + "nll_loss": 0.87890625, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.07070312649011612, + "rewards/margins": 0.08412780612707138, + "rewards/rejected": -0.15488281846046448, + "step": 3490 + }, + { + "epoch": 0.2655941721050235, + "grad_norm": 2.0922319319691103, + "learning_rate": 1.3522468075656264e-06, + "log_odds_chosen": 1.394921898841858, + "log_odds_ratio": -0.44111329317092896, + "logits/chosen": -0.9380859136581421, + "logits/rejected": -0.8345702886581421, + "logps/chosen": -0.717480480670929, + "logps/rejected": -1.7511718273162842, + "loss": 0.9151, + "nll_loss": 0.8675781488418579, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07181396335363388, + "rewards/margins": 0.10313721001148224, + "rewards/rejected": -0.17502442002296448, + "step": 3500 + }, + { + "epoch": 0.26635301259675215, + "grad_norm": 2.038393716254202, + "learning_rate": 1.3503191561115553e-06, + "log_odds_chosen": 1.2841796875, + "log_odds_ratio": -0.4270996153354645, + "logits/chosen": -0.954882800579071, + "logits/rejected": -0.860546886920929, + "logps/chosen": -0.656054675579071, + "logps/rejected": -1.5242187976837158, + "loss": 0.9097, + "nll_loss": 0.872265636920929, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06560058891773224, + "rewards/margins": 0.08687744289636612, + "rewards/rejected": -0.15261229872703552, + "step": 3510 + }, + { + "epoch": 0.2671118530884808, + "grad_norm": 2.113088406087, + "learning_rate": 1.348399724926484e-06, + "log_odds_chosen": 1.34375, + "log_odds_ratio": -0.3807128965854645, + "logits/chosen": -0.9605468511581421, + "logits/rejected": -0.8394531011581421, + "logps/chosen": -0.6573241949081421, + "logps/rejected": -1.5763671398162842, + "loss": 0.938, + "nll_loss": 0.9312499761581421, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06573486328125, + "rewards/margins": 0.09195556491613388, + "rewards/rejected": -0.15773925185203552, + "step": 3520 + }, + { + "epoch": 0.26787069358020943, + "grad_norm": 2.206269408386097, + "learning_rate": 1.346488455751882e-06, + "log_odds_chosen": 1.100976586341858, + "log_odds_ratio": -0.45136719942092896, + "logits/chosen": -0.981249988079071, + "logits/rejected": -0.8841797113418579, + "logps/chosen": -0.708300769329071, + "logps/rejected": -1.443359375, + "loss": 0.9309, + "nll_loss": 0.9097656011581421, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.07087402045726776, + "rewards/margins": 0.07344970852136612, + "rewards/rejected": -0.14433594048023224, + "step": 3530 + }, + { + "epoch": 0.26862953407193807, + "grad_norm": 1.9530518034425253, + "learning_rate": 1.3445852909056286e-06, + "log_odds_chosen": 1.2160155773162842, + "log_odds_ratio": -0.40668946504592896, + "logits/chosen": -0.971875011920929, + "logits/rejected": -0.860546886920929, + "logps/chosen": -0.6458984613418579, + "logps/rejected": -1.461328148841858, + "loss": 0.9214, + "nll_loss": 0.887890636920929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06459961086511612, + "rewards/margins": 0.08159179985523224, + "rewards/rejected": -0.1461181640625, + "step": 3540 + }, + { + "epoch": 0.2693883745636667, + "grad_norm": 1.9384344977682093, + "learning_rate": 1.3426901732747024e-06, + "log_odds_chosen": 1.2693359851837158, + "log_odds_ratio": -0.4195312559604645, + "logits/chosen": -0.9878906011581421, + "logits/rejected": -0.8871093988418579, + "logps/chosen": -0.684863269329071, + "logps/rejected": -1.568945288658142, + "loss": 0.9331, + "nll_loss": 0.9654296636581421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06842041015625, + "rewards/margins": 0.0883331298828125, + "rewards/rejected": -0.15693359076976776, + "step": 3550 + }, + { + "epoch": 0.27014721505539535, + "grad_norm": 2.1127083794668517, + "learning_rate": 1.3408030463079818e-06, + "log_odds_chosen": 1.28515625, + "log_odds_ratio": -0.4278808534145355, + "logits/chosen": -0.9398437738418579, + "logits/rejected": -0.8525390625, + "logps/chosen": -0.697265625, + "logps/rejected": -1.605078101158142, + "loss": 0.9248, + "nll_loss": 0.8998047113418579, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06967773288488388, + "rewards/margins": 0.09077148139476776, + "rewards/rejected": -0.16044922173023224, + "step": 3560 + }, + { + "epoch": 0.270906055547124, + "grad_norm": 2.0473745290079193, + "learning_rate": 1.3389238540091568e-06, + "log_odds_chosen": 1.4441406726837158, + "log_odds_ratio": -0.4049316346645355, + "logits/chosen": -1.032812476158142, + "logits/rejected": -0.912109375, + "logps/chosen": -0.709179699420929, + "logps/rejected": -1.760156273841858, + "loss": 0.9217, + "nll_loss": 0.885546863079071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.07088623195886612, + "rewards/margins": 0.10515137016773224, + "rewards/rejected": -0.176025390625, + "step": 3570 + }, + { + "epoch": 0.27166489603885263, + "grad_norm": 2.0116456378081438, + "learning_rate": 1.337052540929751e-06, + "log_odds_chosen": 1.521875023841858, + "log_odds_ratio": -0.3868164122104645, + "logits/chosen": -0.9634765386581421, + "logits/rejected": -0.8515625, + "logps/chosen": -0.721484363079071, + "logps/rejected": -1.8093750476837158, + "loss": 0.914, + "nll_loss": 0.939257800579071, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0721435546875, + "rewards/margins": 0.10887451469898224, + "rewards/rejected": -0.18093261122703552, + "step": 3580 + }, + { + "epoch": 0.2724237365305813, + "grad_norm": 2.0250387953551834, + "learning_rate": 1.33518905216225e-06, + "log_odds_chosen": 1.1780273914337158, + "log_odds_ratio": -0.4666503965854645, + "logits/chosen": -0.9242187738418579, + "logits/rejected": -0.8359375, + "logps/chosen": -0.71484375, + "logps/rejected": -1.535546898841858, + "loss": 0.9198, + "nll_loss": 0.9498046636581421, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07145996391773224, + "rewards/margins": 0.0820465087890625, + "rewards/rejected": -0.15351562201976776, + "step": 3590 + }, + { + "epoch": 0.2731825770223099, + "grad_norm": 1.8833860642550362, + "learning_rate": 1.3333333333333332e-06, + "log_odds_chosen": 1.3043944835662842, + "log_odds_ratio": -0.4422851502895355, + "logits/chosen": -0.8955078125, + "logits/rejected": -0.804492175579071, + "logps/chosen": -0.66796875, + "logps/rejected": -1.592382788658142, + "loss": 0.9095, + "nll_loss": 0.8248046636581421, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06680908054113388, + "rewards/margins": 0.09229431301355362, + "rewards/rejected": -0.15915527939796448, + "step": 3600 + }, + { + "epoch": 0.27394141751403855, + "grad_norm": 1.860998223645059, + "learning_rate": 1.3314853305972122e-06, + "log_odds_chosen": 1.147851586341858, + "log_odds_ratio": -0.4798339903354645, + "logits/chosen": -1.007226586341858, + "logits/rejected": -0.916015625, + "logps/chosen": -0.6913086175918579, + "logps/rejected": -1.504296898841858, + "loss": 0.9262, + "nll_loss": 0.9146484136581421, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06912841647863388, + "rewards/margins": 0.08133240044116974, + "rewards/rejected": -0.15043945610523224, + "step": 3610 + }, + { + "epoch": 0.2747002580057672, + "grad_norm": 3.411752689796777, + "learning_rate": 1.3296449906290671e-06, + "log_odds_chosen": 1.2785155773162842, + "log_odds_ratio": -0.427490234375, + "logits/chosen": -0.944531261920929, + "logits/rejected": -0.8177734613418579, + "logps/chosen": -0.6968749761581421, + "logps/rejected": -1.5828125476837158, + "loss": 0.9241, + "nll_loss": 0.933789074420929, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06971435248851776, + "rewards/margins": 0.08861084282398224, + "rewards/rejected": -0.15817871689796448, + "step": 3620 + }, + { + "epoch": 0.27545909849749584, + "grad_norm": 1.7878113783437786, + "learning_rate": 1.3278122606185844e-06, + "log_odds_chosen": 1.1787598133087158, + "log_odds_ratio": -0.4697265625, + "logits/chosen": -0.930859386920929, + "logits/rejected": -0.851367175579071, + "logps/chosen": -0.6568359136581421, + "logps/rejected": -1.4609375, + "loss": 0.9242, + "nll_loss": 0.8294922113418579, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.065673828125, + "rewards/margins": 0.080413818359375, + "rewards/rejected": -0.1461181640625, + "step": 3630 + }, + { + "epoch": 0.2762179389892245, + "grad_norm": 2.04196840241614, + "learning_rate": 1.3259870882635918e-06, + "log_odds_chosen": 1.085351586341858, + "log_odds_ratio": -0.43281251192092896, + "logits/chosen": -0.958203136920929, + "logits/rejected": -0.855664074420929, + "logps/chosen": -0.705859363079071, + "logps/rejected": -1.427343726158142, + "loss": 0.9117, + "nll_loss": 0.894726574420929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.07058105617761612, + "rewards/margins": 0.07220458984375, + "rewards/rejected": -0.14274902641773224, + "step": 3640 + }, + { + "epoch": 0.2769767794809531, + "grad_norm": 2.0541773707451956, + "learning_rate": 1.3241694217637886e-06, + "log_odds_chosen": 1.3690917491912842, + "log_odds_ratio": -0.41411131620407104, + "logits/chosen": -0.9505859613418579, + "logits/rejected": -0.845898449420929, + "logps/chosen": -0.6644531488418579, + "logps/rejected": -1.6238281726837158, + "loss": 0.8978, + "nll_loss": 0.8138672113418579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06641845405101776, + "rewards/margins": 0.09598693996667862, + "rewards/rejected": -0.16249999403953552, + "step": 3650 + }, + { + "epoch": 0.27773561997268176, + "grad_norm": 1.85220670548097, + "learning_rate": 1.3223592098145723e-06, + "log_odds_chosen": 1.144677758216858, + "log_odds_ratio": -0.46337890625, + "logits/chosen": -0.990039050579071, + "logits/rejected": -0.8544921875, + "logps/chosen": -0.693359375, + "logps/rejected": -1.516210913658142, + "loss": 0.9182, + "nll_loss": 0.889453113079071, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.0693359375, + "rewards/margins": 0.08226318657398224, + "rewards/rejected": -0.15158692002296448, + "step": 3660 + }, + { + "epoch": 0.2784944604644104, + "grad_norm": 2.0232170372823512, + "learning_rate": 1.3205564016009555e-06, + "log_odds_chosen": 1.1967284679412842, + "log_odds_ratio": -0.44624024629592896, + "logits/chosen": -0.9720703363418579, + "logits/rejected": -0.866015613079071, + "logps/chosen": -0.638671875, + "logps/rejected": -1.4365234375, + "loss": 0.9047, + "nll_loss": 0.837695300579071, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0638427734375, + "rewards/margins": 0.07980956882238388, + "rewards/rejected": -0.14372558891773224, + "step": 3670 + }, + { + "epoch": 0.27925330095613904, + "grad_norm": 2.0496707214558834, + "learning_rate": 1.318760946791574e-06, + "log_odds_chosen": 1.037841796875, + "log_odds_ratio": -0.4786132872104645, + "logits/chosen": -0.9595702886581421, + "logits/rejected": -0.8451172113418579, + "logps/chosen": -0.758007824420929, + "logps/rejected": -1.464453101158142, + "loss": 0.9316, + "nll_loss": 0.9769531488418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07581786811351776, + "rewards/margins": 0.07055969536304474, + "rewards/rejected": -0.1463623046875, + "step": 3680 + }, + { + "epoch": 0.2800121414478677, + "grad_norm": 1.9833909441957962, + "learning_rate": 1.316972795532786e-06, + "log_odds_chosen": 1.102392554283142, + "log_odds_ratio": -0.44877928495407104, + "logits/chosen": -0.970898449420929, + "logits/rejected": -0.8890625238418579, + "logps/chosen": -0.6924804449081421, + "logps/rejected": -1.426367163658142, + "loss": 0.9288, + "nll_loss": 0.881640613079071, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06929931789636612, + "rewards/margins": 0.07329712063074112, + "rewards/rejected": -0.14252929389476776, + "step": 3690 + }, + { + "epoch": 0.2807709819395963, + "grad_norm": 1.8332235239722916, + "learning_rate": 1.3151918984428582e-06, + "log_odds_chosen": 1.079687476158142, + "log_odds_ratio": -0.474365234375, + "logits/chosen": -0.947460949420929, + "logits/rejected": -0.8470703363418579, + "logps/chosen": -0.6900390386581421, + "logps/rejected": -1.4333984851837158, + "loss": 0.9166, + "nll_loss": 0.875781238079071, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06898193061351776, + "rewards/margins": 0.07448425143957138, + "rewards/rejected": -0.14338378608226776, + "step": 3700 + }, + { + "epoch": 0.28152982243132496, + "grad_norm": 2.1768807174108207, + "learning_rate": 1.313418206606237e-06, + "log_odds_chosen": 1.1287109851837158, + "log_odds_ratio": -0.4718261659145355, + "logits/chosen": -0.9134765863418579, + "logits/rejected": -0.8324218988418579, + "logps/chosen": -0.703320324420929, + "logps/rejected": -1.4970703125, + "loss": 0.9236, + "nll_loss": 0.8587890863418579, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07032470405101776, + "rewards/margins": 0.07926025241613388, + "rewards/rejected": -0.14970703423023224, + "step": 3710 + }, + { + "epoch": 0.2822886629230536, + "grad_norm": 1.69205488461686, + "learning_rate": 1.3116516715679057e-06, + "log_odds_chosen": 1.261816382408142, + "log_odds_ratio": -0.42744141817092896, + "logits/chosen": -0.9498046636581421, + "logits/rejected": -0.8570312261581421, + "logps/chosen": -0.66796875, + "logps/rejected": -1.5369141101837158, + "loss": 0.8861, + "nll_loss": 0.8794921636581421, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06682129204273224, + "rewards/margins": 0.086822509765625, + "rewards/rejected": -0.15363769233226776, + "step": 3720 + }, + { + "epoch": 0.2830475034147822, + "grad_norm": 2.172458859919438, + "learning_rate": 1.3098922453278258e-06, + "log_odds_chosen": 1.377343773841858, + "log_odds_ratio": -0.38911134004592896, + "logits/chosen": -0.9644531011581421, + "logits/rejected": -0.8539062738418579, + "logps/chosen": -0.676074206829071, + "logps/rejected": -1.6179687976837158, + "loss": 0.954, + "nll_loss": 0.8990234136581421, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.06757812201976776, + "rewards/margins": 0.09407959133386612, + "rewards/rejected": -0.16171875596046448, + "step": 3730 + }, + { + "epoch": 0.2838063439065108, + "grad_norm": 1.993112848217689, + "learning_rate": 1.3081398803354573e-06, + "log_odds_chosen": 1.075585961341858, + "log_odds_ratio": -0.4784179627895355, + "logits/chosen": -0.939257800579071, + "logits/rejected": -0.861328125, + "logps/chosen": -0.7232421636581421, + "logps/rejected": -1.469140648841858, + "loss": 0.914, + "nll_loss": 0.815234363079071, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07233886420726776, + "rewards/margins": 0.074615478515625, + "rewards/rejected": -0.14699706435203552, + "step": 3740 + }, + { + "epoch": 0.28456518439823947, + "grad_norm": 2.044150243486001, + "learning_rate": 1.3063945294843617e-06, + "log_odds_chosen": 1.310449242591858, + "log_odds_ratio": -0.39531248807907104, + "logits/chosen": -0.927929699420929, + "logits/rejected": -0.8052734136581421, + "logps/chosen": -0.637402355670929, + "logps/rejected": -1.5, + "loss": 0.9004, + "nll_loss": 0.8472656011581421, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06375732272863388, + "rewards/margins": 0.08640136569738388, + "rewards/rejected": -0.15000000596046448, + "step": 3750 + }, + { + "epoch": 0.2853240248899681, + "grad_norm": 2.1374911680271174, + "learning_rate": 1.3046561461068843e-06, + "log_odds_chosen": 1.138281226158142, + "log_odds_ratio": -0.43183594942092896, + "logits/chosen": -0.947070300579071, + "logits/rejected": -0.845507800579071, + "logps/chosen": -0.651171863079071, + "logps/rejected": -1.3779296875, + "loss": 0.907, + "nll_loss": 0.8470703363418579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06514892727136612, + "rewards/margins": 0.07267455756664276, + "rewards/rejected": -0.13779297471046448, + "step": 3760 + }, + { + "epoch": 0.28608286538169675, + "grad_norm": 2.1722206297390354, + "learning_rate": 1.3029246839689124e-06, + "log_odds_chosen": 1.3113524913787842, + "log_odds_ratio": -0.43989259004592896, + "logits/chosen": -0.980664074420929, + "logits/rejected": -0.8580077886581421, + "logps/chosen": -0.712695300579071, + "logps/rejected": -1.656835913658142, + "loss": 0.8966, + "nll_loss": 0.8304687738418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07132568210363388, + "rewards/margins": 0.09425659477710724, + "rewards/rejected": -0.16567382216453552, + "step": 3770 + }, + { + "epoch": 0.2868417058734254, + "grad_norm": 2.1742138914896745, + "learning_rate": 1.3012000972647109e-06, + "log_odds_chosen": 1.387109398841858, + "log_odds_ratio": -0.42460936307907104, + "logits/chosen": -0.951367199420929, + "logits/rejected": -0.8472656011581421, + "logps/chosen": -0.6595703363418579, + "logps/rejected": -1.5859375, + "loss": 0.8949, + "nll_loss": 0.8529297113418579, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06594238430261612, + "rewards/margins": 0.09256591647863388, + "rewards/rejected": -0.15861816704273224, + "step": 3780 + }, + { + "epoch": 0.28760054636515403, + "grad_norm": 2.155127876083863, + "learning_rate": 1.299482340611832e-06, + "log_odds_chosen": 1.07763671875, + "log_odds_ratio": -0.45654296875, + "logits/chosen": -0.9107421636581421, + "logits/rejected": -0.841796875, + "logps/chosen": -0.62109375, + "logps/rejected": -1.2839844226837158, + "loss": 0.9269, + "nll_loss": 0.898632824420929, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06208496168255806, + "rewards/margins": 0.0662841796875, + "rewards/rejected": -0.12841796875, + "step": 3790 + }, + { + "epoch": 0.28835938685688267, + "grad_norm": 1.9850754100896, + "learning_rate": 1.2977713690461003e-06, + "log_odds_chosen": 0.9205077886581421, + "log_odds_ratio": -0.5121093988418579, + "logits/chosen": -0.9189453125, + "logits/rejected": -0.8374999761581421, + "logps/chosen": -0.702343761920929, + "logps/rejected": -1.354101538658142, + "loss": 0.9102, + "nll_loss": 0.8666015863418579, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.07027588039636612, + "rewards/margins": 0.06508179008960724, + "rewards/rejected": -0.13547363877296448, + "step": 3800 + }, + { + "epoch": 0.2891182273486113, + "grad_norm": 3.6873578144144328, + "learning_rate": 1.296067138016669e-06, + "log_odds_chosen": 1.4038574695587158, + "log_odds_ratio": -0.42290037870407104, + "logits/chosen": -0.9017578363418579, + "logits/rejected": -0.8236328363418579, + "logps/chosen": -0.65576171875, + "logps/rejected": -1.642968773841858, + "loss": 0.9158, + "nll_loss": 0.9505859613418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06558837741613388, + "rewards/margins": 0.0986480712890625, + "rewards/rejected": -0.16423340141773224, + "step": 3810 + }, + { + "epoch": 0.28987706784033995, + "grad_norm": 1.7797642030913543, + "learning_rate": 1.294369603381147e-06, + "log_odds_chosen": 1.154687523841858, + "log_odds_ratio": -0.4600585997104645, + "logits/chosen": -0.940234363079071, + "logits/rejected": -0.8539062738418579, + "logps/chosen": -0.6429687738418579, + "logps/rejected": -1.458398461341858, + "loss": 0.9202, + "nll_loss": 0.8675781488418579, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06423339992761612, + "rewards/margins": 0.08149413764476776, + "rewards/rejected": -0.14577636122703552, + "step": 3820 + }, + { + "epoch": 0.2906359083320686, + "grad_norm": 2.069119773208419, + "learning_rate": 1.2926787214007981e-06, + "log_odds_chosen": 1.3681640625, + "log_odds_ratio": -0.40424805879592896, + "logits/chosen": -0.929492175579071, + "logits/rejected": -0.828906238079071, + "logps/chosen": -0.6714843511581421, + "logps/rejected": -1.6003906726837158, + "loss": 0.9061, + "nll_loss": 0.9058593511581421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06715087592601776, + "rewards/margins": 0.092864990234375, + "rewards/rejected": -0.16010741889476776, + "step": 3830 + }, + { + "epoch": 0.29139474882379723, + "grad_norm": 2.1430312635959963, + "learning_rate": 1.2909944487358056e-06, + "log_odds_chosen": 1.35888671875, + "log_odds_ratio": -0.42060548067092896, + "logits/chosen": -0.9111328125, + "logits/rejected": -0.826367199420929, + "logps/chosen": -0.666699230670929, + "logps/rejected": -1.606835961341858, + "loss": 0.9176, + "nll_loss": 0.8921874761581421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06668701022863388, + "rewards/margins": 0.09390868991613388, + "rewards/rejected": -0.16062012314796448, + "step": 3840 + }, + { + "epoch": 0.2921535893155259, + "grad_norm": 1.8334780666623056, + "learning_rate": 1.2893167424406084e-06, + "log_odds_chosen": 1.49951171875, + "log_odds_ratio": -0.3844238221645355, + "logits/chosen": -0.9283202886581421, + "logits/rejected": -0.83203125, + "logps/chosen": -0.6283203363418579, + "logps/rejected": -1.661523461341858, + "loss": 0.9123, + "nll_loss": 0.8617187738418579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06282959133386612, + "rewards/margins": 0.10329589992761612, + "rewards/rejected": -0.1661376953125, + "step": 3850 + }, + { + "epoch": 0.2929124298072545, + "grad_norm": 3.226465128998707, + "learning_rate": 1.2876455599593008e-06, + "log_odds_chosen": 1.295019507408142, + "log_odds_ratio": -0.4169921875, + "logits/chosen": -0.9349609613418579, + "logits/rejected": -0.8218749761581421, + "logps/chosen": -0.687695324420929, + "logps/rejected": -1.5730469226837158, + "loss": 0.9304, + "nll_loss": 0.884960949420929, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06887207180261612, + "rewards/margins": 0.08841247856616974, + "rewards/rejected": -0.15725097060203552, + "step": 3860 + }, + { + "epoch": 0.29367127029898316, + "grad_norm": 1.8748341791580312, + "learning_rate": 1.285980859121099e-06, + "log_odds_chosen": 1.3193359375, + "log_odds_ratio": -0.3985839784145355, + "logits/chosen": -0.947460949420929, + "logits/rejected": -0.849414050579071, + "logps/chosen": -0.6934570074081421, + "logps/rejected": -1.58984375, + "loss": 0.8835, + "nll_loss": 0.816210925579071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06939697265625, + "rewards/margins": 0.089569091796875, + "rewards/rejected": -0.15903320908546448, + "step": 3870 + }, + { + "epoch": 0.2944301107907118, + "grad_norm": 2.0849897268039674, + "learning_rate": 1.2843225981358712e-06, + "log_odds_chosen": 1.206884741783142, + "log_odds_ratio": -0.4189453125, + "logits/chosen": -0.894726574420929, + "logits/rejected": -0.818359375, + "logps/chosen": -0.6875, + "logps/rejected": -1.5177733898162842, + "loss": 0.8965, + "nll_loss": 0.895703136920929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06877441704273224, + "rewards/margins": 0.08306427299976349, + "rewards/rejected": -0.15168456733226776, + "step": 3880 + }, + { + "epoch": 0.29518895128244044, + "grad_norm": 2.096660042103269, + "learning_rate": 1.2826707355897317e-06, + "log_odds_chosen": 1.1281249523162842, + "log_odds_ratio": -0.4625488221645355, + "logits/chosen": -0.9380859136581421, + "logits/rejected": -0.8873046636581421, + "logps/chosen": -0.712890625, + "logps/rejected": -1.5154297351837158, + "loss": 0.9078, + "nll_loss": 0.9111328125, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0712890625, + "rewards/margins": 0.08025512844324112, + "rewards/rejected": -0.15156249701976776, + "step": 3890 + }, + { + "epoch": 0.2959477917741691, + "grad_norm": 2.037013139295946, + "learning_rate": 1.281025230440697e-06, + "log_odds_chosen": 1.2263672351837158, + "log_odds_ratio": -0.4684081971645355, + "logits/chosen": -0.949023425579071, + "logits/rejected": -0.8646484613418579, + "logps/chosen": -0.738964855670929, + "logps/rejected": -1.6062500476837158, + "loss": 0.9104, + "nll_loss": 0.9224609136581421, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07393798977136612, + "rewards/margins": 0.086822509765625, + "rewards/rejected": -0.16074219346046448, + "step": 3900 + }, + { + "epoch": 0.2967066322658977, + "grad_norm": 2.027263988966167, + "learning_rate": 1.2793860420144025e-06, + "log_odds_chosen": 1.141992211341858, + "log_odds_ratio": -0.46015626192092896, + "logits/chosen": -0.947070300579071, + "logits/rejected": -0.85546875, + "logps/chosen": -0.693359375, + "logps/rejected": -1.4386718273162842, + "loss": 0.9024, + "nll_loss": 0.832226574420929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06937255710363388, + "rewards/margins": 0.07454834133386612, + "rewards/rejected": -0.14394530653953552, + "step": 3910 + }, + { + "epoch": 0.29746547275762636, + "grad_norm": 2.401330558412098, + "learning_rate": 1.2777531299998798e-06, + "log_odds_chosen": 1.37109375, + "log_odds_ratio": -0.405517578125, + "logits/chosen": -1.0207030773162842, + "logits/rejected": -0.8681640625, + "logps/chosen": -0.656542956829071, + "logps/rejected": -1.558203101158142, + "loss": 0.9375, + "nll_loss": 0.9273437261581421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.065673828125, + "rewards/margins": 0.09022827446460724, + "rewards/rejected": -0.15598145127296448, + "step": 3920 + }, + { + "epoch": 0.298224313249355, + "grad_norm": 2.0230385894292295, + "learning_rate": 1.2761264544453928e-06, + "log_odds_chosen": 1.116113305091858, + "log_odds_ratio": -0.4505371153354645, + "logits/chosen": -0.9798828363418579, + "logits/rejected": -0.9058593511581421, + "logps/chosen": -0.734570324420929, + "logps/rejected": -1.5070312023162842, + "loss": 0.9004, + "nll_loss": 0.866992175579071, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.07340087741613388, + "rewards/margins": 0.07725830376148224, + "rewards/rejected": -0.15078124403953552, + "step": 3930 + }, + { + "epoch": 0.29898315374108364, + "grad_norm": 2.4045696372396175, + "learning_rate": 1.2745059757543324e-06, + "log_odds_chosen": 1.170654296875, + "log_odds_ratio": -0.45654296875, + "logits/chosen": -0.9185546636581421, + "logits/rejected": -0.806640625, + "logps/chosen": -0.7037109136581421, + "logps/rejected": -1.4855468273162842, + "loss": 0.8907, + "nll_loss": 0.8763672113418579, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.07038573920726776, + "rewards/margins": 0.07807006686925888, + "rewards/rejected": -0.1485595703125, + "step": 3940 + }, + { + "epoch": 0.2997419942328123, + "grad_norm": 2.38418116530518, + "learning_rate": 1.272891654681168e-06, + "log_odds_chosen": 1.058984398841858, + "log_odds_ratio": -0.475830078125, + "logits/chosen": -0.8978515863418579, + "logits/rejected": -0.828320324420929, + "logps/chosen": -0.6246093511581421, + "logps/rejected": -1.282617211341858, + "loss": 0.9058, + "nll_loss": 0.878125011920929, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06243896484375, + "rewards/margins": 0.06582947075366974, + "rewards/rejected": -0.12822265923023224, + "step": 3950 + }, + { + "epoch": 0.3005008347245409, + "grad_norm": 2.2719917645534053, + "learning_rate": 1.2712834523274563e-06, + "log_odds_chosen": 1.1682617664337158, + "log_odds_ratio": -0.43828123807907104, + "logits/chosen": -0.954296886920929, + "logits/rejected": -0.8714843988418579, + "logps/chosen": -0.62060546875, + "logps/rejected": -1.387109398841858, + "loss": 0.9131, + "nll_loss": 0.8638671636581421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06207275390625, + "rewards/margins": 0.07665405422449112, + "rewards/rejected": -0.138671875, + "step": 3960 + }, + { + "epoch": 0.30125967521626956, + "grad_norm": 2.6488593849334277, + "learning_rate": 1.2696813301379032e-06, + "log_odds_chosen": 1.3332030773162842, + "log_odds_ratio": -0.4345703125, + "logits/chosen": -0.9886718988418579, + "logits/rejected": -0.884960949420929, + "logps/chosen": -0.663867175579071, + "logps/rejected": -1.55859375, + "loss": 0.9141, + "nll_loss": 0.8414062261581421, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06636963039636612, + "rewards/margins": 0.08956298977136612, + "rewards/rejected": -0.15590819716453552, + "step": 3970 + }, + { + "epoch": 0.3020185157079982, + "grad_norm": 2.10235152985044, + "learning_rate": 1.2680852498964829e-06, + "log_odds_chosen": 1.461523413658142, + "log_odds_ratio": -0.3689941465854645, + "logits/chosen": -0.960156261920929, + "logits/rejected": -0.847460925579071, + "logps/chosen": -0.628710925579071, + "logps/rejected": -1.6316406726837158, + "loss": 0.9188, + "nll_loss": 0.794921875, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06287841498851776, + "rewards/margins": 0.10035400092601776, + "rewards/rejected": -0.16318359971046448, + "step": 3980 + }, + { + "epoch": 0.30277735619972684, + "grad_norm": 2.270931300000564, + "learning_rate": 1.266495173722607e-06, + "log_odds_chosen": 1.0371582508087158, + "log_odds_ratio": -0.4625000059604645, + "logits/chosen": -0.907031238079071, + "logits/rejected": -0.8451172113418579, + "logps/chosen": -0.685351550579071, + "logps/rejected": -1.3771483898162842, + "loss": 0.9084, + "nll_loss": 0.817187488079071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06854248046875, + "rewards/margins": 0.06908264011144638, + "rewards/rejected": -0.13762207329273224, + "step": 3990 + }, + { + "epoch": 0.30353619669145543, + "grad_norm": 2.1861920101261725, + "learning_rate": 1.2649110640673517e-06, + "log_odds_chosen": 1.5475585460662842, + "log_odds_ratio": -0.3888183534145355, + "logits/chosen": -0.92578125, + "logits/rejected": -0.838085949420929, + "logps/chosen": -0.691113293170929, + "logps/rejected": -1.7919921875, + "loss": 0.9114, + "nll_loss": 0.8662109375, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06910400092601776, + "rewards/margins": 0.11008910834789276, + "rewards/rejected": -0.17917481064796448, + "step": 4000 + }, + { + "epoch": 0.30429503718318407, + "grad_norm": 1.8779121675796782, + "learning_rate": 1.2633328837097308e-06, + "log_odds_chosen": 1.2890625, + "log_odds_ratio": -0.422119140625, + "logits/chosen": -0.9326171875, + "logits/rejected": -0.841015636920929, + "logps/chosen": -0.65625, + "logps/rejected": -1.5244140625, + "loss": 0.903, + "nll_loss": 0.890820324420929, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06556396186351776, + "rewards/margins": 0.08682861179113388, + "rewards/rejected": -0.1524658203125, + "step": 4010 + }, + { + "epoch": 0.3050538776749127, + "grad_norm": 2.01813428939538, + "learning_rate": 1.2617605957530233e-06, + "log_odds_chosen": 1.365332007408142, + "log_odds_ratio": -0.3958496153354645, + "logits/chosen": -0.9375, + "logits/rejected": -0.833984375, + "logps/chosen": -0.6661132574081421, + "logps/rejected": -1.6046874523162842, + "loss": 0.9001, + "nll_loss": 0.8521484136581421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06668701022863388, + "rewards/margins": 0.09389648586511612, + "rewards/rejected": -0.16057129204273224, + "step": 4020 + }, + { + "epoch": 0.30581271816664135, + "grad_norm": 2.4756880333260267, + "learning_rate": 1.2601941636211516e-06, + "log_odds_chosen": 1.533593773841858, + "log_odds_ratio": -0.35151368379592896, + "logits/chosen": -0.97265625, + "logits/rejected": -0.8453124761581421, + "logps/chosen": -0.6441406011581421, + "logps/rejected": -1.714453101158142, + "loss": 0.8684, + "nll_loss": 0.843945324420929, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06441650539636612, + "rewards/margins": 0.10716553032398224, + "rewards/rejected": -0.17145995795726776, + "step": 4030 + }, + { + "epoch": 0.30657155865837, + "grad_norm": 2.1960269276331688, + "learning_rate": 1.2586335510551052e-06, + "log_odds_chosen": 1.3595702648162842, + "log_odds_ratio": -0.40283203125, + "logits/chosen": -0.9234374761581421, + "logits/rejected": -0.826367199420929, + "logps/chosen": -0.689746081829071, + "logps/rejected": -1.6552734375, + "loss": 0.9035, + "nll_loss": 0.8949218988418579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06898193061351776, + "rewards/margins": 0.09656982123851776, + "rewards/rejected": -0.16552734375, + "step": 4040 + }, + { + "epoch": 0.30733039915009863, + "grad_norm": 4.778497260855035, + "learning_rate": 1.2570787221094177e-06, + "log_odds_chosen": 1.20697021484375, + "log_odds_ratio": -0.40961915254592896, + "logits/chosen": -0.929882824420929, + "logits/rejected": -0.8060547113418579, + "logps/chosen": -0.642871081829071, + "logps/rejected": -1.4294922351837158, + "loss": 0.9063, + "nll_loss": 0.8509765863418579, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06430663913488388, + "rewards/margins": 0.07865753024816513, + "rewards/rejected": -0.14291992783546448, + "step": 4050 + }, + { + "epoch": 0.3080892396418273, + "grad_norm": 2.007858230707382, + "learning_rate": 1.255529641148689e-06, + "log_odds_chosen": 1.3185546398162842, + "log_odds_ratio": -0.406005859375, + "logits/chosen": -0.9306640625, + "logits/rejected": -0.8427734375, + "logps/chosen": -0.6982421875, + "logps/rejected": -1.6140625476837158, + "loss": 0.8963, + "nll_loss": 0.87109375, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06978759914636612, + "rewards/margins": 0.09173583984375, + "rewards/rejected": -0.16159668564796448, + "step": 4060 + }, + { + "epoch": 0.3088480801335559, + "grad_norm": 2.0422660259092082, + "learning_rate": 1.2539862728441536e-06, + "log_odds_chosen": 1.2222168445587158, + "log_odds_ratio": -0.46635740995407104, + "logits/chosen": -0.9429687261581421, + "logits/rejected": -0.8658202886581421, + "logps/chosen": -0.699999988079071, + "logps/rejected": -1.578710913658142, + "loss": 0.9181, + "nll_loss": 0.825976550579071, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.0699462890625, + "rewards/margins": 0.0879364013671875, + "rewards/rejected": -0.157958984375, + "step": 4070 + }, + { + "epoch": 0.30960692062528455, + "grad_norm": 2.2480501927541967, + "learning_rate": 1.252448582170299e-06, + "log_odds_chosen": 1.4699218273162842, + "log_odds_ratio": -0.43339842557907104, + "logits/chosen": -0.9517577886581421, + "logits/rejected": -0.873828113079071, + "logps/chosen": -0.7090820074081421, + "logps/rejected": -1.7548828125, + "loss": 0.9077, + "nll_loss": 0.899609386920929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.07089843600988388, + "rewards/margins": 0.10444335639476776, + "rewards/rejected": -0.17539063096046448, + "step": 4080 + }, + { + "epoch": 0.3103657611170132, + "grad_norm": 2.2268249219221, + "learning_rate": 1.2509165344015243e-06, + "log_odds_chosen": 1.2501952648162842, + "log_odds_ratio": -0.46025389432907104, + "logits/chosen": -0.967968761920929, + "logits/rejected": -0.870312511920929, + "logps/chosen": -0.673632800579071, + "logps/rejected": -1.5476562976837158, + "loss": 0.9388, + "nll_loss": 0.914843738079071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06741943210363388, + "rewards/margins": 0.08745117485523224, + "rewards/rejected": -0.15473632514476776, + "step": 4090 + }, + { + "epoch": 0.31112460160874184, + "grad_norm": 2.27005999070546, + "learning_rate": 1.2493900951088486e-06, + "log_odds_chosen": 1.28955078125, + "log_odds_ratio": -0.42329102754592896, + "logits/chosen": -0.951171875, + "logits/rejected": -0.8755859136581421, + "logps/chosen": -0.691601574420929, + "logps/rejected": -1.557031273841858, + "loss": 0.9157, + "nll_loss": 0.8306640386581421, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06904296576976776, + "rewards/margins": 0.0865631103515625, + "rewards/rejected": -0.1556396484375, + "step": 4100 + }, + { + "epoch": 0.3118834421004705, + "grad_norm": 2.1754099073977238, + "learning_rate": 1.2478692301566601e-06, + "log_odds_chosen": 1.5134766101837158, + "log_odds_ratio": -0.4137206971645355, + "logits/chosen": -0.948046863079071, + "logits/rejected": -0.8490234613418579, + "logps/chosen": -0.667773425579071, + "logps/rejected": -1.7433593273162842, + "loss": 0.902, + "nll_loss": 0.8935546875, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06674804538488388, + "rewards/margins": 0.10740967094898224, + "rewards/rejected": -0.17424316704273224, + "step": 4110 + }, + { + "epoch": 0.3126422825921991, + "grad_norm": 1.8991756706793939, + "learning_rate": 1.2463539056995116e-06, + "log_odds_chosen": 1.2667968273162842, + "log_odds_ratio": -0.44287109375, + "logits/chosen": -0.9554687738418579, + "logits/rejected": -0.882617175579071, + "logps/chosen": -0.703125, + "logps/rejected": -1.630859375, + "loss": 0.9017, + "nll_loss": 0.8999999761581421, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07025146484375, + "rewards/margins": 0.09304885566234589, + "rewards/rejected": -0.16325683891773224, + "step": 4120 + }, + { + "epoch": 0.31340112308392776, + "grad_norm": 1.960326000266427, + "learning_rate": 1.2448440881789541e-06, + "log_odds_chosen": 1.252832055091858, + "log_odds_ratio": -0.4205078184604645, + "logits/chosen": -0.9300781488418579, + "logits/rejected": -0.866406261920929, + "logps/chosen": -0.6532226800918579, + "logps/rejected": -1.4568359851837158, + "loss": 0.8844, + "nll_loss": 0.8388671875, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06529541313648224, + "rewards/margins": 0.08049926906824112, + "rewards/rejected": -0.14572754502296448, + "step": 4130 + }, + { + "epoch": 0.3141599635756564, + "grad_norm": 1.9958829149266735, + "learning_rate": 1.2433397443204184e-06, + "log_odds_chosen": 1.1348145008087158, + "log_odds_ratio": -0.4706054627895355, + "logits/chosen": -0.9605468511581421, + "logits/rejected": -0.8822265863418579, + "logps/chosen": -0.6656249761581421, + "logps/rejected": -1.4658203125, + "loss": 0.9019, + "nll_loss": 0.8388671875, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06654052436351776, + "rewards/margins": 0.07998809963464737, + "rewards/rejected": -0.14653320610523224, + "step": 4140 + }, + { + "epoch": 0.31491880406738504, + "grad_norm": 4.091569889843401, + "learning_rate": 1.2418408411301324e-06, + "log_odds_chosen": 1.275976538658142, + "log_odds_ratio": -0.400634765625, + "logits/chosen": -0.9775390625, + "logits/rejected": -0.8160156011581421, + "logps/chosen": -0.6201171875, + "logps/rejected": -1.4548828601837158, + "loss": 0.901, + "nll_loss": 0.78271484375, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06202392652630806, + "rewards/margins": 0.08343505859375, + "rewards/rejected": -0.14548340439796448, + "step": 4150 + }, + { + "epoch": 0.3156776445591137, + "grad_norm": 2.2497340997960076, + "learning_rate": 1.2403473458920844e-06, + "log_odds_chosen": 1.372656226158142, + "log_odds_ratio": -0.43525391817092896, + "logits/chosen": -0.9624999761581421, + "logits/rejected": -0.8412109613418579, + "logps/chosen": -0.692187488079071, + "logps/rejected": -1.6554687023162842, + "loss": 0.8977, + "nll_loss": 0.8486328125, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06931152194738388, + "rewards/margins": 0.09630127251148224, + "rewards/rejected": -0.16550293564796448, + "step": 4160 + }, + { + "epoch": 0.3164364850508423, + "grad_norm": 2.038350353584351, + "learning_rate": 1.2388592261650217e-06, + "log_odds_chosen": 1.199072241783142, + "log_odds_ratio": -0.4593749940395355, + "logits/chosen": -0.9271484613418579, + "logits/rejected": -0.8607422113418579, + "logps/chosen": -0.6734374761581421, + "logps/rejected": -1.5021483898162842, + "loss": 0.8952, + "nll_loss": 0.833984375, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06735839694738388, + "rewards/margins": 0.08281860500574112, + "rewards/rejected": -0.15019531548023224, + "step": 4170 + }, + { + "epoch": 0.31719532554257096, + "grad_norm": 2.1135411907978474, + "learning_rate": 1.2373764497794918e-06, + "log_odds_chosen": 1.4931640625, + "log_odds_ratio": -0.38886719942092896, + "logits/chosen": -0.953320324420929, + "logits/rejected": -0.8460937738418579, + "logps/chosen": -0.63427734375, + "logps/rejected": -1.657812476158142, + "loss": 0.8894, + "nll_loss": 0.837890625, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06341552734375, + "rewards/margins": 0.10235595703125, + "rewards/rejected": -0.16586914658546448, + "step": 4180 + }, + { + "epoch": 0.3179541660342996, + "grad_norm": 2.0657817064350033, + "learning_rate": 1.2358989848349217e-06, + "log_odds_chosen": 1.474707007408142, + "log_odds_ratio": -0.37358397245407104, + "logits/chosen": -0.8857421875, + "logits/rejected": -0.806445300579071, + "logps/chosen": -0.6400390863418579, + "logps/rejected": -1.6335937976837158, + "loss": 0.8925, + "nll_loss": 0.833789050579071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06398925930261612, + "rewards/margins": 0.09937743842601776, + "rewards/rejected": -0.16340331733226776, + "step": 4190 + }, + { + "epoch": 0.31871300652602824, + "grad_norm": 2.182532950524458, + "learning_rate": 1.2344267996967353e-06, + "log_odds_chosen": 1.465917944908142, + "log_odds_ratio": -0.3947509825229645, + "logits/chosen": -0.9203125238418579, + "logits/rejected": -0.852734386920929, + "logps/chosen": -0.662402331829071, + "logps/rejected": -1.6550781726837158, + "loss": 0.908, + "nll_loss": 0.9375, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06618652492761612, + "rewards/margins": 0.09916381537914276, + "rewards/rejected": -0.16538086533546448, + "step": 4200 + }, + { + "epoch": 0.3194718470177569, + "grad_norm": 1.9590389783181057, + "learning_rate": 1.2329598629935076e-06, + "log_odds_chosen": 1.366308569908142, + "log_odds_ratio": -0.4139648377895355, + "logits/chosen": -0.9091796875, + "logits/rejected": -0.8013671636581421, + "logps/chosen": -0.6680663824081421, + "logps/rejected": -1.6267578601837158, + "loss": 0.8914, + "nll_loss": 0.8695312738418579, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06683349609375, + "rewards/margins": 0.09581909328699112, + "rewards/rejected": -0.16259765625, + "step": 4210 + }, + { + "epoch": 0.3202306875094855, + "grad_norm": 2.1182170742214326, + "learning_rate": 1.2314981436141583e-06, + "log_odds_chosen": 1.3098633289337158, + "log_odds_ratio": -0.4046386778354645, + "logits/chosen": -0.9716796875, + "logits/rejected": -0.8667968511581421, + "logps/chosen": -0.619335949420929, + "logps/rejected": -1.520117163658142, + "loss": 0.8848, + "nll_loss": 0.7918945550918579, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06196289137005806, + "rewards/margins": 0.09002685546875, + "rewards/rejected": -0.15190429985523224, + "step": 4220 + }, + { + "epoch": 0.32098952800121416, + "grad_norm": 2.1681970591127184, + "learning_rate": 1.2300416107051802e-06, + "log_odds_chosen": 1.2394530773162842, + "log_odds_ratio": -0.45927733182907104, + "logits/chosen": -0.9613281488418579, + "logits/rejected": -0.832812488079071, + "logps/chosen": -0.72607421875, + "logps/rejected": -1.5896484851837158, + "loss": 0.9274, + "nll_loss": 0.852343738079071, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.07264403998851776, + "rewards/margins": 0.08648681640625, + "rewards/rejected": -0.15913085639476776, + "step": 4230 + }, + { + "epoch": 0.3217483684929428, + "grad_norm": 2.046706232589475, + "learning_rate": 1.2285902336679024e-06, + "log_odds_chosen": 1.12548828125, + "log_odds_ratio": -0.4857421815395355, + "logits/chosen": -0.963085949420929, + "logits/rejected": -0.8150390386581421, + "logps/chosen": -0.7162109613418579, + "logps/rejected": -1.516210913658142, + "loss": 0.9045, + "nll_loss": 0.8978515863418579, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07159423828125, + "rewards/margins": 0.08000488579273224, + "rewards/rejected": -0.151611328125, + "step": 4240 + }, + { + "epoch": 0.32250720898467145, + "grad_norm": 2.291442872750335, + "learning_rate": 1.2271439821557926e-06, + "log_odds_chosen": 1.22265625, + "log_odds_ratio": -0.43427735567092896, + "logits/chosen": -0.926953136920929, + "logits/rejected": -0.859179675579071, + "logps/chosen": -0.6561523675918579, + "logps/rejected": -1.5001952648162842, + "loss": 0.901, + "nll_loss": 0.830859363079071, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06550292670726776, + "rewards/margins": 0.08442382514476776, + "rewards/rejected": -0.14992675185203552, + "step": 4250 + }, + { + "epoch": 0.3232660494764001, + "grad_norm": 2.3934126129829516, + "learning_rate": 1.225702826071791e-06, + "log_odds_chosen": 1.6476562023162842, + "log_odds_ratio": -0.3468261659145355, + "logits/chosen": -0.98828125, + "logits/rejected": -0.870898425579071, + "logps/chosen": -0.622851550579071, + "logps/rejected": -1.7775390148162842, + "loss": 0.8859, + "nll_loss": 0.803417980670929, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06228027492761612, + "rewards/margins": 0.11549071967601776, + "rewards/rejected": -0.17783203721046448, + "step": 4260 + }, + { + "epoch": 0.3240248899681287, + "grad_norm": 2.2978147982794916, + "learning_rate": 1.2242667355656797e-06, + "log_odds_chosen": 1.464257836341858, + "log_odds_ratio": -0.3810058534145355, + "logits/chosen": -0.9378906488418579, + "logits/rejected": -0.811328113079071, + "logps/chosen": -0.6431640386581421, + "logps/rejected": -1.6599609851837158, + "loss": 0.862, + "nll_loss": 0.8465820550918579, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06434325873851776, + "rewards/margins": 0.1016845703125, + "rewards/rejected": -0.16606445610523224, + "step": 4270 + }, + { + "epoch": 0.3247837304598573, + "grad_norm": 2.2231775385940775, + "learning_rate": 1.2228356810314862e-06, + "log_odds_chosen": 1.059661865234375, + "log_odds_ratio": -0.49833983182907104, + "logits/chosen": -0.930859386920929, + "logits/rejected": -0.8203125, + "logps/chosen": -0.7120116949081421, + "logps/rejected": -1.4490234851837158, + "loss": 0.9366, + "nll_loss": 0.8724609613418579, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07120361179113388, + "rewards/margins": 0.07371368259191513, + "rewards/rejected": -0.14499512314796448, + "step": 4280 + }, + { + "epoch": 0.32554257095158595, + "grad_norm": 2.2345119533491937, + "learning_rate": 1.2214096331049186e-06, + "log_odds_chosen": 1.5973632335662842, + "log_odds_ratio": -0.401611328125, + "logits/chosen": -0.9457031488418579, + "logits/rejected": -0.8681640625, + "logps/chosen": -0.6317383050918579, + "logps/rejected": -1.779687523841858, + "loss": 0.8845, + "nll_loss": 0.7544921636581421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06319580227136612, + "rewards/margins": 0.11481933295726776, + "rewards/rejected": -0.17795410752296448, + "step": 4290 + }, + { + "epoch": 0.3263014114433146, + "grad_norm": 2.5710473923093504, + "learning_rate": 1.2199885626608373e-06, + "log_odds_chosen": 1.3624999523162842, + "log_odds_ratio": -0.4041503965854645, + "logits/chosen": -0.9730468988418579, + "logits/rejected": -0.8931640386581421, + "logps/chosen": -0.6630859375, + "logps/rejected": -1.584375023841858, + "loss": 0.902, + "nll_loss": 0.8310546875, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06634521484375, + "rewards/margins": 0.09211425483226776, + "rewards/rejected": -0.15842285752296448, + "step": 4300 + }, + { + "epoch": 0.32706025193504323, + "grad_norm": 1.9495590217172698, + "learning_rate": 1.2185724408107546e-06, + "log_odds_chosen": 1.344335913658142, + "log_odds_ratio": -0.4015136659145355, + "logits/chosen": -0.976367175579071, + "logits/rejected": -0.8623046875, + "logps/chosen": -0.651171863079071, + "logps/rejected": -1.5617187023162842, + "loss": 0.9226, + "nll_loss": 0.8587890863418579, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06514892727136612, + "rewards/margins": 0.09112548828125, + "rewards/rejected": -0.15617676079273224, + "step": 4310 + }, + { + "epoch": 0.3278190924267719, + "grad_norm": 2.0775732902118866, + "learning_rate": 1.2171612389003689e-06, + "log_odds_chosen": 1.409570336341858, + "log_odds_ratio": -0.37944334745407104, + "logits/chosen": -0.9593750238418579, + "logits/rejected": -0.8255859613418579, + "logps/chosen": -0.671679675579071, + "logps/rejected": -1.689453125, + "loss": 0.8989, + "nll_loss": 0.8636718988418579, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06711425632238388, + "rewards/margins": 0.101898193359375, + "rewards/rejected": -0.16921386122703552, + "step": 4320 + }, + { + "epoch": 0.3285779329185005, + "grad_norm": 2.1901595897704644, + "learning_rate": 1.2157549285071297e-06, + "log_odds_chosen": 1.056640625, + "log_odds_ratio": -0.494873046875, + "logits/chosen": -0.951367199420929, + "logits/rejected": -0.8515625, + "logps/chosen": -0.708789050579071, + "logps/rejected": -1.460546851158142, + "loss": 0.9165, + "nll_loss": 0.8740234375, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.07082519680261612, + "rewards/margins": 0.07513427734375, + "rewards/rejected": -0.14604492485523224, + "step": 4330 + }, + { + "epoch": 0.32933677341022916, + "grad_norm": 1.9604302213993103, + "learning_rate": 1.2143534814378327e-06, + "log_odds_chosen": 1.382421851158142, + "log_odds_ratio": -0.41289061307907104, + "logits/chosen": -0.914843738079071, + "logits/rejected": -0.8330078125, + "logps/chosen": -0.682910144329071, + "logps/rejected": -1.63671875, + "loss": 0.8791, + "nll_loss": 0.846386730670929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06829833984375, + "rewards/margins": 0.095367431640625, + "rewards/rejected": -0.16352538764476776, + "step": 4340 + }, + { + "epoch": 0.3300956139019578, + "grad_norm": 2.156153319128, + "learning_rate": 1.2129568697262454e-06, + "log_odds_chosen": 1.302148461341858, + "log_odds_ratio": -0.41865235567092896, + "logits/chosen": -0.9525390863418579, + "logits/rejected": -0.8365234136581421, + "logps/chosen": -0.6341797113418579, + "logps/rejected": -1.4474608898162842, + "loss": 0.8645, + "nll_loss": 0.8104492425918579, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06348876655101776, + "rewards/margins": 0.08118285983800888, + "rewards/rejected": -0.1446533203125, + "step": 4350 + }, + { + "epoch": 0.33085445439368644, + "grad_norm": 2.5408462766533515, + "learning_rate": 1.2115650656307653e-06, + "log_odds_chosen": 1.381445288658142, + "log_odds_ratio": -0.3592773377895355, + "logits/chosen": -0.9085937738418579, + "logits/rejected": -0.8330078125, + "logps/chosen": -0.6597656011581421, + "logps/rejected": -1.552343726158142, + "loss": 0.8761, + "nll_loss": 0.7962890863418579, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06594238430261612, + "rewards/margins": 0.08934326469898224, + "rewards/rejected": -0.15520019829273224, + "step": 4360 + }, + { + "epoch": 0.3316132948854151, + "grad_norm": 2.755454510311766, + "learning_rate": 1.210178041632103e-06, + "log_odds_chosen": 1.542578101158142, + "log_odds_ratio": -0.3629394471645355, + "logits/chosen": -0.909375011920929, + "logits/rejected": -0.854296863079071, + "logps/chosen": -0.62451171875, + "logps/rejected": -1.678125023841858, + "loss": 0.9076, + "nll_loss": 0.865039050579071, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06241454929113388, + "rewards/margins": 0.10528564453125, + "rewards/rejected": -0.16774901747703552, + "step": 4370 + }, + { + "epoch": 0.3323721353771437, + "grad_norm": 2.043167141768478, + "learning_rate": 1.2087957704309988e-06, + "log_odds_chosen": 1.2589843273162842, + "log_odds_ratio": -0.42744141817092896, + "logits/chosen": -0.9208984375, + "logits/rejected": -0.8177734613418579, + "logps/chosen": -0.638476550579071, + "logps/rejected": -1.4943358898162842, + "loss": 0.8986, + "nll_loss": 0.804492175579071, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.0638427734375, + "rewards/margins": 0.08549194037914276, + "rewards/rejected": -0.14938965439796448, + "step": 4380 + }, + { + "epoch": 0.33313097586887236, + "grad_norm": 2.174469428614346, + "learning_rate": 1.2074182249459642e-06, + "log_odds_chosen": 1.512841820716858, + "log_odds_ratio": -0.40498048067092896, + "logits/chosen": -0.9583984613418579, + "logits/rejected": -0.875781238079071, + "logps/chosen": -0.666210949420929, + "logps/rejected": -1.726171851158142, + "loss": 0.8872, + "nll_loss": 0.817675769329071, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06667480617761612, + "rewards/margins": 0.10597991943359375, + "rewards/rejected": -0.17250975966453552, + "step": 4390 + }, + { + "epoch": 0.333889816360601, + "grad_norm": 2.220872324929471, + "learning_rate": 1.2060453783110545e-06, + "log_odds_chosen": 1.1480224132537842, + "log_odds_ratio": -0.46357423067092896, + "logits/chosen": -0.9693359136581421, + "logits/rejected": -0.8525390625, + "logps/chosen": -0.6807616949081421, + "logps/rejected": -1.4871094226837158, + "loss": 0.8837, + "nll_loss": 0.805468738079071, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06805419921875, + "rewards/margins": 0.08069610595703125, + "rewards/rejected": -0.14877930283546448, + "step": 4400 + }, + { + "epoch": 0.33464865685232964, + "grad_norm": 2.2372440653759513, + "learning_rate": 1.2046772038736682e-06, + "log_odds_chosen": 1.3009765148162842, + "log_odds_ratio": -0.434326171875, + "logits/chosen": -0.93359375, + "logits/rejected": -0.8521484136581421, + "logps/chosen": -0.6429687738418579, + "logps/rejected": -1.5300781726837158, + "loss": 0.9141, + "nll_loss": 0.8697265386581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06434325873851776, + "rewards/margins": 0.08879394829273224, + "rewards/rejected": -0.15317383408546448, + "step": 4410 + }, + { + "epoch": 0.3354074973440583, + "grad_norm": 2.2973654270322967, + "learning_rate": 1.2033136751923736e-06, + "log_odds_chosen": 1.101904273033142, + "log_odds_ratio": -0.45087891817092896, + "logits/chosen": -0.919726550579071, + "logits/rejected": -0.7900390625, + "logps/chosen": -0.700976550579071, + "logps/rejected": -1.473242163658142, + "loss": 0.9005, + "nll_loss": 0.849414050579071, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.07012939453125, + "rewards/margins": 0.07722167670726776, + "rewards/rejected": -0.14738769829273224, + "step": 4420 + }, + { + "epoch": 0.3361663378357869, + "grad_norm": 2.247061985239726, + "learning_rate": 1.201954766034762e-06, + "log_odds_chosen": 1.4426758289337158, + "log_odds_ratio": -0.42011719942092896, + "logits/chosen": -0.9330078363418579, + "logits/rejected": -0.845507800579071, + "logps/chosen": -0.661425769329071, + "logps/rejected": -1.6591796875, + "loss": 0.8758, + "nll_loss": 0.8121093511581421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.066162109375, + "rewards/margins": 0.09980468451976776, + "rewards/rejected": -0.16584472358226776, + "step": 4430 + }, + { + "epoch": 0.33692517832751556, + "grad_norm": 2.2570856208655163, + "learning_rate": 1.2006004503753285e-06, + "log_odds_chosen": 1.5849609375, + "log_odds_ratio": -0.4055419862270355, + "logits/chosen": -0.9097656011581421, + "logits/rejected": -0.8033202886581421, + "logps/chosen": -0.640917956829071, + "logps/rejected": -1.806054711341858, + "loss": 0.903, + "nll_loss": 0.824999988079071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06409911811351776, + "rewards/margins": 0.11650238186120987, + "rewards/rejected": -0.1805419921875, + "step": 4440 + }, + { + "epoch": 0.3376840188192442, + "grad_norm": 2.2141600181166363, + "learning_rate": 1.1992507023933782e-06, + "log_odds_chosen": 1.526513695716858, + "log_odds_ratio": -0.3704589903354645, + "logits/chosen": -0.939648449420929, + "logits/rejected": -0.822265625, + "logps/chosen": -0.69140625, + "logps/rejected": -1.748437523841858, + "loss": 0.8751, + "nll_loss": 0.888476550579071, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06917724758386612, + "rewards/margins": 0.10583877563476562, + "rewards/rejected": -0.1751708984375, + "step": 4450 + }, + { + "epoch": 0.33844285931097284, + "grad_norm": 2.0103647610137925, + "learning_rate": 1.1979054964709597e-06, + "log_odds_chosen": 1.1826171875, + "log_odds_ratio": -0.4234375059604645, + "logits/chosen": -0.941210925579071, + "logits/rejected": -0.809374988079071, + "logps/chosen": -0.6451171636581421, + "logps/rejected": -1.4169921875, + "loss": 0.8857, + "nll_loss": 0.794140636920929, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06455077975988388, + "rewards/margins": 0.07726440578699112, + "rewards/rejected": -0.14177246391773224, + "step": 4460 + }, + { + "epoch": 0.3392016998027015, + "grad_norm": 2.344873760987262, + "learning_rate": 1.1965648071908207e-06, + "log_odds_chosen": 1.375, + "log_odds_ratio": -0.39765626192092896, + "logits/chosen": -0.9437500238418579, + "logits/rejected": -0.8138672113418579, + "logps/chosen": -0.626953125, + "logps/rejected": -1.572265625, + "loss": 0.8756, + "nll_loss": 0.788769543170929, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06269530951976776, + "rewards/margins": 0.09456787258386612, + "rewards/rejected": -0.15725097060203552, + "step": 4470 + }, + { + "epoch": 0.3399605402944301, + "grad_norm": 1.9603069157506832, + "learning_rate": 1.1952286093343935e-06, + "log_odds_chosen": 1.318945288658142, + "log_odds_ratio": -0.42138671875, + "logits/chosen": -0.9644531011581421, + "logits/rejected": -0.848437488079071, + "logps/chosen": -0.660449206829071, + "logps/rejected": -1.5525391101837158, + "loss": 0.8975, + "nll_loss": 0.8497070074081421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06605224311351776, + "rewards/margins": 0.08906249701976776, + "rewards/rejected": -0.15517577528953552, + "step": 4480 + }, + { + "epoch": 0.34071938078615877, + "grad_norm": 2.3007253003330135, + "learning_rate": 1.1938968778798005e-06, + "log_odds_chosen": 1.3662109375, + "log_odds_ratio": -0.3960937559604645, + "logits/chosen": -0.921875, + "logits/rejected": -0.7919921875, + "logps/chosen": -0.637890636920929, + "logps/rejected": -1.585546851158142, + "loss": 0.8729, + "nll_loss": 0.8238281011581421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06375732272863388, + "rewards/margins": 0.09487304836511612, + "rewards/rejected": -0.15864257514476776, + "step": 4490 + }, + { + "epoch": 0.3414782212778874, + "grad_norm": 2.0606137302048273, + "learning_rate": 1.1925695879998878e-06, + "log_odds_chosen": 1.36328125, + "log_odds_ratio": -0.41728514432907104, + "logits/chosen": -0.9349609613418579, + "logits/rejected": -0.8619140386581421, + "logps/chosen": -0.6795898675918579, + "logps/rejected": -1.6300780773162842, + "loss": 0.8698, + "nll_loss": 0.849609375, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.0679931640625, + "rewards/margins": 0.09499511867761612, + "rewards/rejected": -0.16298827528953552, + "step": 4500 + }, + { + "epoch": 0.34223706176961605, + "grad_norm": 2.2176914597724995, + "learning_rate": 1.1912467150602794e-06, + "log_odds_chosen": 1.429296851158142, + "log_odds_ratio": -0.3785644471645355, + "logits/chosen": -0.946484386920929, + "logits/rejected": -0.8521484136581421, + "logps/chosen": -0.6634765863418579, + "logps/rejected": -1.657812476158142, + "loss": 0.8836, + "nll_loss": 0.82763671875, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06641845405101776, + "rewards/margins": 0.09949950873851776, + "rewards/rejected": -0.16584472358226776, + "step": 4510 + }, + { + "epoch": 0.3429959022613447, + "grad_norm": 2.588685801085802, + "learning_rate": 1.189928234617459e-06, + "log_odds_chosen": 1.237280249595642, + "log_odds_ratio": -0.4388671815395355, + "logits/chosen": -0.9232422113418579, + "logits/rejected": -0.801953136920929, + "logps/chosen": -0.6878906488418579, + "logps/rejected": -1.5712890625, + "loss": 0.8666, + "nll_loss": 0.8218749761581421, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06878662109375, + "rewards/margins": 0.08840103447437286, + "rewards/rejected": -0.15731200575828552, + "step": 4520 + }, + { + "epoch": 0.34375474275307333, + "grad_norm": 2.320935556141627, + "learning_rate": 1.1886141224168716e-06, + "log_odds_chosen": 1.211328148841858, + "log_odds_ratio": -0.43071287870407104, + "logits/chosen": -0.8705078363418579, + "logits/rejected": -0.8150390386581421, + "logps/chosen": -0.641406238079071, + "logps/rejected": -1.460546851158142, + "loss": 0.8651, + "nll_loss": 0.822070300579071, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06413574516773224, + "rewards/margins": 0.08183594048023224, + "rewards/rejected": -0.14589843153953552, + "step": 4530 + }, + { + "epoch": 0.34451358324480197, + "grad_norm": 2.1049123689733786, + "learning_rate": 1.1873043543910495e-06, + "log_odds_chosen": 1.21533203125, + "log_odds_ratio": -0.42182618379592896, + "logits/chosen": -0.920117199420929, + "logits/rejected": -0.8255859613418579, + "logps/chosen": -0.618945300579071, + "logps/rejected": -1.412500023841858, + "loss": 0.8874, + "nll_loss": 0.83203125, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06195068359375, + "rewards/margins": 0.07939453423023224, + "rewards/rejected": -0.14145508408546448, + "step": 4540 + }, + { + "epoch": 0.34527242373653055, + "grad_norm": 2.1976254676362554, + "learning_rate": 1.1859989066577617e-06, + "log_odds_chosen": 1.4050781726837158, + "log_odds_ratio": -0.388671875, + "logits/chosen": -0.910351574420929, + "logits/rejected": -0.80078125, + "logps/chosen": -0.6504882574081421, + "logps/rejected": -1.6105468273162842, + "loss": 0.8843, + "nll_loss": 0.800976574420929, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06500244140625, + "rewards/margins": 0.0960693359375, + "rewards/rejected": -0.16103515028953552, + "step": 4550 + }, + { + "epoch": 0.3460312642282592, + "grad_norm": 2.1822551995963404, + "learning_rate": 1.1846977555181846e-06, + "log_odds_chosen": 1.103369116783142, + "log_odds_ratio": -0.4722656309604645, + "logits/chosen": -0.8988281488418579, + "logits/rejected": -0.824999988079071, + "logps/chosen": -0.7012695074081421, + "logps/rejected": -1.4494140148162842, + "loss": 0.8828, + "nll_loss": 0.890625, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.07011719048023224, + "rewards/margins": 0.07481994479894638, + "rewards/rejected": -0.14504393935203552, + "step": 4560 + }, + { + "epoch": 0.34679010471998784, + "grad_norm": 2.3308594142253143, + "learning_rate": 1.1834008774550946e-06, + "log_odds_chosen": 1.40966796875, + "log_odds_ratio": -0.40717774629592896, + "logits/chosen": -0.936328113079071, + "logits/rejected": -0.8021484613418579, + "logps/chosen": -0.637011706829071, + "logps/rejected": -1.5871093273162842, + "loss": 0.8749, + "nll_loss": 0.81494140625, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06374511867761612, + "rewards/margins": 0.09494628757238388, + "rewards/rejected": -0.15874023735523224, + "step": 4570 + }, + { + "epoch": 0.3475489452117165, + "grad_norm": 2.4823474237824605, + "learning_rate": 1.1821082491310835e-06, + "log_odds_chosen": 1.223535180091858, + "log_odds_ratio": -0.4647460877895355, + "logits/chosen": -0.913281261920929, + "logits/rejected": -0.8218749761581421, + "logps/chosen": -0.6629883050918579, + "logps/rejected": -1.484375, + "loss": 0.858, + "nll_loss": 0.824414074420929, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06619872897863388, + "rewards/margins": 0.08223648369312286, + "rewards/rejected": -0.14836426079273224, + "step": 4580 + }, + { + "epoch": 0.3483077857034451, + "grad_norm": 2.0314177948784358, + "learning_rate": 1.1808198473867937e-06, + "log_odds_chosen": 1.179785132408142, + "log_odds_ratio": -0.46484375, + "logits/chosen": -1.0146484375, + "logits/rejected": -0.906445324420929, + "logps/chosen": -0.69921875, + "logps/rejected": -1.537500023841858, + "loss": 0.8742, + "nll_loss": 0.8388671875, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06987304985523224, + "rewards/margins": 0.08375243842601776, + "rewards/rejected": -0.15375976264476776, + "step": 4590 + }, + { + "epoch": 0.34906662619517376, + "grad_norm": 1.9804242538388541, + "learning_rate": 1.179535649239177e-06, + "log_odds_chosen": 1.2795898914337158, + "log_odds_ratio": -0.4310058653354645, + "logits/chosen": -0.903515636920929, + "logits/rejected": -0.843554675579071, + "logps/chosen": -0.655957043170929, + "logps/rejected": -1.542578101158142, + "loss": 0.8787, + "nll_loss": 0.8548828363418579, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06556396186351776, + "rewards/margins": 0.08857116848230362, + "rewards/rejected": -0.15422363579273224, + "step": 4600 + }, + { + "epoch": 0.3498254666869024, + "grad_norm": 2.1676793117363498, + "learning_rate": 1.178255631879771e-06, + "log_odds_chosen": 1.33642578125, + "log_odds_ratio": -0.406982421875, + "logits/chosen": -0.940234363079071, + "logits/rejected": -0.830273449420929, + "logps/chosen": -0.685351550579071, + "logps/rejected": -1.626953125, + "loss": 0.886, + "nll_loss": 0.8482421636581421, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06854248046875, + "rewards/margins": 0.09415893256664276, + "rewards/rejected": -0.16267089545726776, + "step": 4610 + }, + { + "epoch": 0.35058430717863104, + "grad_norm": 2.0364662326117187, + "learning_rate": 1.1769797726729992e-06, + "log_odds_chosen": 0.963574230670929, + "log_odds_ratio": -0.4800781309604645, + "logits/chosen": -0.9091796875, + "logits/rejected": -0.803515613079071, + "logps/chosen": -0.69482421875, + "logps/rejected": -1.3136718273162842, + "loss": 0.8694, + "nll_loss": 0.900195300579071, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06943359225988388, + "rewards/margins": 0.06182251125574112, + "rewards/rejected": -0.13129882514476776, + "step": 4620 + }, + { + "epoch": 0.3513431476703597, + "grad_norm": 2.286515715845322, + "learning_rate": 1.1757080491544881e-06, + "log_odds_chosen": 1.3899657726287842, + "log_odds_ratio": -0.4193359315395355, + "logits/chosen": -0.9341796636581421, + "logits/rejected": -0.8255859613418579, + "logps/chosen": -0.683789074420929, + "logps/rejected": -1.6462891101837158, + "loss": 0.8888, + "nll_loss": 0.846972644329071, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06842041015625, + "rewards/margins": 0.09616546332836151, + "rewards/rejected": -0.16464844346046448, + "step": 4630 + }, + { + "epoch": 0.3521019881620883, + "grad_norm": 2.1123173473303924, + "learning_rate": 1.1744404390294068e-06, + "log_odds_chosen": 1.2043945789337158, + "log_odds_ratio": -0.45966798067092896, + "logits/chosen": -0.9322265386581421, + "logits/rejected": -0.8619140386581421, + "logps/chosen": -0.688183605670929, + "logps/rejected": -1.525390625, + "loss": 0.853, + "nll_loss": 0.8505859375, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06875000149011612, + "rewards/margins": 0.08389892429113388, + "rewards/rejected": -0.15251465141773224, + "step": 4640 + }, + { + "epoch": 0.35286082865381696, + "grad_norm": 2.1002667873030645, + "learning_rate": 1.1731769201708264e-06, + "log_odds_chosen": 1.4666016101837158, + "log_odds_ratio": -0.45659178495407104, + "logits/chosen": -0.916210949420929, + "logits/rejected": -0.8314453363418579, + "logps/chosen": -0.671875, + "logps/rejected": -1.7628905773162842, + "loss": 0.8795, + "nll_loss": 0.8500000238418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06715087592601776, + "rewards/margins": 0.10904540866613388, + "rewards/rejected": -0.17622070014476776, + "step": 4650 + }, + { + "epoch": 0.3536196691455456, + "grad_norm": 2.5257160808205312, + "learning_rate": 1.1719174706180952e-06, + "log_odds_chosen": 1.6716797351837158, + "log_odds_ratio": -0.3566650450229645, + "logits/chosen": -0.924609363079071, + "logits/rejected": -0.7876952886581421, + "logps/chosen": -0.6436523199081421, + "logps/rejected": -1.8214843273162842, + "loss": 0.8786, + "nll_loss": 0.822265625, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06437988579273224, + "rewards/margins": 0.11765136569738388, + "rewards/rejected": -0.18222656846046448, + "step": 4660 + }, + { + "epoch": 0.35437850963727424, + "grad_norm": 2.1474539080309687, + "learning_rate": 1.1706620685752386e-06, + "log_odds_chosen": 1.5402343273162842, + "log_odds_ratio": -0.36308592557907104, + "logits/chosen": -0.9468749761581421, + "logits/rejected": -0.8091796636581421, + "logps/chosen": -0.646679699420929, + "logps/rejected": -1.7273437976837158, + "loss": 0.8829, + "nll_loss": 0.829882800579071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06467285007238388, + "rewards/margins": 0.10795898735523224, + "rewards/rejected": -0.17270508408546448, + "step": 4670 + }, + { + "epoch": 0.3551373501290029, + "grad_norm": 2.2943867159959916, + "learning_rate": 1.1694106924093723e-06, + "log_odds_chosen": 1.168481469154358, + "log_odds_ratio": -0.4696289002895355, + "logits/chosen": -0.9359375238418579, + "logits/rejected": -0.836718738079071, + "logps/chosen": -0.728515625, + "logps/rejected": -1.5451171398162842, + "loss": 0.8728, + "nll_loss": 0.8746093511581421, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.07291259616613388, + "rewards/margins": 0.08176727592945099, + "rewards/rejected": -0.15458984673023224, + "step": 4680 + }, + { + "epoch": 0.3558961906207315, + "grad_norm": 2.6928939956255524, + "learning_rate": 1.1681633206491381e-06, + "log_odds_chosen": 1.176171898841858, + "log_odds_ratio": -0.4358886778354645, + "logits/chosen": -0.9214843511581421, + "logits/rejected": -0.7984374761581421, + "logps/chosen": -0.6650390625, + "logps/rejected": -1.473242163658142, + "loss": 0.9016, + "nll_loss": 0.853515625, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06654052436351776, + "rewards/margins": 0.08079834282398224, + "rewards/rejected": -0.147216796875, + "step": 4690 + }, + { + "epoch": 0.35665503111246016, + "grad_norm": 2.6738779587849146, + "learning_rate": 1.1669199319831564e-06, + "log_odds_chosen": 1.326562523841858, + "log_odds_ratio": -0.416015625, + "logits/chosen": -0.915820300579071, + "logits/rejected": -0.813671886920929, + "logps/chosen": -0.674023449420929, + "logps/rejected": -1.578515648841858, + "loss": 0.8946, + "nll_loss": 0.8443359136581421, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.0673828125, + "rewards/margins": 0.09053955227136612, + "rewards/rejected": -0.15786132216453552, + "step": 4700 + }, + { + "epoch": 0.3574138716041888, + "grad_norm": 2.400721974422087, + "learning_rate": 1.1656805052584958e-06, + "log_odds_chosen": 1.5525391101837158, + "log_odds_ratio": -0.41228026151657104, + "logits/chosen": -0.9332031011581421, + "logits/rejected": -0.82421875, + "logps/chosen": -0.6337890625, + "logps/rejected": -1.6853516101837158, + "loss": 0.8857, + "nll_loss": 0.789257824420929, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06333007663488388, + "rewards/margins": 0.10518798977136612, + "rewards/rejected": -0.16850586235523224, + "step": 4710 + }, + { + "epoch": 0.35817271209591744, + "grad_norm": 2.78621158336074, + "learning_rate": 1.164445019479164e-06, + "log_odds_chosen": 1.36474609375, + "log_odds_ratio": -0.4100097715854645, + "logits/chosen": -0.942578136920929, + "logits/rejected": -0.8260742425918579, + "logps/chosen": -0.6576172113418579, + "logps/rejected": -1.57421875, + "loss": 0.8657, + "nll_loss": 0.802929699420929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06569824367761612, + "rewards/margins": 0.09161376953125, + "rewards/rejected": -0.15742187201976776, + "step": 4720 + }, + { + "epoch": 0.3589315525876461, + "grad_norm": 2.1486906743867236, + "learning_rate": 1.1632134538046105e-06, + "log_odds_chosen": 1.153161644935608, + "log_odds_ratio": -0.4811035096645355, + "logits/chosen": -0.9345703125, + "logits/rejected": -0.806835949420929, + "logps/chosen": -0.6722656488418579, + "logps/rejected": -1.4718749523162842, + "loss": 0.8575, + "nll_loss": 0.855664074420929, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06721191108226776, + "rewards/margins": 0.07992400974035263, + "rewards/rejected": -0.14705809950828552, + "step": 4730 + }, + { + "epoch": 0.3596903930793747, + "grad_norm": 2.365164784115729, + "learning_rate": 1.1619857875482536e-06, + "log_odds_chosen": 1.594384789466858, + "log_odds_ratio": -0.3369140625, + "logits/chosen": -0.931640625, + "logits/rejected": -0.832226574420929, + "logps/chosen": -0.630664050579071, + "logps/rejected": -1.719140648841858, + "loss": 0.8798, + "nll_loss": 0.808300793170929, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.0631103515625, + "rewards/margins": 0.10862121731042862, + "rewards/rejected": -0.17164306342601776, + "step": 4740 + }, + { + "epoch": 0.36044923357110337, + "grad_norm": 2.1740905865131164, + "learning_rate": 1.1607620001760185e-06, + "log_odds_chosen": 1.189550757408142, + "log_odds_ratio": -0.46137696504592896, + "logits/chosen": -0.9443359375, + "logits/rejected": -0.8707031011581421, + "logps/chosen": -0.6524413824081421, + "logps/rejected": -1.4599609375, + "loss": 0.8714, + "nll_loss": 0.7978515625, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.06529541313648224, + "rewards/margins": 0.08076782524585724, + "rewards/rejected": -0.14604492485523224, + "step": 4750 + }, + { + "epoch": 0.361208074062832, + "grad_norm": 2.1663302947053142, + "learning_rate": 1.1595420713048968e-06, + "log_odds_chosen": 1.485742211341858, + "log_odds_ratio": -0.3995117247104645, + "logits/chosen": -0.9716796875, + "logits/rejected": -0.8482421636581421, + "logps/chosen": -0.7230468988418579, + "logps/rejected": -1.788671851158142, + "loss": 0.8855, + "nll_loss": 0.843554675579071, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07232666015625, + "rewards/margins": 0.10650940239429474, + "rewards/rejected": -0.17890624701976776, + "step": 4760 + }, + { + "epoch": 0.36196691455456065, + "grad_norm": 3.308691876993286, + "learning_rate": 1.1583259807015182e-06, + "log_odds_chosen": 1.368554711341858, + "log_odds_ratio": -0.4066406190395355, + "logits/chosen": -0.9326171875, + "logits/rejected": -0.822070300579071, + "logps/chosen": -0.694531261920929, + "logps/rejected": -1.6535155773162842, + "loss": 0.88, + "nll_loss": 0.863085925579071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06947021186351776, + "rewards/margins": 0.09576416015625, + "rewards/rejected": -0.16523437201976776, + "step": 4770 + }, + { + "epoch": 0.3627257550462893, + "grad_norm": 2.192361081732721, + "learning_rate": 1.1571137082807434e-06, + "log_odds_chosen": 1.4275391101837158, + "log_odds_ratio": -0.38886719942092896, + "logits/chosen": -0.965039074420929, + "logits/rejected": -0.8587890863418579, + "logps/chosen": -0.6673828363418579, + "logps/rejected": -1.667578101158142, + "loss": 0.8549, + "nll_loss": 0.8187500238418579, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06673584133386612, + "rewards/margins": 0.10012207180261612, + "rewards/rejected": -0.16679687798023224, + "step": 4780 + }, + { + "epoch": 0.36348459553801793, + "grad_norm": 3.3996780477885356, + "learning_rate": 1.155905234104269e-06, + "log_odds_chosen": 1.564660668373108, + "log_odds_ratio": -0.3798828125, + "logits/chosen": -0.943554699420929, + "logits/rejected": -0.795703113079071, + "logps/chosen": -0.6434570550918579, + "logps/rejected": -1.7576172351837158, + "loss": 0.8883, + "nll_loss": 0.8423827886581421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06425781548023224, + "rewards/margins": 0.11158599704504013, + "rewards/rejected": -0.17592772841453552, + "step": 4790 + }, + { + "epoch": 0.36424343602974657, + "grad_norm": 2.185774065398957, + "learning_rate": 1.1547005383792516e-06, + "log_odds_chosen": 1.408593773841858, + "log_odds_ratio": -0.4390625059604645, + "logits/chosen": -0.964062511920929, + "logits/rejected": -0.8746093511581421, + "logps/chosen": -0.65283203125, + "logps/rejected": -1.615234375, + "loss": 0.8545, + "nll_loss": 0.817187488079071, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06531982123851776, + "rewards/margins": 0.09609375149011612, + "rewards/rejected": -0.16140136122703552, + "step": 4800 + }, + { + "epoch": 0.3650022765214752, + "grad_norm": 2.3827746388722084, + "learning_rate": 1.1534996014569446e-06, + "log_odds_chosen": 1.5846679210662842, + "log_odds_ratio": -0.39116209745407104, + "logits/chosen": -0.992382824420929, + "logits/rejected": -0.8853515386581421, + "logps/chosen": -0.662792980670929, + "logps/rejected": -1.7734375, + "loss": 0.8416, + "nll_loss": 0.82470703125, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06625976413488388, + "rewards/margins": 0.11094971001148224, + "rewards/rejected": -0.17724609375, + "step": 4810 + }, + { + "epoch": 0.36576111701320385, + "grad_norm": 2.22383281870511, + "learning_rate": 1.1523024038313547e-06, + "log_odds_chosen": 1.4586913585662842, + "log_odds_ratio": -0.3944335877895355, + "logits/chosen": -0.96875, + "logits/rejected": -0.8539062738418579, + "logps/chosen": -0.692675769329071, + "logps/rejected": -1.717187523841858, + "loss": 0.8684, + "nll_loss": 0.8714843988418579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06920166313648224, + "rewards/margins": 0.10234222561120987, + "rewards/rejected": -0.17170409858226776, + "step": 4820 + }, + { + "epoch": 0.36651995750493244, + "grad_norm": 2.405173511680066, + "learning_rate": 1.1511089261379083e-06, + "log_odds_chosen": 1.4882080554962158, + "log_odds_ratio": -0.40507811307907104, + "logits/chosen": -0.91796875, + "logits/rejected": -0.8207031488418579, + "logps/chosen": -0.664257824420929, + "logps/rejected": -1.7685546875, + "loss": 0.874, + "nll_loss": 0.831250011920929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06644286960363388, + "rewards/margins": 0.11036987602710724, + "rewards/rejected": -0.17680664360523224, + "step": 4830 + }, + { + "epoch": 0.3672787979966611, + "grad_norm": 2.3117284638928, + "learning_rate": 1.149919149152138e-06, + "log_odds_chosen": 1.329003930091858, + "log_odds_ratio": -0.41816407442092896, + "logits/chosen": -0.9652343988418579, + "logits/rejected": -0.866992175579071, + "logps/chosen": -0.6732422113418579, + "logps/rejected": -1.618749976158142, + "loss": 0.8594, + "nll_loss": 0.854687511920929, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06730957329273224, + "rewards/margins": 0.09454345703125, + "rewards/rejected": -0.16188964247703552, + "step": 4840 + }, + { + "epoch": 0.3680376384883897, + "grad_norm": 2.2721615455443147, + "learning_rate": 1.148733053788381e-06, + "log_odds_chosen": 1.3947265148162842, + "log_odds_ratio": -0.4109863340854645, + "logits/chosen": -0.950390636920929, + "logits/rejected": -0.8623046875, + "logps/chosen": -0.6473633050918579, + "logps/rejected": -1.59375, + "loss": 0.8642, + "nll_loss": 0.8150390386581421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.064697265625, + "rewards/margins": 0.09471587836742401, + "rewards/rejected": -0.159423828125, + "step": 4850 + }, + { + "epoch": 0.36879647898011836, + "grad_norm": 2.2251074013039718, + "learning_rate": 1.1475506210984938e-06, + "log_odds_chosen": 1.3505859375, + "log_odds_ratio": -0.40800780057907104, + "logits/chosen": -0.9847656488418579, + "logits/rejected": -0.8919922113418579, + "logps/chosen": -0.66650390625, + "logps/rejected": -1.58984375, + "loss": 0.835, + "nll_loss": 0.805468738079071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06657715141773224, + "rewards/margins": 0.09235839545726776, + "rewards/rejected": -0.15900878608226776, + "step": 4860 + }, + { + "epoch": 0.369555319471847, + "grad_norm": 1.9517990530874827, + "learning_rate": 1.1463718322705807e-06, + "log_odds_chosen": 1.147802710533142, + "log_odds_ratio": -0.43701171875, + "logits/chosen": -0.955273449420929, + "logits/rejected": -0.8939453363418579, + "logps/chosen": -0.6595703363418579, + "logps/rejected": -1.411718726158142, + "loss": 0.8602, + "nll_loss": 0.8057616949081421, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06586913764476776, + "rewards/margins": 0.07521972805261612, + "rewards/rejected": -0.14113768935203552, + "step": 4870 + }, + { + "epoch": 0.37031415996357564, + "grad_norm": 2.4321003116514985, + "learning_rate": 1.1451966686277364e-06, + "log_odds_chosen": 1.424414038658142, + "log_odds_ratio": -0.38740235567092896, + "logits/chosen": -0.9224609136581421, + "logits/rejected": -0.7994140386581421, + "logps/chosen": -0.6337890625, + "logps/rejected": -1.6320312023162842, + "loss": 0.859, + "nll_loss": 0.802734375, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.0633544921875, + "rewards/margins": 0.09980468451976776, + "rewards/rejected": -0.16313476860523224, + "step": 4880 + }, + { + "epoch": 0.3710730004553043, + "grad_norm": 2.3392296736339198, + "learning_rate": 1.1440251116268034e-06, + "log_odds_chosen": 1.458593726158142, + "log_odds_ratio": -0.4004882872104645, + "logits/chosen": -0.9595702886581421, + "logits/rejected": -0.8148437738418579, + "logps/chosen": -0.670703113079071, + "logps/rejected": -1.6671874523162842, + "loss": 0.8698, + "nll_loss": 0.810742199420929, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06708984076976776, + "rewards/margins": 0.09968261420726776, + "rewards/rejected": -0.166748046875, + "step": 4890 + }, + { + "epoch": 0.3718318409470329, + "grad_norm": 2.0434121903510243, + "learning_rate": 1.1428571428571428e-06, + "log_odds_chosen": 1.3869140148162842, + "log_odds_ratio": -0.425048828125, + "logits/chosen": -0.9623047113418579, + "logits/rejected": -0.8697265386581421, + "logps/chosen": -0.664843738079071, + "logps/rejected": -1.6328125, + "loss": 0.8433, + "nll_loss": 0.8255859613418579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06645508110523224, + "rewards/margins": 0.09688720852136612, + "rewards/rejected": -0.1634521484375, + "step": 4900 + }, + { + "epoch": 0.37259068143876156, + "grad_norm": 2.1097305955384544, + "learning_rate": 1.14169274403942e-06, + "log_odds_chosen": 1.365576148033142, + "log_odds_ratio": -0.4039550721645355, + "logits/chosen": -0.9173828363418579, + "logits/rejected": -0.8062499761581421, + "logps/chosen": -0.664257824420929, + "logps/rejected": -1.6240234375, + "loss": 0.836, + "nll_loss": 0.849804699420929, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06641845405101776, + "rewards/margins": 0.09599609673023224, + "rewards/rejected": -0.162353515625, + "step": 4910 + }, + { + "epoch": 0.3733495219304902, + "grad_norm": 2.096457299618778, + "learning_rate": 1.140531897024402e-06, + "log_odds_chosen": 1.4734375476837158, + "log_odds_ratio": -0.4014648497104645, + "logits/chosen": -0.982226550579071, + "logits/rejected": -0.8636718988418579, + "logps/chosen": -0.63330078125, + "logps/rejected": -1.6238281726837158, + "loss": 0.8636, + "nll_loss": 0.770703136920929, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06337890774011612, + "rewards/margins": 0.09904785454273224, + "rewards/rejected": -0.16240234673023224, + "step": 4920 + }, + { + "epoch": 0.37410836242221884, + "grad_norm": 2.625565690686019, + "learning_rate": 1.13937458379177e-06, + "log_odds_chosen": 1.322167992591858, + "log_odds_ratio": -0.45234376192092896, + "logits/chosen": -0.924609363079071, + "logits/rejected": -0.819140613079071, + "logps/chosen": -0.68115234375, + "logps/rejected": -1.6484375, + "loss": 0.856, + "nll_loss": 0.820996105670929, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06809081882238388, + "rewards/margins": 0.09653320163488388, + "rewards/rejected": -0.16477051377296448, + "step": 4930 + }, + { + "epoch": 0.3748672029139475, + "grad_norm": 2.203218780916155, + "learning_rate": 1.1382207864489444e-06, + "log_odds_chosen": 1.2156250476837158, + "log_odds_ratio": -0.43525391817092896, + "logits/chosen": -0.9537109136581421, + "logits/rejected": -0.828320324420929, + "logps/chosen": -0.624804675579071, + "logps/rejected": -1.423437476158142, + "loss": 0.8619, + "nll_loss": 0.8041015863418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06251220405101776, + "rewards/margins": 0.07990722358226776, + "rewards/rejected": -0.14243164658546448, + "step": 4940 + }, + { + "epoch": 0.3756260434056761, + "grad_norm": 2.7540164581142594, + "learning_rate": 1.1370704872299223e-06, + "log_odds_chosen": 1.2926757335662842, + "log_odds_ratio": -0.41874998807907104, + "logits/chosen": -0.9867187738418579, + "logits/rejected": -0.891406238079071, + "logps/chosen": -0.6659179925918579, + "logps/rejected": -1.556640625, + "loss": 0.8565, + "nll_loss": 0.809765636920929, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06658935546875, + "rewards/margins": 0.08905029296875, + "rewards/rejected": -0.15556640923023224, + "step": 4950 + }, + { + "epoch": 0.37638488389740477, + "grad_norm": 2.025269380602589, + "learning_rate": 1.1359236684941295e-06, + "log_odds_chosen": 1.349511742591858, + "log_odds_ratio": -0.41899412870407104, + "logits/chosen": -0.9544922113418579, + "logits/rejected": -0.8314453363418579, + "logps/chosen": -0.6807616949081421, + "logps/rejected": -1.603515625, + "loss": 0.8806, + "nll_loss": 0.882617175579071, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06804199516773224, + "rewards/margins": 0.09243164211511612, + "rewards/rejected": -0.16044922173023224, + "step": 4960 + }, + { + "epoch": 0.3771437243891334, + "grad_norm": 2.2485565467037207, + "learning_rate": 1.1347803127252839e-06, + "log_odds_chosen": 1.569238305091858, + "log_odds_ratio": -0.3885253965854645, + "logits/chosen": -1.0224609375, + "logits/rejected": -0.893359363079071, + "logps/chosen": -0.633496105670929, + "logps/rejected": -1.7433593273162842, + "loss": 0.8377, + "nll_loss": 0.8037109375, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06339111179113388, + "rewards/margins": 0.11101684719324112, + "rewards/rejected": -0.17434081435203552, + "step": 4970 + }, + { + "epoch": 0.37790256488086205, + "grad_norm": 2.165794505306308, + "learning_rate": 1.1336404025302715e-06, + "log_odds_chosen": 1.4425780773162842, + "log_odds_ratio": -0.40278321504592896, + "logits/chosen": -0.9632812738418579, + "logits/rejected": -0.84375, + "logps/chosen": -0.6806640625, + "logps/rejected": -1.71484375, + "loss": 0.8718, + "nll_loss": 0.8779296875, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06804199516773224, + "rewards/margins": 0.10341186821460724, + "rewards/rejected": -0.17160645127296448, + "step": 4980 + }, + { + "epoch": 0.3786614053725907, + "grad_norm": 2.164096377102248, + "learning_rate": 1.1325039206380352e-06, + "log_odds_chosen": 1.0779297351837158, + "log_odds_ratio": -0.47822266817092896, + "logits/chosen": -0.9638671875, + "logits/rejected": -0.884570300579071, + "logps/chosen": -0.641894519329071, + "logps/rejected": -1.3117187023162842, + "loss": 0.8722, + "nll_loss": 0.8316406011581421, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06416015326976776, + "rewards/margins": 0.0670166015625, + "rewards/rejected": -0.13124999403953552, + "step": 4990 + }, + { + "epoch": 0.3794202458643193, + "grad_norm": 2.4681602129978293, + "learning_rate": 1.131370849898476e-06, + "log_odds_chosen": 1.2675902843475342, + "log_odds_ratio": -0.4202636778354645, + "logits/chosen": -0.980273425579071, + "logits/rejected": -0.873046875, + "logps/chosen": -0.6693359613418579, + "logps/rejected": -1.5099608898162842, + "loss": 0.8537, + "nll_loss": 0.8560546636581421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06689453125, + "rewards/margins": 0.08423157036304474, + "rewards/rejected": -0.15097656846046448, + "step": 5000 + }, + { + "epoch": 0.3794202458643193, + "eval_log_odds_chosen": 1.0485382080078125, + "eval_log_odds_ratio": -0.5006263852119446, + "eval_logits/chosen": -0.8404242396354675, + "eval_logits/rejected": -0.757588803768158, + "eval_logps/chosen": -0.7668778300285339, + "eval_logps/rejected": -1.5307999849319458, + "eval_loss": 1.1802747249603271, + "eval_nll_loss": 1.1388089656829834, + "eval_rewards/accuracies": 0.7127641439437866, + "eval_rewards/chosen": -0.07668878138065338, + "eval_rewards/margins": 0.07640629261732101, + "eval_rewards/rejected": -0.15308316051959991, + "eval_runtime": 1322.2772, + "eval_samples_per_second": 71.001, + "eval_steps_per_second": 1.109, + "step": 5000 + }, + { + "epoch": 0.38017908635604797, + "grad_norm": 1.9817618134624426, + "learning_rate": 1.130241173281366e-06, + "log_odds_chosen": 1.358007788658142, + "log_odds_ratio": -0.4292968809604645, + "logits/chosen": -0.988476574420929, + "logits/rejected": -0.8824218511581421, + "logps/chosen": -0.626171886920929, + "logps/rejected": -1.5431640148162842, + "loss": 0.8706, + "nll_loss": 0.7916015386581421, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06263427436351776, + "rewards/margins": 0.09177245944738388, + "rewards/rejected": -0.15444335341453552, + "step": 5010 + }, + { + "epoch": 0.3809379268477766, + "grad_norm": 2.0887243367857313, + "learning_rate": 1.1291148738752732e-06, + "log_odds_chosen": 1.359375, + "log_odds_ratio": -0.43115234375, + "logits/chosen": -0.962890625, + "logits/rejected": -0.859375, + "logps/chosen": -0.662304699420929, + "logps/rejected": -1.6101562976837158, + "loss": 0.8595, + "nll_loss": 0.8021484613418579, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06625976413488388, + "rewards/margins": 0.09468384087085724, + "rewards/rejected": -0.16091307997703552, + "step": 5020 + }, + { + "epoch": 0.38169676733950525, + "grad_norm": 2.3221585209409144, + "learning_rate": 1.1279919348864981e-06, + "log_odds_chosen": 1.262109398841858, + "log_odds_ratio": -0.4559570252895355, + "logits/chosen": -0.986328125, + "logits/rejected": -0.874804675579071, + "logps/chosen": -0.684863269329071, + "logps/rejected": -1.5759766101837158, + "loss": 0.8661, + "nll_loss": 0.780468761920929, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06840820610523224, + "rewards/margins": 0.08903808891773224, + "rewards/rejected": -0.15749511122703552, + "step": 5030 + }, + { + "epoch": 0.3824556078312339, + "grad_norm": 2.4615333295489923, + "learning_rate": 1.126872339638022e-06, + "log_odds_chosen": 1.5412108898162842, + "log_odds_ratio": -0.3860839903354645, + "logits/chosen": -0.995898425579071, + "logits/rejected": -0.8568359613418579, + "logps/chosen": -0.6590820550918579, + "logps/rejected": -1.711328148841858, + "loss": 0.8506, + "nll_loss": 0.8023437261581421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06596679985523224, + "rewards/margins": 0.10516357421875, + "rewards/rejected": -0.17094726860523224, + "step": 5040 + }, + { + "epoch": 0.38321444832296253, + "grad_norm": 2.2087176039301113, + "learning_rate": 1.1257560715684668e-06, + "log_odds_chosen": 1.3230469226837158, + "log_odds_ratio": -0.3921875059604645, + "logits/chosen": -0.953906238079071, + "logits/rejected": -0.861523449420929, + "logps/chosen": -0.6551758050918579, + "logps/rejected": -1.529882788658142, + "loss": 0.8582, + "nll_loss": 0.8031250238418579, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06558837741613388, + "rewards/margins": 0.08754272758960724, + "rewards/rejected": -0.15302734076976776, + "step": 5050 + }, + { + "epoch": 0.38397328881469117, + "grad_norm": 2.57913366088348, + "learning_rate": 1.1246431142310665e-06, + "log_odds_chosen": 1.323144555091858, + "log_odds_ratio": -0.45341795682907104, + "logits/chosen": -0.992382824420929, + "logits/rejected": -0.9234374761581421, + "logps/chosen": -0.674511730670929, + "logps/rejected": -1.611718773841858, + "loss": 0.8318, + "nll_loss": 0.787890613079071, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06746826320886612, + "rewards/margins": 0.09369506686925888, + "rewards/rejected": -0.16118164360523224, + "step": 5060 + }, + { + "epoch": 0.3847321293064198, + "grad_norm": 2.867178072685699, + "learning_rate": 1.1235334512926484e-06, + "log_odds_chosen": 1.4119141101837158, + "log_odds_ratio": -0.44267576932907104, + "logits/chosen": -0.979296863079071, + "logits/rejected": -0.896484375, + "logps/chosen": -0.694531261920929, + "logps/rejected": -1.689843773841858, + "loss": 0.8531, + "nll_loss": 0.8343750238418579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06943359225988388, + "rewards/margins": 0.09953002631664276, + "rewards/rejected": -0.16896972060203552, + "step": 5070 + }, + { + "epoch": 0.38549096979814845, + "grad_norm": 2.4584468576457086, + "learning_rate": 1.1224270665326274e-06, + "log_odds_chosen": 1.535546898841858, + "log_odds_ratio": -0.3919433653354645, + "logits/chosen": -0.9908202886581421, + "logits/rejected": -0.886523425579071, + "logps/chosen": -0.650390625, + "logps/rejected": -1.6845703125, + "loss": 0.8546, + "nll_loss": 0.7789062261581421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.064971923828125, + "rewards/margins": 0.10341797024011612, + "rewards/rejected": -0.16840820014476776, + "step": 5080 + }, + { + "epoch": 0.3862498102898771, + "grad_norm": 2.286984065038291, + "learning_rate": 1.12132394384201e-06, + "log_odds_chosen": 1.4249999523162842, + "log_odds_ratio": -0.3865966796875, + "logits/chosen": -0.9921875, + "logits/rejected": -0.8822265863418579, + "logps/chosen": -0.658984363079071, + "logps/rejected": -1.6378905773162842, + "loss": 0.8675, + "nll_loss": 0.87841796875, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06590576469898224, + "rewards/margins": 0.09788818657398224, + "rewards/rejected": -0.16389159858226776, + "step": 5090 + }, + { + "epoch": 0.38700865078160573, + "grad_norm": 2.296460359770309, + "learning_rate": 1.1202240672224076e-06, + "log_odds_chosen": 1.221777319908142, + "log_odds_ratio": -0.4361816346645355, + "logits/chosen": -1.0087890625, + "logits/rejected": -0.908203125, + "logps/chosen": -0.698046863079071, + "logps/rejected": -1.5496094226837158, + "loss": 0.8566, + "nll_loss": 0.845703125, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06977538764476776, + "rewards/margins": 0.08516845852136612, + "rewards/rejected": -0.15493163466453552, + "step": 5100 + }, + { + "epoch": 0.3877674912733343, + "grad_norm": 2.539802054931241, + "learning_rate": 1.1191274207850654e-06, + "log_odds_chosen": 1.351953148841858, + "log_odds_ratio": -0.3960937559604645, + "logits/chosen": -0.970898449420929, + "logits/rejected": -0.8564453125, + "logps/chosen": -0.65478515625, + "logps/rejected": -1.5607421398162842, + "loss": 0.8639, + "nll_loss": 0.826367199420929, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06551513820886612, + "rewards/margins": 0.09067382663488388, + "rewards/rejected": -0.15610352158546448, + "step": 5110 + }, + { + "epoch": 0.38852633176506296, + "grad_norm": 1.9322585367595608, + "learning_rate": 1.1180339887498948e-06, + "log_odds_chosen": 1.215234398841858, + "log_odds_ratio": -0.4300781190395355, + "logits/chosen": -0.966992199420929, + "logits/rejected": -0.8843749761581421, + "logps/chosen": -0.6258789300918579, + "logps/rejected": -1.404687523841858, + "loss": 0.8496, + "nll_loss": 0.8500000238418579, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06259765475988388, + "rewards/margins": 0.07786254584789276, + "rewards/rejected": -0.14052733778953552, + "step": 5120 + }, + { + "epoch": 0.3892851722567916, + "grad_norm": 2.1834984733601615, + "learning_rate": 1.1169437554445213e-06, + "log_odds_chosen": 1.357666015625, + "log_odds_ratio": -0.4041503965854645, + "logits/chosen": -1.0068359375, + "logits/rejected": -0.9208984375, + "logps/chosen": -0.665332019329071, + "logps/rejected": -1.6199219226837158, + "loss": 0.8493, + "nll_loss": 0.813281238079071, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06654052436351776, + "rewards/margins": 0.09534148871898651, + "rewards/rejected": -0.16188964247703552, + "step": 5130 + }, + { + "epoch": 0.39004401274852024, + "grad_norm": 2.2641930579324483, + "learning_rate": 1.1158567053033413e-06, + "log_odds_chosen": 1.4916503429412842, + "log_odds_ratio": -0.39594727754592896, + "logits/chosen": -0.956835925579071, + "logits/rejected": -0.86328125, + "logps/chosen": -0.6742187738418579, + "logps/rejected": -1.7208983898162842, + "loss": 0.8622, + "nll_loss": 0.7606445550918579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06740722805261612, + "rewards/margins": 0.10485382378101349, + "rewards/rejected": -0.17209473252296448, + "step": 5140 + }, + { + "epoch": 0.3908028532402489, + "grad_norm": 2.757681266959225, + "learning_rate": 1.1147728228665882e-06, + "log_odds_chosen": 1.619531273841858, + "log_odds_ratio": -0.3351074159145355, + "logits/chosen": -1.031835913658142, + "logits/rejected": -0.901562511920929, + "logps/chosen": -0.58447265625, + "logps/rejected": -1.67578125, + "loss": 0.8401, + "nll_loss": 0.7567383050918579, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.05852050706744194, + "rewards/margins": 0.10905762016773224, + "rewards/rejected": -0.16745606064796448, + "step": 5150 + }, + { + "epoch": 0.3915616937319775, + "grad_norm": 2.1808989598699657, + "learning_rate": 1.1136920927794092e-06, + "log_odds_chosen": 1.546289086341858, + "log_odds_ratio": -0.3707519471645355, + "logits/chosen": -0.956250011920929, + "logits/rejected": -0.8568359613418579, + "logps/chosen": -0.635546863079071, + "logps/rejected": -1.701757788658142, + "loss": 0.8418, + "nll_loss": 0.804882824420929, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06348876655101776, + "rewards/margins": 0.10668029636144638, + "rewards/rejected": -0.17021484673023224, + "step": 5160 + }, + { + "epoch": 0.39232053422370616, + "grad_norm": 3.036900593990881, + "learning_rate": 1.1126144997909508e-06, + "log_odds_chosen": 1.5876953601837158, + "log_odds_ratio": -0.40166014432907104, + "logits/chosen": -1.02734375, + "logits/rejected": -0.907421886920929, + "logps/chosen": -0.640429675579071, + "logps/rejected": -1.763281226158142, + "loss": 0.8346, + "nll_loss": 0.832226574420929, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06407471001148224, + "rewards/margins": 0.11224365234375, + "rewards/rejected": -0.17631836235523224, + "step": 5170 + }, + { + "epoch": 0.3930793747154348, + "grad_norm": 2.437818281436921, + "learning_rate": 1.1115400287534568e-06, + "log_odds_chosen": 1.438574194908142, + "log_odds_ratio": -0.40478515625, + "logits/chosen": -0.925976574420929, + "logits/rejected": -0.8587890863418579, + "logps/chosen": -0.6607421636581421, + "logps/rejected": -1.7107422351837158, + "loss": 0.8706, + "nll_loss": 0.8599609136581421, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06611327826976776, + "rewards/margins": 0.10502929985523224, + "rewards/rejected": -0.171142578125, + "step": 5180 + }, + { + "epoch": 0.39383821520716344, + "grad_norm": 2.3360733001189313, + "learning_rate": 1.110468664621372e-06, + "log_odds_chosen": 1.1808593273162842, + "log_odds_ratio": -0.41166990995407104, + "logits/chosen": -0.9156249761581421, + "logits/rejected": -0.8248046636581421, + "logps/chosen": -0.662890613079071, + "logps/rejected": -1.4318358898162842, + "loss": 0.8526, + "nll_loss": 0.8492187261581421, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06633301079273224, + "rewards/margins": 0.0768638625741005, + "rewards/rejected": -0.1431884765625, + "step": 5190 + }, + { + "epoch": 0.3945970556988921, + "grad_norm": 2.241729585825133, + "learning_rate": 1.1094003924504583e-06, + "log_odds_chosen": 1.510156273841858, + "log_odds_ratio": -0.3543945252895355, + "logits/chosen": -0.962695300579071, + "logits/rejected": -0.832812488079071, + "logps/chosen": -0.6650390625, + "logps/rejected": -1.6951172351837158, + "loss": 0.8439, + "nll_loss": 0.7823241949081421, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.0665283203125, + "rewards/margins": 0.10301513969898224, + "rewards/rejected": -0.16958007216453552, + "step": 5200 + }, + { + "epoch": 0.3953558961906207, + "grad_norm": 2.351667347631347, + "learning_rate": 1.1083351973969191e-06, + "log_odds_chosen": 1.4570801258087158, + "log_odds_ratio": -0.397216796875, + "logits/chosen": -0.972460925579071, + "logits/rejected": -0.841015636920929, + "logps/chosen": -0.64404296875, + "logps/rejected": -1.672265648841858, + "loss": 0.826, + "nll_loss": 0.791015625, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06439208984375, + "rewards/margins": 0.10289306938648224, + "rewards/rejected": -0.1673583984375, + "step": 5210 + }, + { + "epoch": 0.39611473668234937, + "grad_norm": 2.390270358393584, + "learning_rate": 1.107273064716533e-06, + "log_odds_chosen": 1.48388671875, + "log_odds_ratio": -0.3770507872104645, + "logits/chosen": -0.9349609613418579, + "logits/rejected": -0.855273425579071, + "logps/chosen": -0.67138671875, + "logps/rejected": -1.67578125, + "loss": 0.8374, + "nll_loss": 0.7816406488418579, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06712646782398224, + "rewards/margins": 0.10045166313648224, + "rewards/rejected": -0.1676025390625, + "step": 5220 + }, + { + "epoch": 0.396873577174078, + "grad_norm": 2.554077173558235, + "learning_rate": 1.1062139797637962e-06, + "log_odds_chosen": 1.140527367591858, + "log_odds_ratio": -0.48124998807907104, + "logits/chosen": -0.966015636920929, + "logits/rejected": -0.837890625, + "logps/chosen": -0.660839855670929, + "logps/rejected": -1.4445312023162842, + "loss": 0.8497, + "nll_loss": 0.812695324420929, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06608887016773224, + "rewards/margins": 0.07825317233800888, + "rewards/rejected": -0.14423827826976776, + "step": 5230 + }, + { + "epoch": 0.39763241766580665, + "grad_norm": 2.455294602481025, + "learning_rate": 1.1051579279910751e-06, + "log_odds_chosen": 1.237548828125, + "log_odds_ratio": -0.4476074278354645, + "logits/chosen": -0.9859374761581421, + "logits/rejected": -0.86328125, + "logps/chosen": -0.6650390625, + "logps/rejected": -1.5402343273162842, + "loss": 0.8415, + "nll_loss": 0.80126953125, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.0665283203125, + "rewards/margins": 0.08743591606616974, + "rewards/rejected": -0.15388183295726776, + "step": 5240 + }, + { + "epoch": 0.3983912581575353, + "grad_norm": 2.0142702638341405, + "learning_rate": 1.1041048949477667e-06, + "log_odds_chosen": 1.4738280773162842, + "log_odds_ratio": -0.421630859375, + "logits/chosen": -0.9613281488418579, + "logits/rejected": -0.848925769329071, + "logps/chosen": -0.6913086175918579, + "logps/rejected": -1.7062499523162842, + "loss": 0.8602, + "nll_loss": 0.864453136920929, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06912841647863388, + "rewards/margins": 0.10150146484375, + "rewards/rejected": -0.17070312798023224, + "step": 5250 + }, + { + "epoch": 0.39915009864926393, + "grad_norm": 2.5825414735616294, + "learning_rate": 1.1030548662794673e-06, + "log_odds_chosen": 1.609765648841858, + "log_odds_ratio": -0.3705078065395355, + "logits/chosen": -0.9417968988418579, + "logits/rejected": -0.849609375, + "logps/chosen": -0.632617175579071, + "logps/rejected": -1.749609351158142, + "loss": 0.8321, + "nll_loss": 0.8251953125, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06322021782398224, + "rewards/margins": 0.11190185695886612, + "rewards/rejected": -0.17499999701976776, + "step": 5260 + }, + { + "epoch": 0.39990893914099257, + "grad_norm": 2.3958073804549107, + "learning_rate": 1.102007827727152e-06, + "log_odds_chosen": 1.192968726158142, + "log_odds_ratio": -0.461181640625, + "logits/chosen": -0.9130859375, + "logits/rejected": -0.821484386920929, + "logps/chosen": -0.652148425579071, + "logps/rejected": -1.4617187976837158, + "loss": 0.8474, + "nll_loss": 0.787304699420929, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06522216647863388, + "rewards/margins": 0.08099365234375, + "rewards/rejected": -0.14616699516773224, + "step": 5270 + }, + { + "epoch": 0.4006677796327212, + "grad_norm": 2.2048879056759705, + "learning_rate": 1.1009637651263607e-06, + "log_odds_chosen": 1.418554663658142, + "log_odds_ratio": -0.4303222596645355, + "logits/chosen": -0.9595702886581421, + "logits/rejected": -0.8666015863418579, + "logps/chosen": -0.721972644329071, + "logps/rejected": -1.7548828125, + "loss": 0.8406, + "nll_loss": 0.835742175579071, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.07216797024011612, + "rewards/margins": 0.103271484375, + "rewards/rejected": -0.17561034858226776, + "step": 5280 + }, + { + "epoch": 0.40142662012444985, + "grad_norm": 2.3839196143077186, + "learning_rate": 1.0999226644063927e-06, + "log_odds_chosen": 1.5810546875, + "log_odds_ratio": -0.41425782442092896, + "logits/chosen": -0.9673827886581421, + "logits/rejected": -0.876757800579071, + "logps/chosen": -0.678417980670929, + "logps/rejected": -1.767187476158142, + "loss": 0.8503, + "nll_loss": 0.7894531488418579, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06787109375, + "rewards/margins": 0.10891113430261612, + "rewards/rejected": -0.17683105170726776, + "step": 5290 + }, + { + "epoch": 0.4021854606161785, + "grad_norm": 2.5262186182515385, + "learning_rate": 1.0988845115895123e-06, + "log_odds_chosen": 1.275976538658142, + "log_odds_ratio": -0.41874998807907104, + "logits/chosen": -0.9677734375, + "logits/rejected": -0.8544921875, + "logps/chosen": -0.6668945550918579, + "logps/rejected": -1.554101586341858, + "loss": 0.8674, + "nll_loss": 0.8570312261581421, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06662597507238388, + "rewards/margins": 0.08864746242761612, + "rewards/rejected": -0.15534667670726776, + "step": 5300 + }, + { + "epoch": 0.40294430110790713, + "grad_norm": 2.451035118002672, + "learning_rate": 1.0978492927901574e-06, + "log_odds_chosen": 1.3566405773162842, + "log_odds_ratio": -0.39653319120407104, + "logits/chosen": -0.9673827886581421, + "logits/rejected": -0.87109375, + "logps/chosen": -0.642382800579071, + "logps/rejected": -1.5675780773162842, + "loss": 0.8693, + "nll_loss": 0.810742199420929, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06419678032398224, + "rewards/margins": 0.09238281100988388, + "rewards/rejected": -0.15671387314796448, + "step": 5310 + }, + { + "epoch": 0.4037031415996358, + "grad_norm": 2.236444996769135, + "learning_rate": 1.0968169942141634e-06, + "log_odds_chosen": 1.2775390148162842, + "log_odds_ratio": -0.4251953065395355, + "logits/chosen": -0.96484375, + "logits/rejected": -0.8701171875, + "logps/chosen": -0.684765636920929, + "logps/rejected": -1.572656273841858, + "loss": 0.8418, + "nll_loss": 0.7955077886581421, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06851806491613388, + "rewards/margins": 0.08876953274011612, + "rewards/rejected": -0.15729980170726776, + "step": 5320 + }, + { + "epoch": 0.4044619820913644, + "grad_norm": 2.0767066319513634, + "learning_rate": 1.0957876021579874e-06, + "log_odds_chosen": 1.3289062976837158, + "log_odds_ratio": -0.42060548067092896, + "logits/chosen": -0.9751952886581421, + "logits/rejected": -0.880078136920929, + "logps/chosen": -0.6249023675918579, + "logps/rejected": -1.520117163658142, + "loss": 0.8516, + "nll_loss": 0.802929699420929, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06256103515625, + "rewards/margins": 0.08948364108800888, + "rewards/rejected": -0.15207520127296448, + "step": 5330 + }, + { + "epoch": 0.40522082258309305, + "grad_norm": 1.9338751696332936, + "learning_rate": 1.0947611030079466e-06, + "log_odds_chosen": 1.228759765625, + "log_odds_ratio": -0.4403320252895355, + "logits/chosen": -0.993359386920929, + "logits/rejected": -0.8968750238418579, + "logps/chosen": -0.6595703363418579, + "logps/rejected": -1.5138671398162842, + "loss": 0.8461, + "nll_loss": 0.8099609613418579, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06594238430261612, + "rewards/margins": 0.08539581298828125, + "rewards/rejected": -0.15126952528953552, + "step": 5340 + }, + { + "epoch": 0.4059796630748217, + "grad_norm": 2.567395908968091, + "learning_rate": 1.0937374832394612e-06, + "log_odds_chosen": 1.610937476158142, + "log_odds_ratio": -0.37236326932907104, + "logits/chosen": -0.942187488079071, + "logits/rejected": -0.8658202886581421, + "logps/chosen": -0.7108398675918579, + "logps/rejected": -1.8488280773162842, + "loss": 0.848, + "nll_loss": 0.817675769329071, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.07108154147863388, + "rewards/margins": 0.11353759467601776, + "rewards/rejected": -0.18464355170726776, + "step": 5350 + }, + { + "epoch": 0.40673850356655034, + "grad_norm": 2.95176322615593, + "learning_rate": 1.092716729416306e-06, + "log_odds_chosen": 1.4933593273162842, + "log_odds_ratio": -0.399169921875, + "logits/chosen": -0.9750000238418579, + "logits/rejected": -0.8755859136581421, + "logps/chosen": -0.6742187738418579, + "logps/rejected": -1.7248046398162842, + "loss": 0.8302, + "nll_loss": 0.746289074420929, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06743164360523224, + "rewards/margins": 0.10502319037914276, + "rewards/rejected": -0.17238768935203552, + "step": 5360 + }, + { + "epoch": 0.407497344058279, + "grad_norm": 2.219558950331525, + "learning_rate": 1.0916988281898703e-06, + "log_odds_chosen": 1.5137939453125, + "log_odds_ratio": -0.43994140625, + "logits/chosen": -0.9740234613418579, + "logits/rejected": -0.837109386920929, + "logps/chosen": -0.635449230670929, + "logps/rejected": -1.719335913658142, + "loss": 0.8543, + "nll_loss": 0.811328113079071, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06352539360523224, + "rewards/margins": 0.10838699340820312, + "rewards/rejected": -0.17202147841453552, + "step": 5370 + }, + { + "epoch": 0.40825618455000756, + "grad_norm": 2.3407798939658866, + "learning_rate": 1.0906837662984237e-06, + "log_odds_chosen": 1.3464844226837158, + "log_odds_ratio": -0.41777342557907104, + "logits/chosen": -1.0304687023162842, + "logits/rejected": -0.9146484136581421, + "logps/chosen": -0.683886706829071, + "logps/rejected": -1.640234351158142, + "loss": 0.8303, + "nll_loss": 0.7933593988418579, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06838379055261612, + "rewards/margins": 0.09565429389476776, + "rewards/rejected": -0.16396483778953552, + "step": 5380 + }, + { + "epoch": 0.4090150250417362, + "grad_norm": 2.3276245562723905, + "learning_rate": 1.089671530566391e-06, + "log_odds_chosen": 1.0404052734375, + "log_odds_ratio": -0.5146484375, + "logits/chosen": -0.977343738079071, + "logits/rejected": -0.8521484136581421, + "logps/chosen": -0.726269543170929, + "logps/rejected": -1.465234398841858, + "loss": 0.8452, + "nll_loss": 0.826953113079071, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.07255859673023224, + "rewards/margins": 0.07400665432214737, + "rewards/rejected": -0.14645996689796448, + "step": 5390 + }, + { + "epoch": 0.40977386553346484, + "grad_norm": 2.0981773535508603, + "learning_rate": 1.0886621079036346e-06, + "log_odds_chosen": 1.3523437976837158, + "log_odds_ratio": -0.4051757752895355, + "logits/chosen": -0.9755859375, + "logits/rejected": -0.8773437738418579, + "logps/chosen": -0.68310546875, + "logps/rejected": -1.6330077648162842, + "loss": 0.8439, + "nll_loss": 0.788281261920929, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06827392429113388, + "rewards/margins": 0.09505005180835724, + "rewards/rejected": -0.1632080078125, + "step": 5400 + }, + { + "epoch": 0.4105327060251935, + "grad_norm": 2.450305687307705, + "learning_rate": 1.0876554853047417e-06, + "log_odds_chosen": 1.370703101158142, + "log_odds_ratio": -0.39628905057907104, + "logits/chosen": -0.9892578125, + "logits/rejected": -0.8949218988418579, + "logps/chosen": -0.6451171636581421, + "logps/rejected": -1.584570288658142, + "loss": 0.8483, + "nll_loss": 0.826953113079071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06447754055261612, + "rewards/margins": 0.09417724609375, + "rewards/rejected": -0.15861816704273224, + "step": 5410 + }, + { + "epoch": 0.4112915465169221, + "grad_norm": 2.2184697279127996, + "learning_rate": 1.0866516498483225e-06, + "log_odds_chosen": 1.385351538658142, + "log_odds_ratio": -0.39497071504592896, + "logits/chosen": -0.948046863079071, + "logits/rejected": -0.892382800579071, + "logps/chosen": -0.658984363079071, + "logps/rejected": -1.6228516101837158, + "loss": 0.8518, + "nll_loss": 0.7818359136581421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06588134914636612, + "rewards/margins": 0.09637451171875, + "rewards/rejected": -0.1622314453125, + "step": 5420 + }, + { + "epoch": 0.41205038700865076, + "grad_norm": 2.229851033600332, + "learning_rate": 1.0856505886963116e-06, + "log_odds_chosen": 1.2664062976837158, + "log_odds_ratio": -0.45341795682907104, + "logits/chosen": -0.953125, + "logits/rejected": -0.886523425579071, + "logps/chosen": -0.6996093988418579, + "logps/rejected": -1.5519530773162842, + "loss": 0.8465, + "nll_loss": 0.783398449420929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07000732421875, + "rewards/margins": 0.08521118015050888, + "rewards/rejected": -0.15524902939796448, + "step": 5430 + }, + { + "epoch": 0.4128092275003794, + "grad_norm": 2.713777376309095, + "learning_rate": 1.0846522890932808e-06, + "log_odds_chosen": 1.4894530773162842, + "log_odds_ratio": -0.3945556581020355, + "logits/chosen": -0.9527343511581421, + "logits/rejected": -0.816601574420929, + "logps/chosen": -0.6708984375, + "logps/rejected": -1.730859398841858, + "loss": 0.847, + "nll_loss": 0.8365234136581421, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06711425632238388, + "rewards/margins": 0.10590209811925888, + "rewards/rejected": -0.1729736328125, + "step": 5440 + }, + { + "epoch": 0.41356806799210805, + "grad_norm": 2.323029731035624, + "learning_rate": 1.0836567383657542e-06, + "log_odds_chosen": 1.615234375, + "log_odds_ratio": -0.34257811307907104, + "logits/chosen": -0.948046863079071, + "logits/rejected": -0.833984375, + "logps/chosen": -0.602832019329071, + "logps/rejected": -1.6828124523162842, + "loss": 0.8152, + "nll_loss": 0.7875000238418579, + "rewards/accuracies": 0.8687499761581421, + "rewards/chosen": -0.06026611477136612, + "rewards/margins": 0.10798339545726776, + "rewards/rejected": -0.16828612983226776, + "step": 5450 + }, + { + "epoch": 0.4143269084838367, + "grad_norm": 2.65671515741868, + "learning_rate": 1.0826639239215334e-06, + "log_odds_chosen": 1.4382812976837158, + "log_odds_ratio": -0.39111328125, + "logits/chosen": -0.9517577886581421, + "logits/rejected": -0.8427734375, + "logps/chosen": -0.6689453125, + "logps/rejected": -1.661523461341858, + "loss": 0.8571, + "nll_loss": 0.7779296636581421, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06689453125, + "rewards/margins": 0.0992431640625, + "rewards/rejected": -0.16616210341453552, + "step": 5460 + }, + { + "epoch": 0.4150857489755653, + "grad_norm": 2.232804110723734, + "learning_rate": 1.0816738332490292e-06, + "log_odds_chosen": 1.450585961341858, + "log_odds_ratio": -0.37749022245407104, + "logits/chosen": -0.953906238079071, + "logits/rejected": -0.822070300579071, + "logps/chosen": -0.6556640863418579, + "logps/rejected": -1.6687500476837158, + "loss": 0.8243, + "nll_loss": 0.7876952886581421, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06551513820886612, + "rewards/margins": 0.101318359375, + "rewards/rejected": -0.16689452528953552, + "step": 5470 + }, + { + "epoch": 0.41584458946729397, + "grad_norm": 8.267218952899983, + "learning_rate": 1.0806864539165982e-06, + "log_odds_chosen": 1.243749976158142, + "log_odds_ratio": -0.47314453125, + "logits/chosen": -0.961132824420929, + "logits/rejected": -0.899218738079071, + "logps/chosen": -0.652050793170929, + "logps/rejected": -1.510156273841858, + "loss": 0.87, + "nll_loss": 0.815136730670929, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06517334282398224, + "rewards/margins": 0.0858154296875, + "rewards/rejected": -0.15097656846046448, + "step": 5480 + }, + { + "epoch": 0.4166034299590226, + "grad_norm": 2.291167751112026, + "learning_rate": 1.0797017735718878e-06, + "log_odds_chosen": 1.391455054283142, + "log_odds_ratio": -0.42216795682907104, + "logits/chosen": -0.925000011920929, + "logits/rejected": -0.825390636920929, + "logps/chosen": -0.68359375, + "logps/rejected": -1.667382836341858, + "loss": 0.8574, + "nll_loss": 0.798828125, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06829833984375, + "rewards/margins": 0.09837951511144638, + "rewards/rejected": -0.166748046875, + "step": 5490 + }, + { + "epoch": 0.41736227045075125, + "grad_norm": 2.827968129506355, + "learning_rate": 1.0787197799411874e-06, + "log_odds_chosen": 1.484716773033142, + "log_odds_ratio": -0.376708984375, + "logits/chosen": -0.966992199420929, + "logits/rejected": -0.884960949420929, + "logps/chosen": -0.6605468988418579, + "logps/rejected": -1.694921851158142, + "loss": 0.8561, + "nll_loss": 0.7572265863418579, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06602783501148224, + "rewards/margins": 0.10356750339269638, + "rewards/rejected": -0.169677734375, + "step": 5500 + }, + { + "epoch": 0.4181211109424799, + "grad_norm": 3.141675362699091, + "learning_rate": 1.0777404608287846e-06, + "log_odds_chosen": 1.377343773841858, + "log_odds_ratio": -0.38818359375, + "logits/chosen": -0.9957031011581421, + "logits/rejected": -0.888476550579071, + "logps/chosen": -0.611621081829071, + "logps/rejected": -1.5291016101837158, + "loss": 0.8614, + "nll_loss": 0.754199206829071, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0611572265625, + "rewards/margins": 0.09177245944738388, + "rewards/rejected": -0.15290527045726776, + "step": 5510 + }, + { + "epoch": 0.41887995143420853, + "grad_norm": 2.2349534850143846, + "learning_rate": 1.0767638041163309e-06, + "log_odds_chosen": 1.5626952648162842, + "log_odds_ratio": -0.3944335877895355, + "logits/chosen": -0.927929699420929, + "logits/rejected": -0.852734386920929, + "logps/chosen": -0.6341797113418579, + "logps/rejected": -1.6746094226837158, + "loss": 0.8157, + "nll_loss": 0.7822265625, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06340332329273224, + "rewards/margins": 0.10406494140625, + "rewards/rejected": -0.16755370795726776, + "step": 5520 + }, + { + "epoch": 0.41963879192593717, + "grad_norm": 2.493059537614275, + "learning_rate": 1.0757897977622107e-06, + "log_odds_chosen": 1.316992163658142, + "log_odds_ratio": -0.4212402403354645, + "logits/chosen": -0.9488281011581421, + "logits/rejected": -0.837109386920929, + "logps/chosen": -0.635546863079071, + "logps/rejected": -1.512109398841858, + "loss": 0.8426, + "nll_loss": 0.796875, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06357421725988388, + "rewards/margins": 0.08762817084789276, + "rewards/rejected": -0.15117187798023224, + "step": 5530 + }, + { + "epoch": 0.4203976324176658, + "grad_norm": 2.32022031502086, + "learning_rate": 1.074818429800918e-06, + "log_odds_chosen": 1.42578125, + "log_odds_ratio": -0.41650390625, + "logits/chosen": -0.9419921636581421, + "logits/rejected": -0.835156261920929, + "logps/chosen": -0.667285144329071, + "logps/rejected": -1.6515624523162842, + "loss": 0.8429, + "nll_loss": 0.8384765386581421, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06676025688648224, + "rewards/margins": 0.09844970703125, + "rewards/rejected": -0.1651611328125, + "step": 5540 + }, + { + "epoch": 0.42115647290939445, + "grad_norm": 2.4153737572601583, + "learning_rate": 1.073849688342439e-06, + "log_odds_chosen": 1.49853515625, + "log_odds_ratio": -0.369140625, + "logits/chosen": -0.978515625, + "logits/rejected": -0.8662109375, + "logps/chosen": -0.6898437738418579, + "logps/rejected": -1.750390648841858, + "loss": 0.8571, + "nll_loss": 0.773632824420929, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06903076171875, + "rewards/margins": 0.1060791015625, + "rewards/rejected": -0.17514649033546448, + "step": 5550 + }, + { + "epoch": 0.4219153134011231, + "grad_norm": 2.1402853638188075, + "learning_rate": 1.07288356157164e-06, + "log_odds_chosen": 1.2134277820587158, + "log_odds_ratio": -0.4410644471645355, + "logits/chosen": -0.933398425579071, + "logits/rejected": -0.8648437261581421, + "logps/chosen": -0.661328136920929, + "logps/rejected": -1.4675781726837158, + "loss": 0.8474, + "nll_loss": 0.796679675579071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06613769382238388, + "rewards/margins": 0.08057556301355362, + "rewards/rejected": -0.14677734673023224, + "step": 5560 + }, + { + "epoch": 0.42267415389285173, + "grad_norm": 2.1941589650936417, + "learning_rate": 1.0719200377476648e-06, + "log_odds_chosen": 1.100195288658142, + "log_odds_ratio": -0.4546875059604645, + "logits/chosen": -0.9546874761581421, + "logits/rejected": -0.8228515386581421, + "logps/chosen": -0.67333984375, + "logps/rejected": -1.409570336341858, + "loss": 0.852, + "nll_loss": 0.7875000238418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06734619289636612, + "rewards/margins": 0.07369232177734375, + "rewards/rejected": -0.14104004204273224, + "step": 5570 + }, + { + "epoch": 0.4234329943845804, + "grad_norm": 2.284806569576881, + "learning_rate": 1.0709591052033317e-06, + "log_odds_chosen": 1.2415039539337158, + "log_odds_ratio": -0.4207519590854645, + "logits/chosen": -0.925000011920929, + "logits/rejected": -0.7876952886581421, + "logps/chosen": -0.6590820550918579, + "logps/rejected": -1.4812500476837158, + "loss": 0.8272, + "nll_loss": 0.8017578125, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06593017280101776, + "rewards/margins": 0.082275390625, + "rewards/rejected": -0.14814452826976776, + "step": 5580 + }, + { + "epoch": 0.424191834876309, + "grad_norm": 2.6557510914633222, + "learning_rate": 1.0700007523445435e-06, + "log_odds_chosen": 1.594335913658142, + "log_odds_ratio": -0.37128907442092896, + "logits/chosen": -0.970703125, + "logits/rejected": -0.8941406011581421, + "logps/chosen": -0.6465820074081421, + "logps/rejected": -1.769921898841858, + "loss": 0.8534, + "nll_loss": 0.8013671636581421, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06468506157398224, + "rewards/margins": 0.11247558891773224, + "rewards/rejected": -0.17707519233226776, + "step": 5590 + }, + { + "epoch": 0.42495067536803766, + "grad_norm": 2.291232417510915, + "learning_rate": 1.0690449676496976e-06, + "log_odds_chosen": 1.4367187023162842, + "log_odds_ratio": -0.3921875059604645, + "logits/chosen": -0.951367199420929, + "logits/rejected": -0.8363281488418579, + "logps/chosen": -0.637988269329071, + "logps/rejected": -1.618749976158142, + "loss": 0.823, + "nll_loss": 0.746777355670929, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.0638427734375, + "rewards/margins": 0.097991943359375, + "rewards/rejected": -0.1617431640625, + "step": 5600 + }, + { + "epoch": 0.4257095158597663, + "grad_norm": 3.146253786769614, + "learning_rate": 1.0680917396691054e-06, + "log_odds_chosen": 1.478784203529358, + "log_odds_ratio": -0.40986329317092896, + "logits/chosen": -0.9580078125, + "logits/rejected": -0.8285156488418579, + "logps/chosen": -0.6039062738418579, + "logps/rejected": -1.641210913658142, + "loss": 0.8523, + "nll_loss": 0.8119140863418579, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06040038913488388, + "rewards/margins": 0.10360107570886612, + "rewards/rejected": -0.16403809189796448, + "step": 5610 + }, + { + "epoch": 0.42646835635149494, + "grad_norm": 2.4839220165258062, + "learning_rate": 1.0671410570244164e-06, + "log_odds_chosen": 1.4324219226837158, + "log_odds_ratio": -0.40996092557907104, + "logits/chosen": -0.8941406011581421, + "logits/rejected": -0.8042968511581421, + "logps/chosen": -0.6836913824081421, + "logps/rejected": -1.702734351158142, + "loss": 0.8397, + "nll_loss": 0.8467773199081421, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06838379055261612, + "rewards/margins": 0.10189209133386612, + "rewards/rejected": -0.17045898735523224, + "step": 5620 + }, + { + "epoch": 0.4272271968432236, + "grad_norm": 2.3854633770220968, + "learning_rate": 1.0661929084080466e-06, + "log_odds_chosen": 1.4597656726837158, + "log_odds_ratio": -0.40644532442092896, + "logits/chosen": -0.9105468988418579, + "logits/rejected": -0.852734386920929, + "logps/chosen": -0.6836913824081421, + "logps/rejected": -1.7257812023162842, + "loss": 0.8363, + "nll_loss": 0.798828125, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06840820610523224, + "rewards/margins": 0.10405273735523224, + "rewards/rejected": -0.17253418266773224, + "step": 5630 + }, + { + "epoch": 0.4279860373349522, + "grad_norm": 2.618874726406942, + "learning_rate": 1.0652472825826149e-06, + "log_odds_chosen": 1.5724608898162842, + "log_odds_ratio": -0.3803955018520355, + "logits/chosen": -0.9599609375, + "logits/rejected": -0.854687511920929, + "logps/chosen": -0.616503894329071, + "logps/rejected": -1.720703125, + "loss": 0.8354, + "nll_loss": 0.762499988079071, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06162109225988388, + "rewards/margins": 0.11040039360523224, + "rewards/rejected": -0.17207030951976776, + "step": 5640 + }, + { + "epoch": 0.42874487782668086, + "grad_norm": 2.9664040671175362, + "learning_rate": 1.0643041683803828e-06, + "log_odds_chosen": 1.3095703125, + "log_odds_ratio": -0.4390625059604645, + "logits/chosen": -0.9208984375, + "logits/rejected": -0.861328125, + "logps/chosen": -0.63720703125, + "logps/rejected": -1.5322265625, + "loss": 0.8451, + "nll_loss": 0.792773425579071, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06373290717601776, + "rewards/margins": 0.08945312350988388, + "rewards/rejected": -0.15314941108226776, + "step": 5650 + }, + { + "epoch": 0.42950371831840944, + "grad_norm": 2.192033753033841, + "learning_rate": 1.063363554702701e-06, + "log_odds_chosen": 1.289282202720642, + "log_odds_ratio": -0.4020019471645355, + "logits/chosen": -0.9697265625, + "logits/rejected": -0.859570324420929, + "logps/chosen": -0.639941394329071, + "logps/rejected": -1.49609375, + "loss": 0.8336, + "nll_loss": 0.766894519329071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06397704780101776, + "rewards/margins": 0.08570556342601776, + "rewards/rejected": -0.149658203125, + "step": 5660 + }, + { + "epoch": 0.4302625588101381, + "grad_norm": 2.3086066108515944, + "learning_rate": 1.0624254305194609e-06, + "log_odds_chosen": 1.379296898841858, + "log_odds_ratio": -0.4307617247104645, + "logits/chosen": -0.9359375238418579, + "logits/rejected": -0.810546875, + "logps/chosen": -0.6513671875, + "logps/rejected": -1.6160156726837158, + "loss": 0.8531, + "nll_loss": 0.800585925579071, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06517334282398224, + "rewards/margins": 0.09659423679113388, + "rewards/rejected": -0.16169433295726776, + "step": 5670 + }, + { + "epoch": 0.4310213993018667, + "grad_norm": 2.064949707230211, + "learning_rate": 1.0614897848685505e-06, + "log_odds_chosen": 1.7146484851837158, + "log_odds_ratio": -0.36894530057907104, + "logits/chosen": -0.953320324420929, + "logits/rejected": -0.887890636920929, + "logps/chosen": -0.686816394329071, + "logps/rejected": -1.9425780773162842, + "loss": 0.8205, + "nll_loss": 0.8172851800918579, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06867675483226776, + "rewards/margins": 0.12564697861671448, + "rewards/rejected": -0.19418945908546448, + "step": 5680 + }, + { + "epoch": 0.43178023979359537, + "grad_norm": 2.651212937316763, + "learning_rate": 1.0605566068553173e-06, + "log_odds_chosen": 1.5587890148162842, + "log_odds_ratio": -0.38862305879592896, + "logits/chosen": -0.927734375, + "logits/rejected": -0.8412109613418579, + "logps/chosen": -0.6478515863418579, + "logps/rejected": -1.733984351158142, + "loss": 0.8239, + "nll_loss": 0.7591797113418579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06478271633386612, + "rewards/margins": 0.10858154296875, + "rewards/rejected": -0.17329101264476776, + "step": 5690 + }, + { + "epoch": 0.432539080285324, + "grad_norm": 2.743960824799714, + "learning_rate": 1.0596258856520351e-06, + "log_odds_chosen": 1.4599609375, + "log_odds_ratio": -0.414306640625, + "logits/chosen": -0.9634765386581421, + "logits/rejected": -0.8667968511581421, + "logps/chosen": -0.645214855670929, + "logps/rejected": -1.668359398841858, + "loss": 0.804, + "nll_loss": 0.743359386920929, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06452636420726776, + "rewards/margins": 0.10244140774011612, + "rewards/rejected": -0.1669921875, + "step": 5700 + }, + { + "epoch": 0.43329792077705265, + "grad_norm": 2.460484764762945, + "learning_rate": 1.0586976104973764e-06, + "log_odds_chosen": 1.5458984375, + "log_odds_ratio": -0.373291015625, + "logits/chosen": -0.994335949420929, + "logits/rejected": -0.8560546636581421, + "logps/chosen": -0.619335949420929, + "logps/rejected": -1.717382788658142, + "loss": 0.823, + "nll_loss": 0.7523437738418579, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06191406399011612, + "rewards/margins": 0.10981445014476776, + "rewards/rejected": -0.17172852158546448, + "step": 5710 + }, + { + "epoch": 0.4340567612687813, + "grad_norm": 2.194192369645987, + "learning_rate": 1.05777177069589e-06, + "log_odds_chosen": 1.3621094226837158, + "log_odds_ratio": -0.41132813692092896, + "logits/chosen": -0.971484363079071, + "logits/rejected": -0.8695312738418579, + "logps/chosen": -0.650195300579071, + "logps/rejected": -1.5964844226837158, + "loss": 0.8224, + "nll_loss": 0.8091796636581421, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06502685695886612, + "rewards/margins": 0.09459228813648224, + "rewards/rejected": -0.15961913764476776, + "step": 5720 + }, + { + "epoch": 0.43481560176050993, + "grad_norm": 2.3534569396435163, + "learning_rate": 1.0568483556174834e-06, + "log_odds_chosen": 1.36181640625, + "log_odds_ratio": -0.37382811307907104, + "logits/chosen": -0.916796863079071, + "logits/rejected": -0.820117175579071, + "logps/chosen": -0.611035168170929, + "logps/rejected": -1.4787108898162842, + "loss": 0.8343, + "nll_loss": 0.780566394329071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06107177585363388, + "rewards/margins": 0.08672485500574112, + "rewards/rejected": -0.14780274033546448, + "step": 5730 + }, + { + "epoch": 0.43557444225223857, + "grad_norm": 2.5341264324790886, + "learning_rate": 1.055927354696909e-06, + "log_odds_chosen": 1.527734398841858, + "log_odds_ratio": -0.40424805879592896, + "logits/chosen": -0.9203125238418579, + "logits/rejected": -0.8306640386581421, + "logps/chosen": -0.6845703125, + "logps/rejected": -1.749609351158142, + "loss": 0.8199, + "nll_loss": 0.8037109375, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06845702975988388, + "rewards/margins": 0.10635986179113388, + "rewards/rejected": -0.17485351860523224, + "step": 5740 + }, + { + "epoch": 0.4363332827439672, + "grad_norm": 2.5400347892221515, + "learning_rate": 1.0550087574332592e-06, + "log_odds_chosen": 1.577734351158142, + "log_odds_ratio": -0.3924804627895355, + "logits/chosen": -0.9095703363418579, + "logits/rejected": -0.828125, + "logps/chosen": -0.6397460699081421, + "logps/rejected": -1.7228515148162842, + "loss": 0.837, + "nll_loss": 0.8037109375, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06395263969898224, + "rewards/margins": 0.10823974758386612, + "rewards/rejected": -0.17226561903953552, + "step": 5750 + }, + { + "epoch": 0.43709212323569585, + "grad_norm": 2.591562426503811, + "learning_rate": 1.0540925533894598e-06, + "log_odds_chosen": 1.265649437904358, + "log_odds_ratio": -0.42963868379592896, + "logits/chosen": -0.980273425579071, + "logits/rejected": -0.8765624761581421, + "logps/chosen": -0.668164074420929, + "logps/rejected": -1.508203148841858, + "loss": 0.8458, + "nll_loss": 0.8193359375, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.0667724609375, + "rewards/margins": 0.08410415798425674, + "rewards/rejected": -0.15097656846046448, + "step": 5760 + }, + { + "epoch": 0.4378509637274245, + "grad_norm": 2.451814486576127, + "learning_rate": 1.053178732191775e-06, + "log_odds_chosen": 1.4832031726837158, + "log_odds_ratio": -0.3822265565395355, + "logits/chosen": -0.9476562738418579, + "logits/rejected": -0.836132824420929, + "logps/chosen": -0.662109375, + "logps/rejected": -1.67578125, + "loss": 0.8261, + "nll_loss": 0.8193359375, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.066162109375, + "rewards/margins": 0.10148315131664276, + "rewards/rejected": -0.1676025390625, + "step": 5770 + }, + { + "epoch": 0.43860980421915313, + "grad_norm": 2.3419467810243377, + "learning_rate": 1.0522672835293127e-06, + "log_odds_chosen": 1.386328101158142, + "log_odds_ratio": -0.41874998807907104, + "logits/chosen": -0.9398437738418579, + "logits/rejected": -0.8355468511581421, + "logps/chosen": -0.6703125238418579, + "logps/rejected": -1.632226586341858, + "loss": 0.8353, + "nll_loss": 0.8271484375, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06697998195886612, + "rewards/margins": 0.09612426906824112, + "rewards/rejected": -0.16318359971046448, + "step": 5780 + }, + { + "epoch": 0.4393686447108818, + "grad_norm": 2.515846614172588, + "learning_rate": 1.0513581971535365e-06, + "log_odds_chosen": 1.548193335533142, + "log_odds_ratio": -0.3951171934604645, + "logits/chosen": -0.9283202886581421, + "logits/rejected": -0.8158203363418579, + "logps/chosen": -0.65869140625, + "logps/rejected": -1.749609351158142, + "loss": 0.8329, + "nll_loss": 0.794921875, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06594238430261612, + "rewards/margins": 0.10897521674633026, + "rewards/rejected": -0.175048828125, + "step": 5790 + }, + { + "epoch": 0.4401274852026104, + "grad_norm": 2.156252240229296, + "learning_rate": 1.0504514628777803e-06, + "log_odds_chosen": 1.318872094154358, + "log_odds_ratio": -0.42827147245407104, + "logits/chosen": -0.949414074420929, + "logits/rejected": -0.7906249761581421, + "logps/chosen": -0.649707019329071, + "logps/rejected": -1.53515625, + "loss": 0.8335, + "nll_loss": 0.8270508050918579, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06499023735523224, + "rewards/margins": 0.088653564453125, + "rewards/rejected": -0.15366211533546448, + "step": 5800 + }, + { + "epoch": 0.44088632569433905, + "grad_norm": 2.4786714654919746, + "learning_rate": 1.0495470705767713e-06, + "log_odds_chosen": 1.6251952648162842, + "log_odds_ratio": -0.3487792909145355, + "logits/chosen": -0.997851550579071, + "logits/rejected": -0.8681640625, + "logps/chosen": -0.653515636920929, + "logps/rejected": -1.7648437023162842, + "loss": 0.8313, + "nll_loss": 0.7705078125, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06534423679113388, + "rewards/margins": 0.11101074516773224, + "rewards/rejected": -0.17646484076976776, + "step": 5810 + }, + { + "epoch": 0.4416451661860677, + "grad_norm": 2.4398295811022597, + "learning_rate": 1.0486450101861527e-06, + "log_odds_chosen": 1.4000976085662842, + "log_odds_ratio": -0.426513671875, + "logits/chosen": -0.969531238079071, + "logits/rejected": -0.8570312261581421, + "logps/chosen": -0.6625000238418579, + "logps/rejected": -1.6564452648162842, + "loss": 0.8354, + "nll_loss": 0.797070324420929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06633301079273224, + "rewards/margins": 0.09940185397863388, + "rewards/rejected": -0.16567382216453552, + "step": 5820 + }, + { + "epoch": 0.44240400667779634, + "grad_norm": 2.282514156983795, + "learning_rate": 1.0477452717020143e-06, + "log_odds_chosen": 1.3572266101837158, + "log_odds_ratio": -0.44023436307907104, + "logits/chosen": -0.9673827886581421, + "logits/rejected": -0.871289074420929, + "logps/chosen": -0.682324230670929, + "logps/rejected": -1.592187523841858, + "loss": 0.8384, + "nll_loss": 0.8333984613418579, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06824950873851776, + "rewards/margins": 0.09095458686351776, + "rewards/rejected": -0.1591796875, + "step": 5830 + }, + { + "epoch": 0.443162847169525, + "grad_norm": 7.328556810853576, + "learning_rate": 1.0468478451804272e-06, + "log_odds_chosen": 1.4638671875, + "log_odds_ratio": -0.39252930879592896, + "logits/chosen": -0.9339843988418579, + "logits/rejected": -0.7919921875, + "logps/chosen": -0.6328125, + "logps/rejected": -1.668554663658142, + "loss": 0.8096, + "nll_loss": 0.7823241949081421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06330566108226776, + "rewards/margins": 0.1036376953125, + "rewards/rejected": -0.16689452528953552, + "step": 5840 + }, + { + "epoch": 0.4439216876612536, + "grad_norm": 2.336676814883782, + "learning_rate": 1.0459527207369814e-06, + "log_odds_chosen": 1.488867163658142, + "log_odds_ratio": -0.40791016817092896, + "logits/chosen": -0.8779296875, + "logits/rejected": -0.8070312738418579, + "logps/chosen": -0.623242199420929, + "logps/rejected": -1.6306641101837158, + "loss": 0.8108, + "nll_loss": 0.71484375, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06234131008386612, + "rewards/margins": 0.10065917670726776, + "rewards/rejected": -0.16298827528953552, + "step": 5850 + }, + { + "epoch": 0.44468052815298226, + "grad_norm": 2.724913929783163, + "learning_rate": 1.0450598885463281e-06, + "log_odds_chosen": 1.1145508289337158, + "log_odds_ratio": -0.46943360567092896, + "logits/chosen": -0.929492175579071, + "logits/rejected": -0.821093738079071, + "logps/chosen": -0.66357421875, + "logps/rejected": -1.383203148841858, + "loss": 0.8195, + "nll_loss": 0.754101574420929, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06639404594898224, + "rewards/margins": 0.07197265326976776, + "rewards/rejected": -0.13837890326976776, + "step": 5860 + }, + { + "epoch": 0.4454393686447109, + "grad_norm": 2.3791532169873104, + "learning_rate": 1.0441693388417282e-06, + "log_odds_chosen": 1.5329101085662842, + "log_odds_ratio": -0.41850584745407104, + "logits/chosen": -0.930468738079071, + "logits/rejected": -0.8050781488418579, + "logps/chosen": -0.6905273199081421, + "logps/rejected": -1.765234351158142, + "loss": 0.8159, + "nll_loss": 0.76953125, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06901855766773224, + "rewards/margins": 0.10758056491613388, + "rewards/rejected": -0.17673340439796448, + "step": 5870 + }, + { + "epoch": 0.44619820913643954, + "grad_norm": 2.5555748035487507, + "learning_rate": 1.0432810619146023e-06, + "log_odds_chosen": 1.203222632408142, + "log_odds_ratio": -0.46269530057907104, + "logits/chosen": -0.9208984375, + "logits/rejected": -0.8460937738418579, + "logps/chosen": -0.67431640625, + "logps/rejected": -1.519921898841858, + "loss": 0.8314, + "nll_loss": 0.8060547113418579, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06743164360523224, + "rewards/margins": 0.08454589545726776, + "rewards/rejected": -0.15202637016773224, + "step": 5880 + }, + { + "epoch": 0.4469570496281682, + "grad_norm": 2.1543997326364472, + "learning_rate": 1.042395048114086e-06, + "log_odds_chosen": 1.38671875, + "log_odds_ratio": -0.39311522245407104, + "logits/chosen": -0.891796886920929, + "logits/rejected": -0.8037109375, + "logps/chosen": -0.644238293170929, + "logps/rejected": -1.60546875, + "loss": 0.8316, + "nll_loss": 0.776562511920929, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06442870944738388, + "rewards/margins": 0.09611816704273224, + "rewards/rejected": -0.16044922173023224, + "step": 5890 + }, + { + "epoch": 0.4477158901198968, + "grad_norm": 2.609080080996203, + "learning_rate": 1.041511287846591e-06, + "log_odds_chosen": 1.197656273841858, + "log_odds_ratio": -0.4347167909145355, + "logits/chosen": -0.9453125, + "logits/rejected": -0.857617199420929, + "logps/chosen": -0.6744140386581421, + "logps/rejected": -1.483984351158142, + "loss": 0.8346, + "nll_loss": 0.81640625, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06744384765625, + "rewards/margins": 0.08093567192554474, + "rewards/rejected": -0.14841309189796448, + "step": 5900 + }, + { + "epoch": 0.44847473061162546, + "grad_norm": 2.4744863652963613, + "learning_rate": 1.0406297715753674e-06, + "log_odds_chosen": 1.3464844226837158, + "log_odds_ratio": -0.40361326932907104, + "logits/chosen": -0.946093738079071, + "logits/rejected": -0.834765613079071, + "logps/chosen": -0.664843738079071, + "logps/rejected": -1.5847656726837158, + "loss": 0.8265, + "nll_loss": 0.7710937261581421, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06647948920726776, + "rewards/margins": 0.092041015625, + "rewards/rejected": -0.1585693359375, + "step": 5910 + }, + { + "epoch": 0.4492335711033541, + "grad_norm": 2.6354067523710953, + "learning_rate": 1.0397504898200726e-06, + "log_odds_chosen": 1.32861328125, + "log_odds_ratio": -0.43876951932907104, + "logits/chosen": -0.9593750238418579, + "logits/rejected": -0.8824218511581421, + "logps/chosen": -0.685351550579071, + "logps/rejected": -1.609765648841858, + "loss": 0.8354, + "nll_loss": 0.803515613079071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06851806491613388, + "rewards/margins": 0.09244994819164276, + "rewards/rejected": -0.1611328125, + "step": 5920 + }, + { + "epoch": 0.4499924115950827, + "grad_norm": 2.749019246142902, + "learning_rate": 1.0388734331563415e-06, + "log_odds_chosen": 1.34130859375, + "log_odds_ratio": -0.43291014432907104, + "logits/chosen": -0.8763672113418579, + "logits/rejected": -0.7671874761581421, + "logps/chosen": -0.6939452886581421, + "logps/rejected": -1.6042969226837158, + "loss": 0.8246, + "nll_loss": 0.7939453125, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06938476860523224, + "rewards/margins": 0.09111938625574112, + "rewards/rejected": -0.16062012314796448, + "step": 5930 + }, + { + "epoch": 0.4507512520868113, + "grad_norm": 2.3947291009117047, + "learning_rate": 1.037998592215364e-06, + "log_odds_chosen": 1.4220702648162842, + "log_odds_ratio": -0.4090820252895355, + "logits/chosen": -0.920703113079071, + "logits/rejected": -0.822070300579071, + "logps/chosen": -0.6778320074081421, + "logps/rejected": -1.656640648841858, + "loss": 0.8298, + "nll_loss": 0.80859375, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06779785454273224, + "rewards/margins": 0.09788818657398224, + "rewards/rejected": -0.16569824516773224, + "step": 5940 + }, + { + "epoch": 0.45151009257853997, + "grad_norm": 2.619990350526729, + "learning_rate": 1.037125957683463e-06, + "log_odds_chosen": 1.7410156726837158, + "log_odds_ratio": -0.347900390625, + "logits/chosen": -0.9263671636581421, + "logits/rejected": -0.7876952886581421, + "logps/chosen": -0.65771484375, + "logps/rejected": -1.9148437976837158, + "loss": 0.8214, + "nll_loss": 0.7865234613418579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06575927883386612, + "rewards/margins": 0.12595215439796448, + "rewards/rejected": -0.191650390625, + "step": 5950 + }, + { + "epoch": 0.4522689330702686, + "grad_norm": 2.257524214102477, + "learning_rate": 1.0362555203016794e-06, + "log_odds_chosen": 1.3131835460662842, + "log_odds_ratio": -0.448974609375, + "logits/chosen": -0.959179699420929, + "logits/rejected": -0.878125011920929, + "logps/chosen": -0.685351550579071, + "logps/rejected": -1.6082031726837158, + "loss": 0.8043, + "nll_loss": 0.728320300579071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06856689602136612, + "rewards/margins": 0.09229431301355362, + "rewards/rejected": -0.1607666015625, + "step": 5960 + }, + { + "epoch": 0.45302777356199725, + "grad_norm": 2.4508968305713505, + "learning_rate": 1.035387270865359e-06, + "log_odds_chosen": 1.4972655773162842, + "log_odds_ratio": -0.3984375, + "logits/chosen": -0.9361327886581421, + "logits/rejected": -0.811328113079071, + "logps/chosen": -0.6631835699081421, + "logps/rejected": -1.7414062023162842, + "loss": 0.8044, + "nll_loss": 0.7499023675918579, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06630859524011612, + "rewards/margins": 0.10785522311925888, + "rewards/rejected": -0.17412109673023224, + "step": 5970 + }, + { + "epoch": 0.4537866140537259, + "grad_norm": 2.478417935322954, + "learning_rate": 1.0345212002237434e-06, + "log_odds_chosen": 1.457421898841858, + "log_odds_ratio": -0.3685058653354645, + "logits/chosen": -0.9369140863418579, + "logits/rejected": -0.799609363079071, + "logps/chosen": -0.6485351324081421, + "logps/rejected": -1.6316406726837158, + "loss": 0.8093, + "nll_loss": 0.786425769329071, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.06480713188648224, + "rewards/margins": 0.09834595024585724, + "rewards/rejected": -0.16323241591453552, + "step": 5980 + }, + { + "epoch": 0.45454545454545453, + "grad_norm": 3.1981452692493337, + "learning_rate": 1.0336572992795644e-06, + "log_odds_chosen": 1.35888671875, + "log_odds_ratio": -0.4383789002895355, + "logits/chosen": -0.9242187738418579, + "logits/rejected": -0.800976574420929, + "logps/chosen": -0.684765636920929, + "logps/rejected": -1.6574218273162842, + "loss": 0.8072, + "nll_loss": 0.788281261920929, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06853027641773224, + "rewards/margins": 0.0973663330078125, + "rewards/rejected": -0.1658935546875, + "step": 5990 + }, + { + "epoch": 0.45530429503718317, + "grad_norm": 2.532856792619757, + "learning_rate": 1.0327955589886444e-06, + "log_odds_chosen": 1.615625023841858, + "log_odds_ratio": -0.37158203125, + "logits/chosen": -0.939453125, + "logits/rejected": -0.814453125, + "logps/chosen": -0.6898437738418579, + "logps/rejected": -1.8449218273162842, + "loss": 0.8198, + "nll_loss": 0.8472656011581421, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.0689697265625, + "rewards/margins": 0.11558838188648224, + "rewards/rejected": -0.18459472060203552, + "step": 6000 + }, + { + "epoch": 0.4560631355289118, + "grad_norm": 2.3624301181871474, + "learning_rate": 1.0319359703594971e-06, + "log_odds_chosen": 1.311914086341858, + "log_odds_ratio": -0.4254394471645355, + "logits/chosen": -0.9566406011581421, + "logits/rejected": -0.818164050579071, + "logps/chosen": -0.666015625, + "logps/rejected": -1.562109351158142, + "loss": 0.8213, + "nll_loss": 0.7979491949081421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06658935546875, + "rewards/margins": 0.08953857421875, + "rewards/rejected": -0.15605469048023224, + "step": 6010 + }, + { + "epoch": 0.45682197602064045, + "grad_norm": 2.4822771588709673, + "learning_rate": 1.0310785244529341e-06, + "log_odds_chosen": 1.3820312023162842, + "log_odds_ratio": -0.4217773377895355, + "logits/chosen": -0.934765636920929, + "logits/rejected": -0.8529297113418579, + "logps/chosen": -0.6429687738418579, + "logps/rejected": -1.639062523841858, + "loss": 0.8556, + "nll_loss": 0.7791992425918579, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06430663913488388, + "rewards/margins": 0.09968872368335724, + "rewards/rejected": -0.1640625, + "step": 6020 + }, + { + "epoch": 0.4575808165123691, + "grad_norm": 2.3007797386812587, + "learning_rate": 1.0302232123816746e-06, + "log_odds_chosen": 1.421289086341858, + "log_odds_ratio": -0.4197753965854645, + "logits/chosen": -0.9466797113418579, + "logits/rejected": -0.802539050579071, + "logps/chosen": -0.6351562738418579, + "logps/rejected": -1.6257812976837158, + "loss": 0.8092, + "nll_loss": 0.7437499761581421, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06351318210363388, + "rewards/margins": 0.09913329780101776, + "rewards/rejected": -0.16262206435203552, + "step": 6030 + }, + { + "epoch": 0.45833965700409773, + "grad_norm": 3.105742996146195, + "learning_rate": 1.0293700253099576e-06, + "log_odds_chosen": 1.6677734851837158, + "log_odds_ratio": -0.410400390625, + "logits/chosen": -0.9009765386581421, + "logits/rejected": -0.808789074420929, + "logps/chosen": -0.6617187261581421, + "logps/rejected": -1.882421851158142, + "loss": 0.823, + "nll_loss": 0.7767578363418579, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06622314453125, + "rewards/margins": 0.12197265774011612, + "rewards/rejected": -0.18820801377296448, + "step": 6040 + }, + { + "epoch": 0.4590984974958264, + "grad_norm": 2.1351879873106387, + "learning_rate": 1.02851895445316e-06, + "log_odds_chosen": 1.170751929283142, + "log_odds_ratio": -0.4510742127895355, + "logits/chosen": -0.943359375, + "logits/rejected": -0.823437511920929, + "logps/chosen": -0.6552734375, + "logps/rejected": -1.4640624523162842, + "loss": 0.8282, + "nll_loss": 0.7867187261581421, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06549072265625, + "rewards/margins": 0.0809326171875, + "rewards/rejected": -0.14643554389476776, + "step": 6050 + }, + { + "epoch": 0.459857337987555, + "grad_norm": 2.3780810708738698, + "learning_rate": 1.0276699910774159e-06, + "log_odds_chosen": 1.5333983898162842, + "log_odds_ratio": -0.37885743379592896, + "logits/chosen": -0.939648449420929, + "logits/rejected": -0.862500011920929, + "logps/chosen": -0.6524413824081421, + "logps/rejected": -1.6730468273162842, + "loss": 0.8051, + "nll_loss": 0.7705078125, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06528320163488388, + "rewards/margins": 0.10205078125, + "rewards/rejected": -0.16730956733226776, + "step": 6060 + }, + { + "epoch": 0.46061617847928366, + "grad_norm": 2.687672189857859, + "learning_rate": 1.0268231264992398e-06, + "log_odds_chosen": 1.437744140625, + "log_odds_ratio": -0.3941406309604645, + "logits/chosen": -0.9205077886581421, + "logits/rejected": -0.822070300579071, + "logps/chosen": -0.6304687261581421, + "logps/rejected": -1.582617163658142, + "loss": 0.8198, + "nll_loss": 0.772656261920929, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06304931640625, + "rewards/margins": 0.09518585354089737, + "rewards/rejected": -0.15826416015625, + "step": 6070 + }, + { + "epoch": 0.4613750189710123, + "grad_norm": 2.170795848214354, + "learning_rate": 1.0259783520851542e-06, + "log_odds_chosen": 1.331640601158142, + "log_odds_ratio": -0.42583006620407104, + "logits/chosen": -0.970507800579071, + "logits/rejected": -0.886914074420929, + "logps/chosen": -0.59814453125, + "logps/rejected": -1.482421875, + "loss": 0.816, + "nll_loss": 0.7505859136581421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.05979003757238388, + "rewards/margins": 0.08842773735523224, + "rewards/rejected": -0.14826659858226776, + "step": 6080 + }, + { + "epoch": 0.46213385946274094, + "grad_norm": 2.665480296823221, + "learning_rate": 1.0251356592513193e-06, + "log_odds_chosen": 1.791015625, + "log_odds_ratio": -0.35053712129592896, + "logits/chosen": -0.967578113079071, + "logits/rejected": -0.824023425579071, + "logps/chosen": -0.6451171636581421, + "logps/rejected": -1.951562523841858, + "loss": 0.7965, + "nll_loss": 0.797656238079071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06446532905101776, + "rewards/margins": 0.13068847358226776, + "rewards/rejected": -0.195068359375, + "step": 6090 + }, + { + "epoch": 0.4628926999544696, + "grad_norm": 2.5087426355227778, + "learning_rate": 1.0242950394631678e-06, + "log_odds_chosen": 1.780664086341858, + "log_odds_ratio": -0.3534179627895355, + "logits/chosen": -0.8794921636581421, + "logits/rejected": -0.821093738079071, + "logps/chosen": -0.711132824420929, + "logps/rejected": -1.9728515148162842, + "loss": 0.8093, + "nll_loss": 0.7499023675918579, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.07117919623851776, + "rewards/margins": 0.12614135444164276, + "rewards/rejected": -0.19716796278953552, + "step": 6100 + }, + { + "epoch": 0.4636515404461982, + "grad_norm": 2.4831413934098805, + "learning_rate": 1.0234564842350404e-06, + "log_odds_chosen": 1.2590820789337158, + "log_odds_ratio": -0.440185546875, + "logits/chosen": -0.947070300579071, + "logits/rejected": -0.8208984136581421, + "logps/chosen": -0.673828125, + "logps/rejected": -1.556249976158142, + "loss": 0.8492, + "nll_loss": 0.8207031488418579, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06745605170726776, + "rewards/margins": 0.08823852241039276, + "rewards/rejected": -0.15573731064796448, + "step": 6110 + }, + { + "epoch": 0.46441038093792686, + "grad_norm": 2.653542006002146, + "learning_rate": 1.0226199851298272e-06, + "log_odds_chosen": 1.540502905845642, + "log_odds_ratio": -0.43144530057907104, + "logits/chosen": -0.9339843988418579, + "logits/rejected": -0.827929675579071, + "logps/chosen": -0.706250011920929, + "logps/rejected": -1.783789038658142, + "loss": 0.7819, + "nll_loss": 0.736132800579071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.07059326022863388, + "rewards/margins": 0.10786743462085724, + "rewards/rejected": -0.17832031846046448, + "step": 6120 + }, + { + "epoch": 0.4651692214296555, + "grad_norm": 2.1548415042723716, + "learning_rate": 1.0217855337586106e-06, + "log_odds_chosen": 1.2821533679962158, + "log_odds_ratio": -0.42268067598342896, + "logits/chosen": -0.9935547113418579, + "logits/rejected": -0.866992175579071, + "logps/chosen": -0.6748046875, + "logps/rejected": -1.5068359375, + "loss": 0.8128, + "nll_loss": 0.7559570074081421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06744384765625, + "rewards/margins": 0.08310546725988388, + "rewards/rejected": -0.15068359673023224, + "step": 6130 + }, + { + "epoch": 0.46592806192138414, + "grad_norm": 2.307825449488109, + "learning_rate": 1.0209531217803119e-06, + "log_odds_chosen": 1.13671875, + "log_odds_ratio": -0.4474121034145355, + "logits/chosen": -0.973437488079071, + "logits/rejected": -0.8666015863418579, + "logps/chosen": -0.6343749761581421, + "logps/rejected": -1.3761718273162842, + "loss": 0.8027, + "nll_loss": 0.7076171636581421, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06343994289636612, + "rewards/margins": 0.07416381686925888, + "rewards/rejected": -0.1376953125, + "step": 6140 + }, + { + "epoch": 0.4666869024131128, + "grad_norm": 2.8040635992428307, + "learning_rate": 1.0201227409013412e-06, + "log_odds_chosen": 1.5504882335662842, + "log_odds_ratio": -0.40693360567092896, + "logits/chosen": -0.8951171636581421, + "logits/rejected": -0.7806640863418579, + "logps/chosen": -0.7017577886581421, + "logps/rejected": -1.7919921875, + "loss": 0.8257, + "nll_loss": 0.786328136920929, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.07017822563648224, + "rewards/margins": 0.10901489108800888, + "rewards/rejected": -0.17917481064796448, + "step": 6150 + }, + { + "epoch": 0.4674457429048414, + "grad_norm": 2.2831418752658346, + "learning_rate": 1.0192943828752509e-06, + "log_odds_chosen": 1.371679663658142, + "log_odds_ratio": -0.4322753846645355, + "logits/chosen": -0.917187511920929, + "logits/rejected": -0.841015636920929, + "logps/chosen": -0.641308605670929, + "logps/rejected": -1.593164086341858, + "loss": 0.8173, + "nll_loss": 0.727246105670929, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06419678032398224, + "rewards/margins": 0.09511718899011612, + "rewards/rejected": -0.15927734971046448, + "step": 6160 + }, + { + "epoch": 0.46820458339657006, + "grad_norm": 2.2630773718540613, + "learning_rate": 1.0184680395023912e-06, + "log_odds_chosen": 1.4542968273162842, + "log_odds_ratio": -0.3931640684604645, + "logits/chosen": -0.974609375, + "logits/rejected": -0.87890625, + "logps/chosen": -0.6356445550918579, + "logps/rejected": -1.6287109851837158, + "loss": 0.8154, + "nll_loss": 0.709765613079071, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06364746391773224, + "rewards/margins": 0.09923706203699112, + "rewards/rejected": -0.16269531846046448, + "step": 6170 + }, + { + "epoch": 0.4689634238882987, + "grad_norm": 2.6098171460108888, + "learning_rate": 1.0176437026295688e-06, + "log_odds_chosen": 1.3972656726837158, + "log_odds_ratio": -0.4120117127895355, + "logits/chosen": -0.857421875, + "logits/rejected": -0.7744140625, + "logps/chosen": -0.6465820074081421, + "logps/rejected": -1.575781226158142, + "loss": 0.8127, + "nll_loss": 0.8021484613418579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06464843451976776, + "rewards/margins": 0.09293518215417862, + "rewards/rejected": -0.1575927734375, + "step": 6180 + }, + { + "epoch": 0.46972226438002734, + "grad_norm": 2.6375229216028724, + "learning_rate": 1.0168213641497094e-06, + "log_odds_chosen": 1.298437476158142, + "log_odds_ratio": -0.4339355528354645, + "logits/chosen": -0.9283202886581421, + "logits/rejected": -0.8431640863418579, + "logps/chosen": -0.6709960699081421, + "logps/rejected": -1.575781226158142, + "loss": 0.7993, + "nll_loss": 0.732226550579071, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06699218600988388, + "rewards/margins": 0.09050293266773224, + "rewards/rejected": -0.15751953423023224, + "step": 6190 + }, + { + "epoch": 0.470481104871756, + "grad_norm": 3.3544143519561898, + "learning_rate": 1.016001016001524e-06, + "log_odds_chosen": 1.2083008289337158, + "log_odds_ratio": -0.4463867247104645, + "logits/chosen": -0.9027343988418579, + "logits/rejected": -0.794726550579071, + "logps/chosen": -0.6932617425918579, + "logps/rejected": -1.521875023841858, + "loss": 0.8244, + "nll_loss": 0.7562500238418579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06927490234375, + "rewards/margins": 0.08277587592601776, + "rewards/rejected": -0.15219727158546448, + "step": 6200 + }, + { + "epoch": 0.47123994536348457, + "grad_norm": 2.487557177113271, + "learning_rate": 1.0151826501691747e-06, + "log_odds_chosen": 1.4021484851837158, + "log_odds_ratio": -0.3809570372104645, + "logits/chosen": -0.879687488079071, + "logits/rejected": -0.802734375, + "logps/chosen": -0.625683605670929, + "logps/rejected": -1.536523461341858, + "loss": 0.7946, + "nll_loss": 0.7621093988418579, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06257323920726776, + "rewards/margins": 0.09113769233226776, + "rewards/rejected": -0.15361328423023224, + "step": 6210 + }, + { + "epoch": 0.4719987858552132, + "grad_norm": 2.418866186404193, + "learning_rate": 1.0143662586819475e-06, + "log_odds_chosen": 1.589453101158142, + "log_odds_ratio": -0.3534912168979645, + "logits/chosen": -0.966601550579071, + "logits/rejected": -0.80078125, + "logps/chosen": -0.6705077886581421, + "logps/rejected": -1.813085913658142, + "loss": 0.8171, + "nll_loss": 0.7392578125, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06708984076976776, + "rewards/margins": 0.11421509087085724, + "rewards/rejected": -0.18132324516773224, + "step": 6220 + }, + { + "epoch": 0.47275762634694185, + "grad_norm": 2.381482919906414, + "learning_rate": 1.0135518336139257e-06, + "log_odds_chosen": 1.4799315929412842, + "log_odds_ratio": -0.4188476502895355, + "logits/chosen": -0.9701172113418579, + "logits/rejected": -0.8316406011581421, + "logps/chosen": -0.659375011920929, + "logps/rejected": -1.708398461341858, + "loss": 0.8183, + "nll_loss": 0.7705078125, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06590576469898224, + "rewards/margins": 0.10494537651538849, + "rewards/rejected": -0.17062988877296448, + "step": 6230 + }, + { + "epoch": 0.4735164668386705, + "grad_norm": 2.3309362231959567, + "learning_rate": 1.0127393670836666e-06, + "log_odds_chosen": 1.4430663585662842, + "log_odds_ratio": -0.40898436307907104, + "logits/chosen": -0.9566406011581421, + "logits/rejected": -0.8369140625, + "logps/chosen": -0.6279296875, + "logps/rejected": -1.6201171875, + "loss": 0.8014, + "nll_loss": 0.7728515863418579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06281737983226776, + "rewards/margins": 0.09914550930261612, + "rewards/rejected": -0.16201171278953552, + "step": 6240 + }, + { + "epoch": 0.47427530733039913, + "grad_norm": 2.5804457389263047, + "learning_rate": 1.0119288512538813e-06, + "log_odds_chosen": 1.963281273841858, + "log_odds_ratio": -0.35100096464157104, + "logits/chosen": -0.9498046636581421, + "logits/rejected": -0.8374999761581421, + "logps/chosen": -0.628222644329071, + "logps/rejected": -1.9855468273162842, + "loss": 0.8272, + "nll_loss": 0.7630859613418579, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06281737983226776, + "rewards/margins": 0.13579101860523224, + "rewards/rejected": -0.19858399033546448, + "step": 6250 + }, + { + "epoch": 0.4750341478221278, + "grad_norm": 2.3869552633023035, + "learning_rate": 1.0111202783311173e-06, + "log_odds_chosen": 1.5701172351837158, + "log_odds_ratio": -0.3877929747104645, + "logits/chosen": -0.9560546875, + "logits/rejected": -0.8099609613418579, + "logps/chosen": -0.6529296636581421, + "logps/rejected": -1.7900390625, + "loss": 0.816, + "nll_loss": 0.8255859613418579, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.0653076171875, + "rewards/margins": 0.11373291164636612, + "rewards/rejected": -0.17902831733226776, + "step": 6260 + }, + { + "epoch": 0.4757929883138564, + "grad_norm": 2.3005688557187263, + "learning_rate": 1.010313640565443e-06, + "log_odds_chosen": 1.3795166015625, + "log_odds_ratio": -0.40571290254592896, + "logits/chosen": -0.9964843988418579, + "logits/rejected": -0.866406261920929, + "logps/chosen": -0.664355456829071, + "logps/rejected": -1.5773437023162842, + "loss": 0.8188, + "nll_loss": 0.745800793170929, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06641845405101776, + "rewards/margins": 0.09126739203929901, + "rewards/rejected": -0.15764160454273224, + "step": 6270 + }, + { + "epoch": 0.47655182880558505, + "grad_norm": 2.5197130483717154, + "learning_rate": 1.0095089302501373e-06, + "log_odds_chosen": 1.5861327648162842, + "log_odds_ratio": -0.35136717557907104, + "logits/chosen": -0.9964843988418579, + "logits/rejected": -0.834765613079071, + "logps/chosen": -0.6175781488418579, + "logps/rejected": -1.708593726158142, + "loss": 0.8025, + "nll_loss": 0.7518554925918579, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06170654296875, + "rewards/margins": 0.10915527492761612, + "rewards/rejected": -0.17084960639476776, + "step": 6280 + }, + { + "epoch": 0.4773106692973137, + "grad_norm": 2.078318456287781, + "learning_rate": 1.0087061397213787e-06, + "log_odds_chosen": 1.7483398914337158, + "log_odds_ratio": -0.34101563692092896, + "logits/chosen": -1.021875023841858, + "logits/rejected": -0.9097656011581421, + "logps/chosen": -0.6278320550918579, + "logps/rejected": -1.861328125, + "loss": 0.7998, + "nll_loss": 0.7367187738418579, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06281737983226776, + "rewards/margins": 0.12324218451976776, + "rewards/rejected": -0.18601074814796448, + "step": 6290 + }, + { + "epoch": 0.47806950978904234, + "grad_norm": 2.5485572768603566, + "learning_rate": 1.007905261357939e-06, + "log_odds_chosen": 1.2209961414337158, + "log_odds_ratio": -0.473388671875, + "logits/chosen": -0.9798828363418579, + "logits/rejected": -0.8949218988418579, + "logps/chosen": -0.6318359375, + "logps/rejected": -1.4533202648162842, + "loss": 0.8182, + "nll_loss": 0.750781238079071, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.06322021782398224, + "rewards/margins": 0.08211059868335724, + "rewards/rejected": -0.14533691108226776, + "step": 6300 + }, + { + "epoch": 0.478828350280771, + "grad_norm": 3.300544617659391, + "learning_rate": 1.0071062875808811e-06, + "log_odds_chosen": 1.6111328601837158, + "log_odds_ratio": -0.3555664122104645, + "logits/chosen": -0.974804699420929, + "logits/rejected": -0.8792968988418579, + "logps/chosen": -0.622851550579071, + "logps/rejected": -1.7429687976837158, + "loss": 0.8169, + "nll_loss": 0.763671875, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.062255859375, + "rewards/margins": 0.11188964545726776, + "rewards/rejected": -0.17421874403953552, + "step": 6310 + }, + { + "epoch": 0.4795871907724996, + "grad_norm": 2.406231745232268, + "learning_rate": 1.0063092108532552e-06, + "log_odds_chosen": 1.4111328125, + "log_odds_ratio": -0.42919921875, + "logits/chosen": -0.931445300579071, + "logits/rejected": -0.8218749761581421, + "logps/chosen": -0.6539062261581421, + "logps/rejected": -1.661523461341858, + "loss": 0.8042, + "nll_loss": 0.8306640386581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06540527194738388, + "rewards/margins": 0.10084228217601776, + "rewards/rejected": -0.16633300483226776, + "step": 6320 + }, + { + "epoch": 0.48034603126422826, + "grad_norm": 2.4878751333253684, + "learning_rate": 1.005514023679802e-06, + "log_odds_chosen": 1.4918944835662842, + "log_odds_ratio": -0.4136718809604645, + "logits/chosen": -0.966796875, + "logits/rejected": -0.849414050579071, + "logps/chosen": -0.675097644329071, + "logps/rejected": -1.759765625, + "loss": 0.8069, + "nll_loss": 0.761523425579071, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06749267876148224, + "rewards/margins": 0.10855712741613388, + "rewards/rejected": -0.17607422173023224, + "step": 6330 + }, + { + "epoch": 0.4811048717559569, + "grad_norm": 2.7882353969131612, + "learning_rate": 1.0047207186066567e-06, + "log_odds_chosen": 1.155615210533142, + "log_odds_ratio": -0.4690918028354645, + "logits/chosen": -0.9164062738418579, + "logits/rejected": -0.8017578125, + "logps/chosen": -0.670214831829071, + "logps/rejected": -1.4445312023162842, + "loss": 0.8218, + "nll_loss": 0.829296886920929, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.0670166015625, + "rewards/margins": 0.0775909423828125, + "rewards/rejected": -0.14462891221046448, + "step": 6340 + }, + { + "epoch": 0.48186371224768554, + "grad_norm": 2.58489950967846, + "learning_rate": 1.0039292882210538e-06, + "log_odds_chosen": 1.283203125, + "log_odds_ratio": -0.40375977754592896, + "logits/chosen": -0.9554687738418579, + "logits/rejected": -0.861523449420929, + "logps/chosen": -0.64013671875, + "logps/rejected": -1.497460961341858, + "loss": 0.8111, + "nll_loss": 0.7305663824081421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06403808295726776, + "rewards/margins": 0.08568725734949112, + "rewards/rejected": -0.14975586533546448, + "step": 6350 + }, + { + "epoch": 0.4826225527394142, + "grad_norm": 4.037771382759174, + "learning_rate": 1.0031397251510382e-06, + "log_odds_chosen": 1.4892578125, + "log_odds_ratio": -0.3987060487270355, + "logits/chosen": -0.963085949420929, + "logits/rejected": -0.8248046636581421, + "logps/chosen": -0.640917956829071, + "logps/rejected": -1.655859351158142, + "loss": 0.8145, + "nll_loss": 0.7591797113418579, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06409911811351776, + "rewards/margins": 0.10162963718175888, + "rewards/rejected": -0.16569824516773224, + "step": 6360 + }, + { + "epoch": 0.4833813932311428, + "grad_norm": 2.564331922881601, + "learning_rate": 1.0023520220651762e-06, + "log_odds_chosen": 1.2976562976837158, + "log_odds_ratio": -0.42290037870407104, + "logits/chosen": -0.9613281488418579, + "logits/rejected": -0.876171886920929, + "logps/chosen": -0.6351562738418579, + "logps/rejected": -1.515234351158142, + "loss": 0.8257, + "nll_loss": 0.7596679925918579, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06354980170726776, + "rewards/margins": 0.08784790337085724, + "rewards/rejected": -0.1514892578125, + "step": 6370 + }, + { + "epoch": 0.48414023372287146, + "grad_norm": 2.470819586298383, + "learning_rate": 1.0015661716722687e-06, + "log_odds_chosen": 1.3885009288787842, + "log_odds_ratio": -0.39545899629592896, + "logits/chosen": -0.9779297113418579, + "logits/rejected": -0.871874988079071, + "logps/chosen": -0.6226562261581421, + "logps/rejected": -1.562109351158142, + "loss": 0.8322, + "nll_loss": 0.7376953363418579, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06226806715130806, + "rewards/margins": 0.09392700344324112, + "rewards/rejected": -0.15616455674171448, + "step": 6380 + }, + { + "epoch": 0.4848990742146001, + "grad_norm": 2.3090696991088766, + "learning_rate": 1.0007821667210687e-06, + "log_odds_chosen": 1.4011719226837158, + "log_odds_ratio": -0.41938477754592896, + "logits/chosen": -0.947070300579071, + "logits/rejected": -0.853320300579071, + "logps/chosen": -0.6456054449081421, + "logps/rejected": -1.5625, + "loss": 0.7884, + "nll_loss": 0.7489258050918579, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06453857570886612, + "rewards/margins": 0.09173583984375, + "rewards/rejected": -0.15617676079273224, + "step": 6390 + }, + { + "epoch": 0.48565791470632874, + "grad_norm": 2.483183154500476, + "learning_rate": 1e-06, + "log_odds_chosen": 1.47705078125, + "log_odds_ratio": -0.42402344942092896, + "logits/chosen": -0.971875011920929, + "logits/rejected": -0.8421875238418579, + "logps/chosen": -0.678906261920929, + "logps/rejected": -1.7238280773162842, + "loss": 0.816, + "nll_loss": 0.767382800579071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06785888969898224, + "rewards/margins": 0.10455322265625, + "rewards/rejected": -0.17243652045726776, + "step": 6400 + }, + { + "epoch": 0.4864167551980574, + "grad_norm": 3.452449471753027, + "learning_rate": 9.992196643368784e-07, + "log_odds_chosen": 1.265527367591858, + "log_odds_ratio": -0.4453125, + "logits/chosen": -0.952929675579071, + "logits/rejected": -0.860546886920929, + "logps/chosen": -0.7083984613418579, + "logps/rejected": -1.617578148841858, + "loss": 0.8028, + "nll_loss": 0.761914074420929, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07075195014476776, + "rewards/margins": 0.09090881049633026, + "rewards/rejected": -0.16179199516773224, + "step": 6410 + }, + { + "epoch": 0.487175595689786, + "grad_norm": 2.524952848845386, + "learning_rate": 9.984411525986355e-07, + "log_odds_chosen": 1.578515648841858, + "log_odds_ratio": -0.37608641386032104, + "logits/chosen": -0.9580078125, + "logits/rejected": -0.8207031488418579, + "logps/chosen": -0.664355456829071, + "logps/rejected": -1.790624976158142, + "loss": 0.8046, + "nll_loss": 0.752148449420929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06643066555261612, + "rewards/margins": 0.11257324367761612, + "rewards/rejected": -0.17915038764476776, + "step": 6420 + }, + { + "epoch": 0.48793443618151466, + "grad_norm": 2.8427253252952314, + "learning_rate": 9.97664457691046e-07, + "log_odds_chosen": 1.7546875476837158, + "log_odds_ratio": -0.3233398497104645, + "logits/chosen": -0.9697265625, + "logits/rejected": -0.8939453363418579, + "logps/chosen": -0.6312500238418579, + "logps/rejected": -1.8351562023162842, + "loss": 0.7977, + "nll_loss": 0.74560546875, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06314697116613388, + "rewards/margins": 0.12041015923023224, + "rewards/rejected": -0.18354491889476776, + "step": 6430 + }, + { + "epoch": 0.4886932766732433, + "grad_norm": 2.5527692131010222, + "learning_rate": 9.968895725584535e-07, + "log_odds_chosen": 1.484960913658142, + "log_odds_ratio": -0.3592285215854645, + "logits/chosen": -0.968945324420929, + "logits/rejected": -0.8492187261581421, + "logps/chosen": -0.614453136920929, + "logps/rejected": -1.6326172351837158, + "loss": 0.8045, + "nll_loss": 0.768359363079071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06147461012005806, + "rewards/margins": 0.10172118991613388, + "rewards/rejected": -0.16325683891773224, + "step": 6440 + }, + { + "epoch": 0.48945211716497194, + "grad_norm": 2.815766479517196, + "learning_rate": 9.961164901835046e-07, + "log_odds_chosen": 1.7703125476837158, + "log_odds_ratio": -0.32661134004592896, + "logits/chosen": -0.9476562738418579, + "logits/rejected": -0.829296886920929, + "logps/chosen": -0.61962890625, + "logps/rejected": -1.865234375, + "loss": 0.7901, + "nll_loss": 0.714062511920929, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06196289137005806, + "rewards/margins": 0.12456054985523224, + "rewards/rejected": -0.1865234375, + "step": 6450 + }, + { + "epoch": 0.4902109576567006, + "grad_norm": 2.292149341426057, + "learning_rate": 9.95345203586879e-07, + "log_odds_chosen": 1.44140625, + "log_odds_ratio": -0.36518555879592896, + "logits/chosen": -0.9150390625, + "logits/rejected": -0.779101550579071, + "logps/chosen": -0.6884765625, + "logps/rejected": -1.656640648841858, + "loss": 0.7973, + "nll_loss": 0.783203125, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06885986030101776, + "rewards/margins": 0.09699706733226776, + "rewards/rejected": -0.16594238579273224, + "step": 6460 + }, + { + "epoch": 0.4909697981484292, + "grad_norm": 2.659140977437507, + "learning_rate": 9.94575705827027e-07, + "log_odds_chosen": 1.412866234779358, + "log_odds_ratio": -0.4181152284145355, + "logits/chosen": -0.9476562738418579, + "logits/rejected": -0.8441406488418579, + "logps/chosen": -0.655078113079071, + "logps/rejected": -1.6271483898162842, + "loss": 0.8285, + "nll_loss": 0.7847656011581421, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06549072265625, + "rewards/margins": 0.09720764309167862, + "rewards/rejected": -0.16262206435203552, + "step": 6470 + }, + { + "epoch": 0.4917286386401578, + "grad_norm": 2.85017131233847, + "learning_rate": 9.938079899999065e-07, + "log_odds_chosen": 1.4617187976837158, + "log_odds_ratio": -0.419921875, + "logits/chosen": -0.943554699420929, + "logits/rejected": -0.8587890863418579, + "logps/chosen": -0.647753894329071, + "logps/rejected": -1.6892578601837158, + "loss": 0.7926, + "nll_loss": 0.7222656011581421, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06478271633386612, + "rewards/margins": 0.10407714545726776, + "rewards/rejected": -0.16892090439796448, + "step": 6480 + }, + { + "epoch": 0.49248747913188645, + "grad_norm": 2.9059717253538317, + "learning_rate": 9.930420492387219e-07, + "log_odds_chosen": 1.4695312976837158, + "log_odds_ratio": -0.37470704317092896, + "logits/chosen": -0.941601574420929, + "logits/rejected": -0.8447265625, + "logps/chosen": -0.6338866949081421, + "logps/rejected": -1.6521484851837158, + "loss": 0.8109, + "nll_loss": 0.6924804449081421, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06337890774011612, + "rewards/margins": 0.101806640625, + "rewards/rejected": -0.1651611328125, + "step": 6490 + }, + { + "epoch": 0.4932463196236151, + "grad_norm": 2.691907233454959, + "learning_rate": 9.922778767136676e-07, + "log_odds_chosen": 1.397314429283142, + "log_odds_ratio": -0.4439453184604645, + "logits/chosen": -0.913281261920929, + "logits/rejected": -0.836132824420929, + "logps/chosen": -0.6612304449081421, + "logps/rejected": -1.5908203125, + "loss": 0.8096, + "nll_loss": 0.744433581829071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06613769382238388, + "rewards/margins": 0.09300384670495987, + "rewards/rejected": -0.15898437798023224, + "step": 6500 + }, + { + "epoch": 0.49400516011534373, + "grad_norm": 2.5959200764511006, + "learning_rate": 9.915154656316713e-07, + "log_odds_chosen": 1.4679687023162842, + "log_odds_ratio": -0.4046386778354645, + "logits/chosen": -0.9632812738418579, + "logits/rejected": -0.8490234613418579, + "logps/chosen": -0.5916992425918579, + "logps/rejected": -1.601953148841858, + "loss": 0.7864, + "nll_loss": 0.708789050579071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.05917968600988388, + "rewards/margins": 0.10110016167163849, + "rewards/rejected": -0.160400390625, + "step": 6510 + }, + { + "epoch": 0.4947640006070724, + "grad_norm": 3.02227371619474, + "learning_rate": 9.907548092361398e-07, + "log_odds_chosen": 1.3546874523162842, + "log_odds_ratio": -0.45415037870407104, + "logits/chosen": -0.994921863079071, + "logits/rejected": -0.900195300579071, + "logps/chosen": -0.67236328125, + "logps/rejected": -1.646093726158142, + "loss": 0.7978, + "nll_loss": 0.7759765386581421, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.06719970703125, + "rewards/margins": 0.09732665866613388, + "rewards/rejected": -0.1644287109375, + "step": 6520 + }, + { + "epoch": 0.495522841098801, + "grad_norm": 2.5968513939390645, + "learning_rate": 9.899959008067097e-07, + "log_odds_chosen": 1.599267601966858, + "log_odds_ratio": -0.38164061307907104, + "logits/chosen": -0.9339843988418579, + "logits/rejected": -0.8763672113418579, + "logps/chosen": -0.6392577886581421, + "logps/rejected": -1.762109398841858, + "loss": 0.7659, + "nll_loss": 0.692187488079071, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06394042819738388, + "rewards/margins": 0.11228637397289276, + "rewards/rejected": -0.17622070014476776, + "step": 6530 + }, + { + "epoch": 0.49628168159052966, + "grad_norm": 3.582994884785442, + "learning_rate": 9.892387336589959e-07, + "log_odds_chosen": 1.3213622570037842, + "log_odds_ratio": -0.44208985567092896, + "logits/chosen": -0.9892578125, + "logits/rejected": -0.857617199420929, + "logps/chosen": -0.6802734136581421, + "logps/rejected": -1.584570288658142, + "loss": 0.8045, + "nll_loss": 0.7720702886581421, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06806640326976776, + "rewards/margins": 0.090362548828125, + "rewards/rejected": -0.158447265625, + "step": 6540 + }, + { + "epoch": 0.4970405220822583, + "grad_norm": 2.6768237442220606, + "learning_rate": 9.884833011443446e-07, + "log_odds_chosen": 1.5231444835662842, + "log_odds_ratio": -0.38959962129592896, + "logits/chosen": -0.970507800579071, + "logits/rejected": -0.8369140625, + "logps/chosen": -0.6744140386581421, + "logps/rejected": -1.736328125, + "loss": 0.8018, + "nll_loss": 0.8306640386581421, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06745605170726776, + "rewards/margins": 0.10619811713695526, + "rewards/rejected": -0.173583984375, + "step": 6550 + }, + { + "epoch": 0.49779936257398694, + "grad_norm": 2.5392687392566686, + "learning_rate": 9.877295966495897e-07, + "log_odds_chosen": 1.53759765625, + "log_odds_ratio": -0.36699217557907104, + "logits/chosen": -0.985156238079071, + "logits/rejected": -0.8607422113418579, + "logps/chosen": -0.621874988079071, + "logps/rejected": -1.692773461341858, + "loss": 0.7968, + "nll_loss": 0.708203136920929, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06223144382238388, + "rewards/margins": 0.10717163234949112, + "rewards/rejected": -0.16948242485523224, + "step": 6560 + }, + { + "epoch": 0.4985582030657156, + "grad_norm": 2.53456526882817, + "learning_rate": 9.86977613596807e-07, + "log_odds_chosen": 1.7412109375, + "log_odds_ratio": -0.38007813692092896, + "logits/chosen": -1.0089843273162842, + "logits/rejected": -0.8724609613418579, + "logps/chosen": -0.658203125, + "logps/rejected": -1.911718726158142, + "loss": 0.7795, + "nll_loss": 0.7701171636581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06582031399011612, + "rewards/margins": 0.12550048530101776, + "rewards/rejected": -0.19113770127296448, + "step": 6570 + }, + { + "epoch": 0.4993170435574442, + "grad_norm": 12.048243333811214, + "learning_rate": 9.862273454430757e-07, + "log_odds_chosen": 1.641015648841858, + "log_odds_ratio": -0.3609375059604645, + "logits/chosen": -0.935742199420929, + "logits/rejected": -0.828906238079071, + "logps/chosen": -0.637890636920929, + "logps/rejected": -1.822265625, + "loss": 0.7977, + "nll_loss": 0.7635742425918579, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06374511867761612, + "rewards/margins": 0.11837158352136612, + "rewards/rejected": -0.18215331435203552, + "step": 6580 + }, + { + "epoch": 0.5000758840491729, + "grad_norm": 2.7112507013505214, + "learning_rate": 9.85478785680238e-07, + "log_odds_chosen": 1.4053223133087158, + "log_odds_ratio": -0.4013671875, + "logits/chosen": -0.940625011920929, + "logits/rejected": -0.8060547113418579, + "logps/chosen": -0.644335925579071, + "logps/rejected": -1.6544921398162842, + "loss": 0.8105, + "nll_loss": 0.7601562738418579, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.064453125, + "rewards/margins": 0.10089111328125, + "rewards/rejected": -0.16542968153953552, + "step": 6590 + }, + { + "epoch": 0.5008347245409015, + "grad_norm": 2.4884711966903086, + "learning_rate": 9.847319278346618e-07, + "log_odds_chosen": 1.386816382408142, + "log_odds_ratio": -0.4229736328125, + "logits/chosen": -0.931835949420929, + "logits/rejected": -0.8599609136581421, + "logps/chosen": -0.6346679925918579, + "logps/rejected": -1.5812499523162842, + "loss": 0.8052, + "nll_loss": 0.7777343988418579, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06348876655101776, + "rewards/margins": 0.094635009765625, + "rewards/rejected": -0.15817871689796448, + "step": 6600 + }, + { + "epoch": 0.5015935650326302, + "grad_norm": 2.4527795187527057, + "learning_rate": 9.839867654670063e-07, + "log_odds_chosen": 1.3405272960662842, + "log_odds_ratio": -0.4638671875, + "logits/chosen": -0.9136718511581421, + "logits/rejected": -0.814453125, + "logps/chosen": -0.6714843511581421, + "logps/rejected": -1.635351538658142, + "loss": 0.7844, + "nll_loss": 0.7486327886581421, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06706543266773224, + "rewards/margins": 0.09641418606042862, + "rewards/rejected": -0.16340331733226776, + "step": 6610 + }, + { + "epoch": 0.5023524055243588, + "grad_norm": 2.5479226445182865, + "learning_rate": 9.832432921719876e-07, + "log_odds_chosen": 1.3992798328399658, + "log_odds_ratio": -0.3910156190395355, + "logits/chosen": -0.9623047113418579, + "logits/rejected": -0.8433593511581421, + "logps/chosen": -0.6591796875, + "logps/rejected": -1.641015648841858, + "loss": 0.802, + "nll_loss": 0.721484363079071, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06589355319738388, + "rewards/margins": 0.09831543266773224, + "rewards/rejected": -0.16408690810203552, + "step": 6620 + }, + { + "epoch": 0.5031112460160874, + "grad_norm": 3.0071032712651777, + "learning_rate": 9.825015015781493e-07, + "log_odds_chosen": 1.422216773033142, + "log_odds_ratio": -0.4296875, + "logits/chosen": -0.9609375, + "logits/rejected": -0.818554699420929, + "logps/chosen": -0.6146484613418579, + "logps/rejected": -1.611914038658142, + "loss": 0.7841, + "nll_loss": 0.7835937738418579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0614013671875, + "rewards/margins": 0.09964065253734589, + "rewards/rejected": -0.16115722060203552, + "step": 6630 + }, + { + "epoch": 0.5038700865078161, + "grad_norm": 2.5699009966789226, + "learning_rate": 9.81761387347632e-07, + "log_odds_chosen": 1.316015601158142, + "log_odds_ratio": -0.45805662870407104, + "logits/chosen": -0.978710949420929, + "logits/rejected": -0.8802734613418579, + "logps/chosen": -0.6343749761581421, + "logps/rejected": -1.5720703601837158, + "loss": 0.7949, + "nll_loss": 0.7490234375, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06349487602710724, + "rewards/margins": 0.09367676079273224, + "rewards/rejected": -0.15717773139476776, + "step": 6640 + }, + { + "epoch": 0.5046289269995446, + "grad_norm": 3.156751365213722, + "learning_rate": 9.810229431759452e-07, + "log_odds_chosen": 1.333398461341858, + "log_odds_ratio": -0.4351562559604645, + "logits/chosen": -0.961132824420929, + "logits/rejected": -0.8550781011581421, + "logps/chosen": -0.6317383050918579, + "logps/rejected": -1.5480468273162842, + "loss": 0.7878, + "nll_loss": 0.742968738079071, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06318359076976776, + "rewards/margins": 0.09176330268383026, + "rewards/rejected": -0.15485839545726776, + "step": 6650 + }, + { + "epoch": 0.5053877674912733, + "grad_norm": 2.5289706810829857, + "learning_rate": 9.802861627917437e-07, + "log_odds_chosen": 1.618188500404358, + "log_odds_ratio": -0.3940673768520355, + "logits/chosen": -1.00390625, + "logits/rejected": -0.8685547113418579, + "logps/chosen": -0.682910144329071, + "logps/rejected": -1.8113281726837158, + "loss": 0.8064, + "nll_loss": 0.7100585699081421, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06828613579273224, + "rewards/margins": 0.11281128227710724, + "rewards/rejected": -0.18125000596046448, + "step": 6660 + }, + { + "epoch": 0.5061466079830019, + "grad_norm": 2.7523920564694198, + "learning_rate": 9.795510399566016e-07, + "log_odds_chosen": 1.1799805164337158, + "log_odds_ratio": -0.4410156309604645, + "logits/chosen": -0.93359375, + "logits/rejected": -0.80078125, + "logps/chosen": -0.6509765386581421, + "logps/rejected": -1.4345703125, + "loss": 0.7868, + "nll_loss": 0.713671863079071, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06510009616613388, + "rewards/margins": 0.07830810546875, + "rewards/rejected": -0.14338378608226776, + "step": 6670 + }, + { + "epoch": 0.5069054484747306, + "grad_norm": 2.640847612481933, + "learning_rate": 9.788175684647926e-07, + "log_odds_chosen": 1.442773461341858, + "log_odds_ratio": -0.37919920682907104, + "logits/chosen": -0.9419921636581421, + "logits/rejected": -0.830078125, + "logps/chosen": -0.619824230670929, + "logps/rejected": -1.601171851158142, + "loss": 0.788, + "nll_loss": 0.7431640625, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06198730319738388, + "rewards/margins": 0.09815673530101776, + "rewards/rejected": -0.16010741889476776, + "step": 6680 + }, + { + "epoch": 0.5076642889664592, + "grad_norm": 2.818182715455829, + "learning_rate": 9.780857421430687e-07, + "log_odds_chosen": 1.58154296875, + "log_odds_ratio": -0.3661132752895355, + "logits/chosen": -1.0029296875, + "logits/rejected": -0.8433593511581421, + "logps/chosen": -0.6211913824081421, + "logps/rejected": -1.732812523841858, + "loss": 0.7782, + "nll_loss": 0.747265636920929, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06218261644244194, + "rewards/margins": 0.11107178032398224, + "rewards/rejected": -0.17324218153953552, + "step": 6690 + }, + { + "epoch": 0.5084231294581879, + "grad_norm": 2.294944830860249, + "learning_rate": 9.773555548504417e-07, + "log_odds_chosen": 1.3458983898162842, + "log_odds_ratio": -0.40214842557907104, + "logits/chosen": -0.943164050579071, + "logits/rejected": -0.8192383050918579, + "logps/chosen": -0.6675781011581421, + "logps/rejected": -1.562109351158142, + "loss": 0.7806, + "nll_loss": 0.7562500238418579, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06678466498851776, + "rewards/margins": 0.08942870795726776, + "rewards/rejected": -0.15615233778953552, + "step": 6700 + }, + { + "epoch": 0.5091819699499165, + "grad_norm": 2.6900119883631857, + "learning_rate": 9.76627000477968e-07, + "log_odds_chosen": 1.305078148841858, + "log_odds_ratio": -0.4170898497104645, + "logits/chosen": -0.9849609136581421, + "logits/rejected": -0.876953125, + "logps/chosen": -0.657421886920929, + "logps/rejected": -1.5378906726837158, + "loss": 0.791, + "nll_loss": 0.7315429449081421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06569824367761612, + "rewards/margins": 0.087982177734375, + "rewards/rejected": -0.15371093153953552, + "step": 6710 + }, + { + "epoch": 0.5099408104416452, + "grad_norm": 2.245518934976984, + "learning_rate": 9.75900072948533e-07, + "log_odds_chosen": 1.2332031726837158, + "log_odds_ratio": -0.4161621034145355, + "logits/chosen": -0.980664074420929, + "logits/rejected": -0.838671863079071, + "logps/chosen": -0.646191418170929, + "logps/rejected": -1.4474608898162842, + "loss": 0.7994, + "nll_loss": 0.751953125, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06459961086511612, + "rewards/margins": 0.080230712890625, + "rewards/rejected": -0.14479979872703552, + "step": 6720 + }, + { + "epoch": 0.5106996509333738, + "grad_norm": 2.3214095844998477, + "learning_rate": 9.751747662166388e-07, + "log_odds_chosen": 1.365820288658142, + "log_odds_ratio": -0.41303712129592896, + "logits/chosen": -0.975781261920929, + "logits/rejected": -0.8677734136581421, + "logps/chosen": -0.6664062738418579, + "logps/rejected": -1.587304711341858, + "loss": 0.8016, + "nll_loss": 0.693066418170929, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06666259467601776, + "rewards/margins": 0.09200439602136612, + "rewards/rejected": -0.15866699814796448, + "step": 6730 + }, + { + "epoch": 0.5114584914251025, + "grad_norm": 2.7639231282830115, + "learning_rate": 9.744510742681917e-07, + "log_odds_chosen": 1.160546898841858, + "log_odds_ratio": -0.4703125059604645, + "logits/chosen": -1.024804711341858, + "logits/rejected": -0.891406238079071, + "logps/chosen": -0.6795898675918579, + "logps/rejected": -1.43359375, + "loss": 0.8038, + "nll_loss": 0.7606445550918579, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06793212890625, + "rewards/margins": 0.07551880180835724, + "rewards/rejected": -0.1434326171875, + "step": 6740 + }, + { + "epoch": 0.512217331916831, + "grad_norm": 2.6184261171577625, + "learning_rate": 9.737289911202953e-07, + "log_odds_chosen": 1.494042992591858, + "log_odds_ratio": -0.3863769471645355, + "logits/chosen": -1.0046875476837158, + "logits/rejected": -0.8541015386581421, + "logps/chosen": -0.6490234136581421, + "logps/rejected": -1.7179687023162842, + "loss": 0.8003, + "nll_loss": 0.7718750238418579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06492920219898224, + "rewards/margins": 0.10683593899011612, + "rewards/rejected": -0.1717529296875, + "step": 6750 + }, + { + "epoch": 0.5129761724085597, + "grad_norm": 3.23821521030104, + "learning_rate": 9.730085108210398e-07, + "log_odds_chosen": 1.39306640625, + "log_odds_ratio": -0.40678709745407104, + "logits/chosen": -0.9076172113418579, + "logits/rejected": -0.849804699420929, + "logps/chosen": -0.6294921636581421, + "logps/rejected": -1.568750023841858, + "loss": 0.7786, + "nll_loss": 0.7955077886581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06291504204273224, + "rewards/margins": 0.09385986626148224, + "rewards/rejected": -0.15690918266773224, + "step": 6760 + }, + { + "epoch": 0.5137350129002883, + "grad_norm": 2.5228606567360137, + "learning_rate": 9.72289627449298e-07, + "log_odds_chosen": 1.4433472156524658, + "log_odds_ratio": -0.410888671875, + "logits/chosen": -0.963671863079071, + "logits/rejected": -0.877148449420929, + "logps/chosen": -0.647656261920929, + "logps/rejected": -1.6749999523162842, + "loss": 0.7765, + "nll_loss": 0.6961914300918579, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06474609673023224, + "rewards/margins": 0.10265503078699112, + "rewards/rejected": -0.16755370795726776, + "step": 6770 + }, + { + "epoch": 0.514493853392017, + "grad_norm": 2.631470233438765, + "learning_rate": 9.715723351145206e-07, + "log_odds_chosen": 1.5310547351837158, + "log_odds_ratio": -0.38579100370407104, + "logits/chosen": -0.966992199420929, + "logits/rejected": -0.884765625, + "logps/chosen": -0.6060546636581421, + "logps/rejected": -1.646093726158142, + "loss": 0.7962, + "nll_loss": 0.700488269329071, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.060546875, + "rewards/margins": 0.10415039211511612, + "rewards/rejected": -0.16472168266773224, + "step": 6780 + }, + { + "epoch": 0.5152526938837456, + "grad_norm": 2.5546199165416588, + "learning_rate": 9.70856627956532e-07, + "log_odds_chosen": 1.447265625, + "log_odds_ratio": -0.3866943418979645, + "logits/chosen": -0.9027343988418579, + "logits/rejected": -0.817187488079071, + "logps/chosen": -0.643261730670929, + "logps/rejected": -1.619140625, + "loss": 0.801, + "nll_loss": 0.76953125, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.0643310546875, + "rewards/margins": 0.09769286960363388, + "rewards/rejected": -0.16188964247703552, + "step": 6790 + }, + { + "epoch": 0.5160115343754743, + "grad_norm": 2.5953715660511416, + "learning_rate": 9.701425001453318e-07, + "log_odds_chosen": 1.3435547351837158, + "log_odds_ratio": -0.41630858182907104, + "logits/chosen": -0.966015636920929, + "logits/rejected": -0.863085925579071, + "logps/chosen": -0.6380859613418579, + "logps/rejected": -1.5373046398162842, + "loss": 0.8171, + "nll_loss": 0.7525390386581421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06385497748851776, + "rewards/margins": 0.08985595405101776, + "rewards/rejected": -0.15378418564796448, + "step": 6800 + }, + { + "epoch": 0.5167703748672029, + "grad_norm": 2.7439030924941736, + "learning_rate": 9.694299458808932e-07, + "log_odds_chosen": 1.505273461341858, + "log_odds_ratio": -0.37578123807907104, + "logits/chosen": -0.980273425579071, + "logits/rejected": -0.836132824420929, + "logps/chosen": -0.6016601324081421, + "logps/rejected": -1.5646483898162842, + "loss": 0.7872, + "nll_loss": 0.69091796875, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06013183668255806, + "rewards/margins": 0.09639892727136612, + "rewards/rejected": -0.15645751357078552, + "step": 6810 + }, + { + "epoch": 0.5175292153589316, + "grad_norm": 2.423228834350587, + "learning_rate": 9.687189593929655e-07, + "log_odds_chosen": 1.675390601158142, + "log_odds_ratio": -0.3683105409145355, + "logits/chosen": -0.9546874761581421, + "logits/rejected": -0.810546875, + "logps/chosen": -0.6268554925918579, + "logps/rejected": -1.775390625, + "loss": 0.7869, + "nll_loss": 0.742871105670929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06267090141773224, + "rewards/margins": 0.11484374850988388, + "rewards/rejected": -0.177490234375, + "step": 6820 + }, + { + "epoch": 0.5182880558506602, + "grad_norm": 2.828370049907918, + "learning_rate": 9.680095349408789e-07, + "log_odds_chosen": 1.562597632408142, + "log_odds_ratio": -0.37763673067092896, + "logits/chosen": -0.950976550579071, + "logits/rejected": -0.823046863079071, + "logps/chosen": -0.6673828363418579, + "logps/rejected": -1.7605469226837158, + "loss": 0.7772, + "nll_loss": 0.768750011920929, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06669922173023224, + "rewards/margins": 0.10936889797449112, + "rewards/rejected": -0.17617186903953552, + "step": 6830 + }, + { + "epoch": 0.5190468963423889, + "grad_norm": 2.7637432716296315, + "learning_rate": 9.673016668133487e-07, + "log_odds_chosen": 1.343652367591858, + "log_odds_ratio": -0.3913818299770355, + "logits/chosen": -0.938671886920929, + "logits/rejected": -0.8287109136581421, + "logps/chosen": -0.5897461175918579, + "logps/rejected": -1.456640601158142, + "loss": 0.7817, + "nll_loss": 0.754687488079071, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0589599609375, + "rewards/margins": 0.08671798557043076, + "rewards/rejected": -0.14567871391773224, + "step": 6840 + }, + { + "epoch": 0.5198057368341175, + "grad_norm": 2.941099804178467, + "learning_rate": 9.66595349328283e-07, + "log_odds_chosen": 1.196435570716858, + "log_odds_ratio": -0.4708496034145355, + "logits/chosen": -0.958984375, + "logits/rejected": -0.8564453125, + "logps/chosen": -0.6904296875, + "logps/rejected": -1.497656226158142, + "loss": 0.7827, + "nll_loss": 0.768359363079071, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06904296576976776, + "rewards/margins": 0.08078613132238388, + "rewards/rejected": -0.1500244140625, + "step": 6850 + }, + { + "epoch": 0.5205645773258462, + "grad_norm": 3.353435743055801, + "learning_rate": 9.658905768325902e-07, + "log_odds_chosen": 1.538476586341858, + "log_odds_ratio": -0.37773436307907104, + "logits/chosen": -1.01171875, + "logits/rejected": -0.8740234375, + "logps/chosen": -0.6664062738418579, + "logps/rejected": -1.7140624523162842, + "loss": 0.785, + "nll_loss": 0.6810547113418579, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.066650390625, + "rewards/margins": 0.10468444973230362, + "rewards/rejected": -0.17128905653953552, + "step": 6860 + }, + { + "epoch": 0.5213234178175747, + "grad_norm": 2.46441643656723, + "learning_rate": 9.651873437019902e-07, + "log_odds_chosen": 1.263085961341858, + "log_odds_ratio": -0.46015626192092896, + "logits/chosen": -0.918749988079071, + "logits/rejected": -0.8193359375, + "logps/chosen": -0.631640613079071, + "logps/rejected": -1.5158202648162842, + "loss": 0.7856, + "nll_loss": 0.7313476800918579, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06308593600988388, + "rewards/margins": 0.08848877251148224, + "rewards/rejected": -0.15163573622703552, + "step": 6870 + }, + { + "epoch": 0.5220822583093034, + "grad_norm": 2.5169144272910224, + "learning_rate": 9.644856443408243e-07, + "log_odds_chosen": 1.239160180091858, + "log_odds_ratio": -0.4374023377895355, + "logits/chosen": -0.9375, + "logits/rejected": -0.8082031011581421, + "logps/chosen": -0.6591796875, + "logps/rejected": -1.454687476158142, + "loss": 0.8084, + "nll_loss": 0.7230468988418579, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06593017280101776, + "rewards/margins": 0.07955627143383026, + "rewards/rejected": -0.1455078125, + "step": 6880 + }, + { + "epoch": 0.522841098801032, + "grad_norm": 2.7243050305242664, + "learning_rate": 9.637854731818697e-07, + "log_odds_chosen": 1.463281273841858, + "log_odds_ratio": -0.4019531309604645, + "logits/chosen": -0.9173828363418579, + "logits/rejected": -0.8265625238418579, + "logps/chosen": -0.660839855670929, + "logps/rejected": -1.705664038658142, + "loss": 0.8129, + "nll_loss": 0.7754882574081421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06612548977136612, + "rewards/margins": 0.10466308891773224, + "rewards/rejected": -0.17070312798023224, + "step": 6890 + }, + { + "epoch": 0.5235999392927606, + "grad_norm": 2.2732163303788164, + "learning_rate": 9.630868246861536e-07, + "log_odds_chosen": 1.528710961341858, + "log_odds_ratio": -0.37177735567092896, + "logits/chosen": -0.979296863079071, + "logits/rejected": -0.8746093511581421, + "logps/chosen": -0.7076171636581421, + "logps/rejected": -1.7531249523162842, + "loss": 0.7604, + "nll_loss": 0.7767578363418579, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.07081298530101776, + "rewards/margins": 0.10456542670726776, + "rewards/rejected": -0.17521972954273224, + "step": 6900 + }, + { + "epoch": 0.5243587797844893, + "grad_norm": 2.6619644677274734, + "learning_rate": 9.623896933427685e-07, + "log_odds_chosen": 1.188623070716858, + "log_odds_ratio": -0.482666015625, + "logits/chosen": -0.908984363079071, + "logits/rejected": -0.841015636920929, + "logps/chosen": -0.660937488079071, + "logps/rejected": -1.4660155773162842, + "loss": 0.793, + "nll_loss": 0.7635742425918579, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06613769382238388, + "rewards/margins": 0.0804443359375, + "rewards/rejected": -0.14658203721046448, + "step": 6910 + }, + { + "epoch": 0.5251176202762179, + "grad_norm": 7.819901412826592, + "learning_rate": 9.6169407366869e-07, + "log_odds_chosen": 1.4423828125, + "log_odds_ratio": -0.40668946504592896, + "logits/chosen": -0.9076172113418579, + "logits/rejected": -0.7935546636581421, + "logps/chosen": -0.6460937261581421, + "logps/rejected": -1.642968773841858, + "loss": 0.7908, + "nll_loss": 0.774609386920929, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06459961086511612, + "rewards/margins": 0.09953613579273224, + "rewards/rejected": -0.16401366889476776, + "step": 6920 + }, + { + "epoch": 0.5258764607679466, + "grad_norm": 2.7180232459008815, + "learning_rate": 9.609999602085963e-07, + "log_odds_chosen": 1.313574194908142, + "log_odds_ratio": -0.4557128846645355, + "logits/chosen": -0.9195312261581421, + "logits/rejected": -0.842968761920929, + "logps/chosen": -0.716796875, + "logps/rejected": -1.6369140148162842, + "loss": 0.7905, + "nll_loss": 0.8287109136581421, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07176513969898224, + "rewards/margins": 0.09190063178539276, + "rewards/rejected": -0.16350097954273224, + "step": 6930 + }, + { + "epoch": 0.5266353012596752, + "grad_norm": 3.1306929570721507, + "learning_rate": 9.603073475346872e-07, + "log_odds_chosen": 1.4545409679412842, + "log_odds_ratio": -0.40385740995407104, + "logits/chosen": -0.9380859136581421, + "logits/rejected": -0.819531261920929, + "logps/chosen": -0.68017578125, + "logps/rejected": -1.6746094226837158, + "loss": 0.7622, + "nll_loss": 0.69921875, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06804199516773224, + "rewards/margins": 0.09943847358226776, + "rewards/rejected": -0.16750487685203552, + "step": 6940 + }, + { + "epoch": 0.5273941417514039, + "grad_norm": 2.7259073532868507, + "learning_rate": 9.596162302465074e-07, + "log_odds_chosen": 1.589257836341858, + "log_odds_ratio": -0.380615234375, + "logits/chosen": -0.947460949420929, + "logits/rejected": -0.8343750238418579, + "logps/chosen": -0.6329101324081421, + "logps/rejected": -1.7296874523162842, + "loss": 0.7846, + "nll_loss": 0.6851562261581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06326904147863388, + "rewards/margins": 0.10976715385913849, + "rewards/rejected": -0.17312011122703552, + "step": 6950 + }, + { + "epoch": 0.5281529822431325, + "grad_norm": 2.4986627117247826, + "learning_rate": 9.589266029707683e-07, + "log_odds_chosen": 1.505859375, + "log_odds_ratio": -0.3985351622104645, + "logits/chosen": -0.9332031011581421, + "logits/rejected": -0.838085949420929, + "logps/chosen": -0.6187499761581421, + "logps/rejected": -1.602148413658142, + "loss": 0.8016, + "nll_loss": 0.7486327886581421, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06187744066119194, + "rewards/margins": 0.09835205227136612, + "rewards/rejected": -0.16022948920726776, + "step": 6960 + }, + { + "epoch": 0.5289118227348611, + "grad_norm": 2.457685121496299, + "learning_rate": 9.582384603611731e-07, + "log_odds_chosen": 1.729882836341858, + "log_odds_ratio": -0.38330078125, + "logits/chosen": -0.966601550579071, + "logits/rejected": -0.871874988079071, + "logps/chosen": -0.6161133050918579, + "logps/rejected": -1.8074219226837158, + "loss": 0.7659, + "nll_loss": 0.720703125, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06159668043255806, + "rewards/margins": 0.11909179389476776, + "rewards/rejected": -0.18083496391773224, + "step": 6970 + }, + { + "epoch": 0.5296706632265897, + "grad_norm": 2.921512624034416, + "learning_rate": 9.575517970982428e-07, + "log_odds_chosen": 1.4179198741912842, + "log_odds_ratio": -0.40620118379592896, + "logits/chosen": -0.9623047113418579, + "logits/rejected": -0.890429675579071, + "logps/chosen": -0.638867199420929, + "logps/rejected": -1.629296898841858, + "loss": 0.7772, + "nll_loss": 0.7080078125, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.0638427734375, + "rewards/margins": 0.09915771335363388, + "rewards/rejected": -0.1629638671875, + "step": 6980 + }, + { + "epoch": 0.5304295037183184, + "grad_norm": 2.8538488287444066, + "learning_rate": 9.568666078891436e-07, + "log_odds_chosen": 1.3582031726837158, + "log_odds_ratio": -0.39306640625, + "logits/chosen": -0.9498046636581421, + "logits/rejected": -0.819531261920929, + "logps/chosen": -0.6591796875, + "logps/rejected": -1.591406226158142, + "loss": 0.7526, + "nll_loss": 0.7904297113418579, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06595458835363388, + "rewards/margins": 0.09334106743335724, + "rewards/rejected": -0.15913085639476776, + "step": 6990 + }, + { + "epoch": 0.531188344210047, + "grad_norm": 2.471161964707584, + "learning_rate": 9.561828874675149e-07, + "log_odds_chosen": 1.565820336341858, + "log_odds_ratio": -0.38359373807907104, + "logits/chosen": -0.9423828125, + "logits/rejected": -0.802929699420929, + "logps/chosen": -0.648242175579071, + "logps/rejected": -1.710351586341858, + "loss": 0.7897, + "nll_loss": 0.7093750238418579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06485595554113388, + "rewards/margins": 0.10623779147863388, + "rewards/rejected": -0.17106933891773224, + "step": 7000 + }, + { + "epoch": 0.5319471847017757, + "grad_norm": 2.504221714063816, + "learning_rate": 9.555006305933e-07, + "log_odds_chosen": 1.356542944908142, + "log_odds_ratio": -0.3905273377895355, + "logits/chosen": -0.898632824420929, + "logits/rejected": -0.8265625238418579, + "logps/chosen": -0.6851562261581421, + "logps/rejected": -1.6232421398162842, + "loss": 0.7684, + "nll_loss": 0.834765613079071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06850586086511612, + "rewards/margins": 0.09375915676355362, + "rewards/rejected": -0.162353515625, + "step": 7010 + }, + { + "epoch": 0.5327060251935043, + "grad_norm": 2.6455109230239247, + "learning_rate": 9.548198320525771e-07, + "log_odds_chosen": 1.588476538658142, + "log_odds_ratio": -0.39013671875, + "logits/chosen": -0.959179699420929, + "logits/rejected": -0.856640636920929, + "logps/chosen": -0.6739257574081421, + "logps/rejected": -1.765625, + "loss": 0.7577, + "nll_loss": 0.716015636920929, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06734619289636612, + "rewards/margins": 0.10916747897863388, + "rewards/rejected": -0.17644043266773224, + "step": 7020 + }, + { + "epoch": 0.533464865685233, + "grad_norm": 2.665659525877553, + "learning_rate": 9.54140486657392e-07, + "log_odds_chosen": 1.4528319835662842, + "log_odds_ratio": -0.3826660215854645, + "logits/chosen": -0.935546875, + "logits/rejected": -0.815234363079071, + "logps/chosen": -0.613476574420929, + "logps/rejected": -1.526953101158142, + "loss": 0.7832, + "nll_loss": 0.7515624761581421, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06130371242761612, + "rewards/margins": 0.09135131537914276, + "rewards/rejected": -0.15273436903953552, + "step": 7030 + }, + { + "epoch": 0.5342237061769616, + "grad_norm": 3.477850454953123, + "learning_rate": 9.534625892455922e-07, + "log_odds_chosen": 1.335852026939392, + "log_odds_ratio": -0.421142578125, + "logits/chosen": -0.9087890386581421, + "logits/rejected": -0.842968761920929, + "logps/chosen": -0.7191406488418579, + "logps/rejected": -1.6472656726837158, + "loss": 0.7656, + "nll_loss": 0.7748047113418579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.07187499850988388, + "rewards/margins": 0.09270019829273224, + "rewards/rejected": -0.16469725966453552, + "step": 7040 + }, + { + "epoch": 0.5349825466686903, + "grad_norm": 2.5109526690815698, + "learning_rate": 9.527861346806618e-07, + "log_odds_chosen": 1.248388648033142, + "log_odds_ratio": -0.4857421815395355, + "logits/chosen": -0.928906261920929, + "logits/rejected": -0.7884765863418579, + "logps/chosen": -0.633984386920929, + "logps/rejected": -1.505468726158142, + "loss": 0.7672, + "nll_loss": 0.7417968511581421, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06337890774011612, + "rewards/margins": 0.08709106594324112, + "rewards/rejected": -0.15053710341453552, + "step": 7050 + }, + { + "epoch": 0.5357413871604189, + "grad_norm": 2.805501394008409, + "learning_rate": 9.521111178515582e-07, + "log_odds_chosen": 1.509179711341858, + "log_odds_ratio": -0.3857421875, + "logits/chosen": -0.904296875, + "logits/rejected": -0.7822265625, + "logps/chosen": -0.633984386920929, + "logps/rejected": -1.6769530773162842, + "loss": 0.7786, + "nll_loss": 0.756054699420929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06345214694738388, + "rewards/margins": 0.104248046875, + "rewards/rejected": -0.16767577826976776, + "step": 7060 + }, + { + "epoch": 0.5365002276521476, + "grad_norm": 2.523528338253233, + "learning_rate": 9.514375336725502e-07, + "log_odds_chosen": 1.3629882335662842, + "log_odds_ratio": -0.42631834745407104, + "logits/chosen": -0.9833984375, + "logits/rejected": -0.8382812738418579, + "logps/chosen": -0.656445324420929, + "logps/rejected": -1.5763671398162842, + "loss": 0.7856, + "nll_loss": 0.67919921875, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.065673828125, + "rewards/margins": 0.09210205078125, + "rewards/rejected": -0.15781250596046448, + "step": 7070 + }, + { + "epoch": 0.5372590681438761, + "grad_norm": 2.347057885123951, + "learning_rate": 9.507653770830566e-07, + "log_odds_chosen": 1.683508276939392, + "log_odds_ratio": -0.36762696504592896, + "logits/chosen": -0.959179699420929, + "logits/rejected": -0.841015636920929, + "logps/chosen": -0.6412109136581421, + "logps/rejected": -1.8171875476837158, + "loss": 0.7849, + "nll_loss": 0.792187511920929, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06411132961511612, + "rewards/margins": 0.11753387749195099, + "rewards/rejected": -0.18178710341453552, + "step": 7080 + }, + { + "epoch": 0.5380179086356048, + "grad_norm": 2.4445964133345646, + "learning_rate": 9.500946430474869e-07, + "log_odds_chosen": 1.446874976158142, + "log_odds_ratio": -0.37883299589157104, + "logits/chosen": -0.9027343988418579, + "logits/rejected": -0.7875000238418579, + "logps/chosen": -0.636425793170929, + "logps/rejected": -1.626562476158142, + "loss": 0.7882, + "nll_loss": 0.75244140625, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06364746391773224, + "rewards/margins": 0.09907837212085724, + "rewards/rejected": -0.16274413466453552, + "step": 7090 + }, + { + "epoch": 0.5387767491273334, + "grad_norm": 2.730352892938648, + "learning_rate": 9.494253265550825e-07, + "log_odds_chosen": 1.4993164539337158, + "log_odds_ratio": -0.3963867127895355, + "logits/chosen": -0.9150390625, + "logits/rejected": -0.825390636920929, + "logps/chosen": -0.663281261920929, + "logps/rejected": -1.6980469226837158, + "loss": 0.7832, + "nll_loss": 0.739062488079071, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.0662841796875, + "rewards/margins": 0.10351105034351349, + "rewards/rejected": -0.16989746689796448, + "step": 7100 + }, + { + "epoch": 0.5395355896190621, + "grad_norm": 2.45968923221001, + "learning_rate": 9.4875742261976e-07, + "log_odds_chosen": 1.5222656726837158, + "log_odds_ratio": -0.38286131620407104, + "logits/chosen": -0.9779297113418579, + "logits/rejected": -0.851367175579071, + "logps/chosen": -0.6460937261581421, + "logps/rejected": -1.715234398841858, + "loss": 0.7741, + "nll_loss": 0.7432616949081421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06462402641773224, + "rewards/margins": 0.10682372748851776, + "rewards/rejected": -0.1715087890625, + "step": 7110 + }, + { + "epoch": 0.5402944301107907, + "grad_norm": 2.724854084462044, + "learning_rate": 9.480909262799544e-07, + "log_odds_chosen": 1.492040991783142, + "log_odds_ratio": -0.4017578065395355, + "logits/chosen": -0.942187488079071, + "logits/rejected": -0.8265625238418579, + "logps/chosen": -0.655468761920929, + "logps/rejected": -1.6925780773162842, + "loss": 0.7486, + "nll_loss": 0.72607421875, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06557617336511612, + "rewards/margins": 0.10375366359949112, + "rewards/rejected": -0.1693115234375, + "step": 7120 + }, + { + "epoch": 0.5410532706025194, + "grad_norm": 2.494648878304497, + "learning_rate": 9.47425832598465e-07, + "log_odds_chosen": 1.3213622570037842, + "log_odds_ratio": -0.434814453125, + "logits/chosen": -0.93359375, + "logits/rejected": -0.8197265863418579, + "logps/chosen": -0.6761718988418579, + "logps/rejected": -1.625390648841858, + "loss": 0.7838, + "nll_loss": 0.800976574420929, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.067626953125, + "rewards/margins": 0.09482727199792862, + "rewards/rejected": -0.16259765625, + "step": 7130 + }, + { + "epoch": 0.541812111094248, + "grad_norm": 2.6638397848095696, + "learning_rate": 9.467621366623017e-07, + "log_odds_chosen": 1.5519530773162842, + "log_odds_ratio": -0.39545899629592896, + "logits/chosen": -0.921093761920929, + "logits/rejected": -0.7964843511581421, + "logps/chosen": -0.6626952886581421, + "logps/rejected": -1.768945336341858, + "loss": 0.7815, + "nll_loss": 0.726855456829071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06627197563648224, + "rewards/margins": 0.11057128757238388, + "rewards/rejected": -0.17678222060203552, + "step": 7140 + }, + { + "epoch": 0.5425709515859767, + "grad_norm": 2.2888844362962604, + "learning_rate": 9.46099833582532e-07, + "log_odds_chosen": 1.6906249523162842, + "log_odds_ratio": -0.3595214784145355, + "logits/chosen": -0.9134765863418579, + "logits/rejected": -0.8218749761581421, + "logps/chosen": -0.6312500238418579, + "logps/rejected": -1.814062476158142, + "loss": 0.7649, + "nll_loss": 0.70263671875, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.0631103515625, + "rewards/margins": 0.11835937201976776, + "rewards/rejected": -0.18137207627296448, + "step": 7150 + }, + { + "epoch": 0.5433297920777053, + "grad_norm": 3.630002360712264, + "learning_rate": 9.45438918494131e-07, + "log_odds_chosen": 1.832617163658142, + "log_odds_ratio": -0.3507324159145355, + "logits/chosen": -0.929882824420929, + "logits/rejected": -0.8212890625, + "logps/chosen": -0.681835949420929, + "logps/rejected": -2.0171875953674316, + "loss": 0.7708, + "nll_loss": 0.73046875, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06818847358226776, + "rewards/margins": 0.13364258408546448, + "rewards/rejected": -0.20185546576976776, + "step": 7160 + }, + { + "epoch": 0.5440886325694338, + "grad_norm": 2.300448851841177, + "learning_rate": 9.447793865558291e-07, + "log_odds_chosen": 1.4615478515625, + "log_odds_ratio": -0.3848632872104645, + "logits/chosen": -0.942187488079071, + "logits/rejected": -0.8187500238418579, + "logps/chosen": -0.65380859375, + "logps/rejected": -1.6953125, + "loss": 0.7932, + "nll_loss": 0.796093761920929, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06539306789636612, + "rewards/margins": 0.10420074313879013, + "rewards/rejected": -0.16962890326976776, + "step": 7170 + }, + { + "epoch": 0.5448474730611625, + "grad_norm": 2.3612558524621354, + "learning_rate": 9.441212329499659e-07, + "log_odds_chosen": 1.5128905773162842, + "log_odds_ratio": -0.382080078125, + "logits/chosen": -0.932812511920929, + "logits/rejected": -0.801953136920929, + "logps/chosen": -0.6617187261581421, + "logps/rejected": -1.707617163658142, + "loss": 0.7934, + "nll_loss": 0.7515624761581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06614990532398224, + "rewards/margins": 0.10458984225988388, + "rewards/rejected": -0.1707763671875, + "step": 7180 + }, + { + "epoch": 0.5456063135528911, + "grad_norm": 2.5962116569427045, + "learning_rate": 9.434644528823399e-07, + "log_odds_chosen": 1.394384741783142, + "log_odds_ratio": -0.44873046875, + "logits/chosen": -0.956250011920929, + "logits/rejected": -0.8392578363418579, + "logps/chosen": -0.664257824420929, + "logps/rejected": -1.638671875, + "loss": 0.7885, + "nll_loss": 0.7142578363418579, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06644286960363388, + "rewards/margins": 0.09754333645105362, + "rewards/rejected": -0.1640625, + "step": 7190 + }, + { + "epoch": 0.5463651540446198, + "grad_norm": 2.291738804842518, + "learning_rate": 9.428090415820634e-07, + "log_odds_chosen": 1.4025390148162842, + "log_odds_ratio": -0.400390625, + "logits/chosen": -0.975781261920929, + "logits/rejected": -0.862500011920929, + "logps/chosen": -0.6993163824081421, + "logps/rejected": -1.662500023841858, + "loss": 0.7892, + "nll_loss": 0.7564452886581421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06992187350988388, + "rewards/margins": 0.09614257514476776, + "rewards/rejected": -0.16618652641773224, + "step": 7200 + }, + { + "epoch": 0.5471239945363484, + "grad_norm": 2.857177751327088, + "learning_rate": 9.42154994301416e-07, + "log_odds_chosen": 1.5506713390350342, + "log_odds_ratio": -0.35979002714157104, + "logits/chosen": -0.9984375238418579, + "logits/rejected": -0.8648437261581421, + "logps/chosen": -0.637499988079071, + "logps/rejected": -1.7216796875, + "loss": 0.7661, + "nll_loss": 0.670214831829071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06369628757238388, + "rewards/margins": 0.10839919745922089, + "rewards/rejected": -0.17219237983226776, + "step": 7210 + }, + { + "epoch": 0.5478828350280771, + "grad_norm": 2.8996143660225693, + "learning_rate": 9.415023063157008e-07, + "log_odds_chosen": 1.229150414466858, + "log_odds_ratio": -0.4491210877895355, + "logits/chosen": -0.9974609613418579, + "logits/rejected": -0.8580077886581421, + "logps/chosen": -0.6839843988418579, + "logps/rejected": -1.5398437976837158, + "loss": 0.7583, + "nll_loss": 0.759082019329071, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.06840820610523224, + "rewards/margins": 0.08558349311351776, + "rewards/rejected": -0.15397949516773224, + "step": 7220 + }, + { + "epoch": 0.5486416755198057, + "grad_norm": 3.03582504638056, + "learning_rate": 9.408509729231009e-07, + "log_odds_chosen": 1.4357421398162842, + "log_odds_ratio": -0.37236326932907104, + "logits/chosen": -0.938281238079071, + "logits/rejected": -0.83984375, + "logps/chosen": -0.6533203125, + "logps/rejected": -1.646875023841858, + "loss": 0.7786, + "nll_loss": 0.7642577886581421, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06527099758386612, + "rewards/margins": 0.09946288913488388, + "rewards/rejected": -0.164794921875, + "step": 7230 + }, + { + "epoch": 0.5494005160115344, + "grad_norm": 2.2835769534267847, + "learning_rate": 9.402009894445369e-07, + "log_odds_chosen": 1.3484375476837158, + "log_odds_ratio": -0.4205566346645355, + "logits/chosen": -0.9837890863418579, + "logits/rejected": -0.84765625, + "logps/chosen": -0.67822265625, + "logps/rejected": -1.6003906726837158, + "loss": 0.7593, + "nll_loss": 0.708789050579071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06776122748851776, + "rewards/margins": 0.092132568359375, + "rewards/rejected": -0.16008301079273224, + "step": 7240 + }, + { + "epoch": 0.550159356503263, + "grad_norm": 2.5675895626757943, + "learning_rate": 9.395523512235255e-07, + "log_odds_chosen": 1.58642578125, + "log_odds_ratio": -0.3625244200229645, + "logits/chosen": -0.9984375238418579, + "logits/rejected": -0.8628906011581421, + "logps/chosen": -0.6869140863418579, + "logps/rejected": -1.8132812976837158, + "loss": 0.8094, + "nll_loss": 0.8158203363418579, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.0687255859375, + "rewards/margins": 0.11270751804113388, + "rewards/rejected": -0.18154296278953552, + "step": 7250 + }, + { + "epoch": 0.5509181969949917, + "grad_norm": 2.8513970220491127, + "learning_rate": 9.389050536260404e-07, + "log_odds_chosen": 1.2263672351837158, + "log_odds_ratio": -0.4541015625, + "logits/chosen": -0.9242187738418579, + "logits/rejected": -0.8033202886581421, + "logps/chosen": -0.70263671875, + "logps/rejected": -1.5343749523162842, + "loss": 0.7804, + "nll_loss": 0.833789050579071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.07023926079273224, + "rewards/margins": 0.08338622748851776, + "rewards/rejected": -0.153564453125, + "step": 7260 + }, + { + "epoch": 0.5516770374867203, + "grad_norm": 2.6185408123150844, + "learning_rate": 9.382590920403722e-07, + "log_odds_chosen": 1.608789086341858, + "log_odds_ratio": -0.36479490995407104, + "logits/chosen": -0.9400390386581421, + "logits/rejected": -0.85546875, + "logps/chosen": -0.60595703125, + "logps/rejected": -1.7121093273162842, + "loss": 0.7618, + "nll_loss": 0.6788085699081421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06058349460363388, + "rewards/margins": 0.11055908352136612, + "rewards/rejected": -0.171142578125, + "step": 7270 + }, + { + "epoch": 0.552435877978449, + "grad_norm": 2.333030605374203, + "learning_rate": 9.376144618769908e-07, + "log_odds_chosen": 1.5675780773162842, + "log_odds_ratio": -0.3802246153354645, + "logits/chosen": -1.0050780773162842, + "logits/rejected": -0.8599609136581421, + "logps/chosen": -0.6201171875, + "logps/rejected": -1.7009766101837158, + "loss": 0.7653, + "nll_loss": 0.7071288824081421, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06198730319738388, + "rewards/margins": 0.10809326171875, + "rewards/rejected": -0.17014160752296448, + "step": 7280 + }, + { + "epoch": 0.5531947184701775, + "grad_norm": 2.6131519079953214, + "learning_rate": 9.369711585684086e-07, + "log_odds_chosen": 1.2822265625, + "log_odds_ratio": -0.44013673067092896, + "logits/chosen": -0.9886718988418579, + "logits/rejected": -0.8525390625, + "logps/chosen": -0.662304699420929, + "logps/rejected": -1.5626952648162842, + "loss": 0.7707, + "nll_loss": 0.72900390625, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06623534858226776, + "rewards/margins": 0.09002685546875, + "rewards/rejected": -0.15632323920726776, + "step": 7290 + }, + { + "epoch": 0.5539535589619062, + "grad_norm": 2.602124099770009, + "learning_rate": 9.363291775690445e-07, + "log_odds_chosen": 1.3025391101837158, + "log_odds_ratio": -0.42451173067092896, + "logits/chosen": -0.970507800579071, + "logits/rejected": -0.833203136920929, + "logps/chosen": -0.655468761920929, + "logps/rejected": -1.5193359851837158, + "loss": 0.7686, + "nll_loss": 0.752734363079071, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06553955376148224, + "rewards/margins": 0.08654022216796875, + "rewards/rejected": -0.15200194716453552, + "step": 7300 + }, + { + "epoch": 0.5547123994536348, + "grad_norm": 2.541079998732178, + "learning_rate": 9.356885143550886e-07, + "log_odds_chosen": 1.4660155773162842, + "log_odds_ratio": -0.374267578125, + "logits/chosen": -0.9468749761581421, + "logits/rejected": -0.843945324420929, + "logps/chosen": -0.6307617425918579, + "logps/rejected": -1.593164086341858, + "loss": 0.7722, + "nll_loss": 0.7533203363418579, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06306152045726776, + "rewards/margins": 0.09632568061351776, + "rewards/rejected": -0.15939942002296448, + "step": 7310 + }, + { + "epoch": 0.5554712399453635, + "grad_norm": 2.8886481274368596, + "learning_rate": 9.350491644243688e-07, + "log_odds_chosen": 1.4816405773162842, + "log_odds_ratio": -0.40327149629592896, + "logits/chosen": -0.955078125, + "logits/rejected": -0.848437488079071, + "logps/chosen": -0.647167980670929, + "logps/rejected": -1.68359375, + "loss": 0.7802, + "nll_loss": 0.7841796875, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06475830078125, + "rewards/margins": 0.10378418117761612, + "rewards/rejected": -0.16840820014476776, + "step": 7320 + }, + { + "epoch": 0.5562300804370921, + "grad_norm": 4.859820064597355, + "learning_rate": 9.344111232962179e-07, + "log_odds_chosen": 1.6296875476837158, + "log_odds_ratio": -0.3758789002895355, + "logits/chosen": -0.9189453125, + "logits/rejected": -0.7923828363418579, + "logps/chosen": -0.6424804925918579, + "logps/rejected": -1.816015601158142, + "loss": 0.783, + "nll_loss": 0.7422851324081421, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06423339992761612, + "rewards/margins": 0.11756591498851776, + "rewards/rejected": -0.18178710341453552, + "step": 7330 + }, + { + "epoch": 0.5569889209288208, + "grad_norm": 2.743787058516089, + "learning_rate": 9.337743865113415e-07, + "log_odds_chosen": 1.3921387195587158, + "log_odds_ratio": -0.435791015625, + "logits/chosen": -0.884765625, + "logits/rejected": -0.790820300579071, + "logps/chosen": -0.650683581829071, + "logps/rejected": -1.6320312023162842, + "loss": 0.7939, + "nll_loss": 0.771679699420929, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06510009616613388, + "rewards/margins": 0.09806518256664276, + "rewards/rejected": -0.16325683891773224, + "step": 7340 + }, + { + "epoch": 0.5577477614205494, + "grad_norm": 2.5182450847624196, + "learning_rate": 9.331389496316868e-07, + "log_odds_chosen": 1.4314453601837158, + "log_odds_ratio": -0.4581054747104645, + "logits/chosen": -0.9527343511581421, + "logits/rejected": -0.865039050579071, + "logps/chosen": -0.679882824420929, + "logps/rejected": -1.7507812976837158, + "loss": 0.7553, + "nll_loss": 0.7884765863418579, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.0679931640625, + "rewards/margins": 0.10709228366613388, + "rewards/rejected": -0.17495116591453552, + "step": 7350 + }, + { + "epoch": 0.5585066019122781, + "grad_norm": 2.6070795396854747, + "learning_rate": 9.325048082403138e-07, + "log_odds_chosen": 1.6416015625, + "log_odds_ratio": -0.3251586854457855, + "logits/chosen": -1.013085961341858, + "logits/rejected": -0.880664050579071, + "logps/chosen": -0.631054699420929, + "logps/rejected": -1.7824218273162842, + "loss": 0.7724, + "nll_loss": 0.7201172113418579, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06306152045726776, + "rewards/margins": 0.11510010063648224, + "rewards/rejected": -0.17802734673023224, + "step": 7360 + }, + { + "epoch": 0.5592654424040067, + "grad_norm": 2.4632773844481686, + "learning_rate": 9.318719579412648e-07, + "log_odds_chosen": 1.5597655773162842, + "log_odds_ratio": -0.36591798067092896, + "logits/chosen": -1.0085937976837158, + "logits/rejected": -0.874804675579071, + "logps/chosen": -0.6073242425918579, + "logps/rejected": -1.6640625, + "loss": 0.7795, + "nll_loss": 0.679003894329071, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06072998046875, + "rewards/margins": 0.10567627102136612, + "rewards/rejected": -0.16643066704273224, + "step": 7370 + }, + { + "epoch": 0.5600242828957354, + "grad_norm": 2.7971555454862536, + "learning_rate": 9.312403943594374e-07, + "log_odds_chosen": 1.55517578125, + "log_odds_ratio": -0.37092286348342896, + "logits/chosen": -0.9365234375, + "logits/rejected": -0.8089843988418579, + "logps/chosen": -0.6265624761581421, + "logps/rejected": -1.689843773841858, + "loss": 0.76, + "nll_loss": 0.6898437738418579, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06264648586511612, + "rewards/margins": 0.10626526176929474, + "rewards/rejected": -0.16887207329273224, + "step": 7380 + }, + { + "epoch": 0.5607831233874639, + "grad_norm": 2.18946765104726, + "learning_rate": 9.306101131404582e-07, + "log_odds_chosen": 1.263281226158142, + "log_odds_ratio": -0.455810546875, + "logits/chosen": -0.939453125, + "logits/rejected": -0.8482421636581421, + "logps/chosen": -0.708691418170929, + "logps/rejected": -1.584375023841858, + "loss": 0.7792, + "nll_loss": 0.738574206829071, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.07081298530101776, + "rewards/margins": 0.08762206882238388, + "rewards/rejected": -0.15837402641773224, + "step": 7390 + }, + { + "epoch": 0.5615419638791926, + "grad_norm": 2.4849622577410293, + "learning_rate": 9.299811099505542e-07, + "log_odds_chosen": 1.4933593273162842, + "log_odds_ratio": -0.37299805879592896, + "logits/chosen": -0.9720703363418579, + "logits/rejected": -0.8687499761581421, + "logps/chosen": -0.6246093511581421, + "logps/rejected": -1.637109398841858, + "loss": 0.7551, + "nll_loss": 0.712109386920929, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06248779222369194, + "rewards/margins": 0.10123290866613388, + "rewards/rejected": -0.16374512016773224, + "step": 7400 + }, + { + "epoch": 0.5623008043709212, + "grad_norm": 2.7129427140327294, + "learning_rate": 9.293533804764305e-07, + "log_odds_chosen": 1.316308617591858, + "log_odds_ratio": -0.411376953125, + "logits/chosen": -1.0128905773162842, + "logits/rejected": -0.899218738079071, + "logps/chosen": -0.623046875, + "logps/rejected": -1.483789086341858, + "loss": 0.8028, + "nll_loss": 0.684863269329071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06226806715130806, + "rewards/margins": 0.08603973686695099, + "rewards/rejected": -0.14833983778953552, + "step": 7410 + }, + { + "epoch": 0.5630596448626499, + "grad_norm": 2.9744175857021578, + "learning_rate": 9.28726920425144e-07, + "log_odds_chosen": 1.170800805091858, + "log_odds_ratio": -0.4571777284145355, + "logits/chosen": -0.938281238079071, + "logits/rejected": -0.8373047113418579, + "logps/chosen": -0.6426757574081421, + "logps/rejected": -1.4669921398162842, + "loss": 0.7705, + "nll_loss": 0.788378894329071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06427001953125, + "rewards/margins": 0.08228149265050888, + "rewards/rejected": -0.14655761420726776, + "step": 7420 + }, + { + "epoch": 0.5638184853543785, + "grad_norm": 2.5004492863317953, + "learning_rate": 9.281017255239815e-07, + "log_odds_chosen": 1.53466796875, + "log_odds_ratio": -0.35053712129592896, + "logits/chosen": -0.9234374761581421, + "logits/rejected": -0.8046875, + "logps/chosen": -0.594042956829071, + "logps/rejected": -1.5988280773162842, + "loss": 0.7576, + "nll_loss": 0.6942383050918579, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.05938720703125, + "rewards/margins": 0.10055847465991974, + "rewards/rejected": -0.15988770127296448, + "step": 7430 + }, + { + "epoch": 0.5645773258461072, + "grad_norm": 2.6145819020877896, + "learning_rate": 9.274777915203365e-07, + "log_odds_chosen": 1.552148461341858, + "log_odds_ratio": -0.4241699278354645, + "logits/chosen": -0.975390613079071, + "logits/rejected": -0.815234363079071, + "logps/chosen": -0.653613269329071, + "logps/rejected": -1.755468726158142, + "loss": 0.7526, + "nll_loss": 0.716601550579071, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06539306789636612, + "rewards/margins": 0.1102294921875, + "rewards/rejected": -0.17563477158546448, + "step": 7440 + }, + { + "epoch": 0.5653361663378358, + "grad_norm": 2.46646475946503, + "learning_rate": 9.268551141815875e-07, + "log_odds_chosen": 1.64697265625, + "log_odds_ratio": -0.33745115995407104, + "logits/chosen": -1.0177733898162842, + "logits/rejected": -0.8833984136581421, + "logps/chosen": -0.5980468988418579, + "logps/rejected": -1.7195312976837158, + "loss": 0.766, + "nll_loss": 0.709765613079071, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.05979003757238388, + "rewards/margins": 0.11211548000574112, + "rewards/rejected": -0.1719970703125, + "step": 7450 + }, + { + "epoch": 0.5660950068295644, + "grad_norm": 3.2265705628290307, + "learning_rate": 9.262336892949784e-07, + "log_odds_chosen": 1.1706054210662842, + "log_odds_ratio": -0.4654296934604645, + "logits/chosen": -0.9546874761581421, + "logits/rejected": -0.8480468988418579, + "logps/chosen": -0.6761718988418579, + "logps/rejected": -1.470312476158142, + "loss": 0.7954, + "nll_loss": 0.75146484375, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06760253757238388, + "rewards/margins": 0.07932128757238388, + "rewards/rejected": -0.14697265625, + "step": 7460 + }, + { + "epoch": 0.5668538473212931, + "grad_norm": 9.852639130727077, + "learning_rate": 9.256135126674977e-07, + "log_odds_chosen": 1.3734862804412842, + "log_odds_ratio": -0.4510742127895355, + "logits/chosen": -0.8988281488418579, + "logits/rejected": -0.78515625, + "logps/chosen": -0.650683581829071, + "logps/rejected": -1.618749976158142, + "loss": 0.7742, + "nll_loss": 0.7337890863418579, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06513671576976776, + "rewards/margins": 0.09698486328125, + "rewards/rejected": -0.16196289658546448, + "step": 7470 + }, + { + "epoch": 0.5676126878130217, + "grad_norm": 2.994123638724084, + "learning_rate": 9.249945801257605e-07, + "log_odds_chosen": 1.362207055091858, + "log_odds_ratio": -0.4122558534145355, + "logits/chosen": -0.955273449420929, + "logits/rejected": -0.841015636920929, + "logps/chosen": -0.645800769329071, + "logps/rejected": -1.5632812976837158, + "loss": 0.7589, + "nll_loss": 0.7157226800918579, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06458739936351776, + "rewards/margins": 0.09183349460363388, + "rewards/rejected": -0.15629883110523224, + "step": 7480 + }, + { + "epoch": 0.5683715283047504, + "grad_norm": 2.704469295825329, + "learning_rate": 9.243768875158902e-07, + "log_odds_chosen": 1.5084228515625, + "log_odds_ratio": -0.3988281190395355, + "logits/chosen": -0.962109386920929, + "logits/rejected": -0.81640625, + "logps/chosen": -0.622265636920929, + "logps/rejected": -1.668554663658142, + "loss": 0.7624, + "nll_loss": 0.683886706829071, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06221923977136612, + "rewards/margins": 0.10472564399242401, + "rewards/rejected": -0.16682128608226776, + "step": 7490 + }, + { + "epoch": 0.5691303687964789, + "grad_norm": 2.655294393971407, + "learning_rate": 9.23760430703401e-07, + "log_odds_chosen": 1.625390648841858, + "log_odds_ratio": -0.3265624940395355, + "logits/chosen": -0.97265625, + "logits/rejected": -0.8177734613418579, + "logps/chosen": -0.593945324420929, + "logps/rejected": -1.656640648841858, + "loss": 0.758, + "nll_loss": 0.705078125, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.05936279147863388, + "rewards/margins": 0.10634765774011612, + "rewards/rejected": -0.16572265326976776, + "step": 7500 + }, + { + "epoch": 0.5698892092882076, + "grad_norm": 2.810283775483581, + "learning_rate": 9.231452055730832e-07, + "log_odds_chosen": 1.548437476158142, + "log_odds_ratio": -0.3837890625, + "logits/chosen": -0.976757824420929, + "logits/rejected": -0.861132800579071, + "logps/chosen": -0.6836913824081421, + "logps/rejected": -1.798437476158142, + "loss": 0.7585, + "nll_loss": 0.7183593511581421, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06831054389476776, + "rewards/margins": 0.1116943359375, + "rewards/rejected": -0.18007811903953552, + "step": 7510 + }, + { + "epoch": 0.5706480497799362, + "grad_norm": 2.4341374442114714, + "learning_rate": 9.225312080288851e-07, + "log_odds_chosen": 1.733007788658142, + "log_odds_ratio": -0.354248046875, + "logits/chosen": -0.978710949420929, + "logits/rejected": -0.8564453125, + "logps/chosen": -0.599902331829071, + "logps/rejected": -1.8406250476837158, + "loss": 0.7651, + "nll_loss": 0.686328113079071, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06000976637005806, + "rewards/margins": 0.12403564155101776, + "rewards/rejected": -0.18405762314796448, + "step": 7520 + }, + { + "epoch": 0.5714068902716649, + "grad_norm": 2.199907644850221, + "learning_rate": 9.219184339938013e-07, + "log_odds_chosen": 1.6378905773162842, + "log_odds_ratio": -0.39191895723342896, + "logits/chosen": -0.9605468511581421, + "logits/rejected": -0.8564453125, + "logps/chosen": -0.6197265386581421, + "logps/rejected": -1.7966797351837158, + "loss": 0.7591, + "nll_loss": 0.7176758050918579, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06201171875, + "rewards/margins": 0.11766357719898224, + "rewards/rejected": -0.1795654296875, + "step": 7530 + }, + { + "epoch": 0.5721657307633935, + "grad_norm": 2.9140947393774304, + "learning_rate": 9.213068794097574e-07, + "log_odds_chosen": 1.7209961414337158, + "log_odds_ratio": -0.3708252012729645, + "logits/chosen": -1.0173828601837158, + "logits/rejected": -0.905078113079071, + "logps/chosen": -0.659375011920929, + "logps/rejected": -1.873046875, + "loss": 0.7672, + "nll_loss": 0.686328113079071, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06590576469898224, + "rewards/margins": 0.12152099609375, + "rewards/rejected": -0.1875, + "step": 7540 + }, + { + "epoch": 0.5729245712551222, + "grad_norm": 3.004383231876261, + "learning_rate": 9.206965402374975e-07, + "log_odds_chosen": 1.6552734375, + "log_odds_ratio": -0.3800292909145355, + "logits/chosen": -0.938281238079071, + "logits/rejected": -0.8626953363418579, + "logps/chosen": -0.6751953363418579, + "logps/rejected": -1.8523437976837158, + "loss": 0.7652, + "nll_loss": 0.758105456829071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06746826320886612, + "rewards/margins": 0.11782226711511612, + "rewards/rejected": -0.18540039658546448, + "step": 7550 + }, + { + "epoch": 0.5736834117468508, + "grad_norm": 2.3547857415398457, + "learning_rate": 9.200874124564723e-07, + "log_odds_chosen": 1.6642577648162842, + "log_odds_ratio": -0.36689454317092896, + "logits/chosen": -0.99609375, + "logits/rejected": -0.8695312738418579, + "logps/chosen": -0.641894519329071, + "logps/rejected": -1.8214843273162842, + "loss": 0.763, + "nll_loss": 0.710742175579071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06413574516773224, + "rewards/margins": 0.1180419921875, + "rewards/rejected": -0.18212890625, + "step": 7560 + }, + { + "epoch": 0.5744422522385795, + "grad_norm": 2.772280051032957, + "learning_rate": 9.194794920647274e-07, + "log_odds_chosen": 1.484375, + "log_odds_ratio": -0.36372071504592896, + "logits/chosen": -0.9947265386581421, + "logits/rejected": -0.8666015863418579, + "logps/chosen": -0.5865234136581421, + "logps/rejected": -1.560937523841858, + "loss": 0.7458, + "nll_loss": 0.6258789300918579, + "rewards/accuracies": 0.8687499761581421, + "rewards/chosen": -0.05865478515625, + "rewards/margins": 0.097503662109375, + "rewards/rejected": -0.15605469048023224, + "step": 7570 + }, + { + "epoch": 0.5752010927303081, + "grad_norm": 3.1964210191522557, + "learning_rate": 9.188727750787932e-07, + "log_odds_chosen": 1.5912597179412842, + "log_odds_ratio": -0.37580567598342896, + "logits/chosen": -0.953125, + "logits/rejected": -0.8505859375, + "logps/chosen": -0.6123046875, + "logps/rejected": -1.6921875476837158, + "loss": 0.7539, + "nll_loss": 0.7627929449081421, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06124267727136612, + "rewards/margins": 0.10802002251148224, + "rewards/rejected": -0.16933593153953552, + "step": 7580 + }, + { + "epoch": 0.5759599332220368, + "grad_norm": 2.6368306769769903, + "learning_rate": 9.182672575335757e-07, + "log_odds_chosen": 1.7109375, + "log_odds_ratio": -0.3613037168979645, + "logits/chosen": -1.008398413658142, + "logits/rejected": -0.8812500238418579, + "logps/chosen": -0.64404296875, + "logps/rejected": -1.81640625, + "loss": 0.7541, + "nll_loss": 0.708691418170929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.0643310546875, + "rewards/margins": 0.11726073920726776, + "rewards/rejected": -0.18166503310203552, + "step": 7590 + }, + { + "epoch": 0.5767187737137653, + "grad_norm": 4.500822297677708, + "learning_rate": 9.176629354822469e-07, + "log_odds_chosen": 1.347070336341858, + "log_odds_ratio": -0.45947265625, + "logits/chosen": -0.9664062261581421, + "logits/rejected": -0.8373047113418579, + "logps/chosen": -0.69189453125, + "logps/rejected": -1.6886718273162842, + "loss": 0.7851, + "nll_loss": 0.711718738079071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06920166313648224, + "rewards/margins": 0.09984131157398224, + "rewards/rejected": -0.16896972060203552, + "step": 7600 + }, + { + "epoch": 0.577477614205494, + "grad_norm": 3.035540435406703, + "learning_rate": 9.170598049961371e-07, + "log_odds_chosen": 1.3517577648162842, + "log_odds_ratio": -0.42626953125, + "logits/chosen": -0.9671875238418579, + "logits/rejected": -0.853710949420929, + "logps/chosen": -0.6034179925918579, + "logps/rejected": -1.5107421875, + "loss": 0.7478, + "nll_loss": 0.6773437261581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06031494215130806, + "rewards/margins": 0.09078369289636612, + "rewards/rejected": -0.15097656846046448, + "step": 7610 + }, + { + "epoch": 0.5782364546972226, + "grad_norm": 2.9484914911615463, + "learning_rate": 9.164578621646276e-07, + "log_odds_chosen": 1.277441382408142, + "log_odds_ratio": -0.4228515625, + "logits/chosen": -0.9839843511581421, + "logits/rejected": -0.8873046636581421, + "logps/chosen": -0.642578125, + "logps/rejected": -1.5056641101837158, + "loss": 0.76, + "nll_loss": 0.679980456829071, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.06428222358226776, + "rewards/margins": 0.08623047173023224, + "rewards/rejected": -0.1505126953125, + "step": 7620 + }, + { + "epoch": 0.5789952951889513, + "grad_norm": 2.819793441355401, + "learning_rate": 9.15857103095044e-07, + "log_odds_chosen": 1.2861816883087158, + "log_odds_ratio": -0.4524902403354645, + "logits/chosen": -0.9585937261581421, + "logits/rejected": -0.8486328125, + "logps/chosen": -0.722363293170929, + "logps/rejected": -1.591406226158142, + "loss": 0.7638, + "nll_loss": 0.747265636920929, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.07225342094898224, + "rewards/margins": 0.08688811957836151, + "rewards/rejected": -0.15908202528953552, + "step": 7630 + }, + { + "epoch": 0.5797541356806799, + "grad_norm": 2.5153252582631693, + "learning_rate": 9.15257523912551e-07, + "log_odds_chosen": 1.3693358898162842, + "log_odds_ratio": -0.4251953065395355, + "logits/chosen": -0.927539050579071, + "logits/rejected": -0.836132824420929, + "logps/chosen": -0.625683605670929, + "logps/rejected": -1.5832030773162842, + "loss": 0.7611, + "nll_loss": 0.757617175579071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06252441555261612, + "rewards/margins": 0.09589233249425888, + "rewards/rejected": -0.158447265625, + "step": 7640 + }, + { + "epoch": 0.5805129761724086, + "grad_norm": 2.7496077089311344, + "learning_rate": 9.146591207600472e-07, + "log_odds_chosen": 1.626953125, + "log_odds_ratio": -0.3426269590854645, + "logits/chosen": -0.9466797113418579, + "logits/rejected": -0.80078125, + "logps/chosen": -0.6019531488418579, + "logps/rejected": -1.703125, + "loss": 0.7707, + "nll_loss": 0.682324230670929, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06020507961511612, + "rewards/margins": 0.11014404147863388, + "rewards/rejected": -0.17050781846046448, + "step": 7650 + }, + { + "epoch": 0.5812718166641372, + "grad_norm": 2.837450024163419, + "learning_rate": 9.140618897980601e-07, + "log_odds_chosen": 1.63916015625, + "log_odds_ratio": -0.38605958223342896, + "logits/chosen": -0.9662109613418579, + "logits/rejected": -0.820117175579071, + "logps/chosen": -0.69580078125, + "logps/rejected": -1.8357422351837158, + "loss": 0.7624, + "nll_loss": 0.7529296875, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06965331733226776, + "rewards/margins": 0.11403503268957138, + "rewards/rejected": -0.18361815810203552, + "step": 7660 + }, + { + "epoch": 0.5820306571558659, + "grad_norm": 2.7485560693912507, + "learning_rate": 9.134658272046442e-07, + "log_odds_chosen": 1.76171875, + "log_odds_ratio": -0.31635743379592896, + "logits/chosen": -1.0505859851837158, + "logits/rejected": -0.9251953363418579, + "logps/chosen": -0.636035144329071, + "logps/rejected": -1.891015648841858, + "loss": 0.7707, + "nll_loss": 0.6905273199081421, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06361083686351776, + "rewards/margins": 0.125457763671875, + "rewards/rejected": -0.18911132216453552, + "step": 7670 + }, + { + "epoch": 0.5827894976475945, + "grad_norm": 2.348448657361946, + "learning_rate": 9.128709291752768e-07, + "log_odds_chosen": 1.8796875476837158, + "log_odds_ratio": -0.3135009706020355, + "logits/chosen": -0.903515636920929, + "logits/rejected": -0.7607421875, + "logps/chosen": -0.6141601800918579, + "logps/rejected": -1.969140648841858, + "loss": 0.7625, + "nll_loss": 0.6856445074081421, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.06138915941119194, + "rewards/margins": 0.13565674424171448, + "rewards/rejected": -0.197021484375, + "step": 7680 + }, + { + "epoch": 0.5835483381393232, + "grad_norm": 2.8538406403221233, + "learning_rate": 9.122771919227568e-07, + "log_odds_chosen": 1.5810546875, + "log_odds_ratio": -0.3882812559604645, + "logits/chosen": -0.9818359613418579, + "logits/rejected": -0.8955078125, + "logps/chosen": -0.593554675579071, + "logps/rejected": -1.635156273841858, + "loss": 0.7663, + "nll_loss": 0.7206054925918579, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.05935058742761612, + "rewards/margins": 0.1041259765625, + "rewards/rejected": -0.16352538764476776, + "step": 7690 + }, + { + "epoch": 0.5843071786310517, + "grad_norm": 2.5178389786443205, + "learning_rate": 9.116846116771035e-07, + "log_odds_chosen": 1.4381835460662842, + "log_odds_ratio": -0.39301759004592896, + "logits/chosen": -0.951367199420929, + "logits/rejected": -0.854296863079071, + "logps/chosen": -0.6426757574081421, + "logps/rejected": -1.6025390625, + "loss": 0.7505, + "nll_loss": 0.7191406488418579, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06423339992761612, + "rewards/margins": 0.09596557915210724, + "rewards/rejected": -0.16015625, + "step": 7700 + }, + { + "epoch": 0.5850660191227804, + "grad_norm": 2.284825516468035, + "learning_rate": 9.110931846854553e-07, + "log_odds_chosen": 1.4861328601837158, + "log_odds_ratio": -0.4158691465854645, + "logits/chosen": -0.9330078363418579, + "logits/rejected": -0.8111327886581421, + "logps/chosen": -0.707714855670929, + "logps/rejected": -1.7546875476837158, + "loss": 0.7757, + "nll_loss": 0.793749988079071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.07072754204273224, + "rewards/margins": 0.10472412407398224, + "rewards/rejected": -0.17534179985523224, + "step": 7710 + }, + { + "epoch": 0.585824859614509, + "grad_norm": 2.8547151587086987, + "learning_rate": 9.105029072119708e-07, + "log_odds_chosen": 1.75146484375, + "log_odds_ratio": -0.32563477754592896, + "logits/chosen": -0.9185546636581421, + "logits/rejected": -0.8388671875, + "logps/chosen": -0.645214855670929, + "logps/rejected": -1.889062523841858, + "loss": 0.7611, + "nll_loss": 0.713671863079071, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06451416015625, + "rewards/margins": 0.12446288764476776, + "rewards/rejected": -0.18886718153953552, + "step": 7720 + }, + { + "epoch": 0.5865837001062376, + "grad_norm": 2.589481144735132, + "learning_rate": 9.099137755377291e-07, + "log_odds_chosen": 1.5060546398162842, + "log_odds_ratio": -0.37822264432907104, + "logits/chosen": -0.921093761920929, + "logits/rejected": -0.8125, + "logps/chosen": -0.6312500238418579, + "logps/rejected": -1.6433594226837158, + "loss": 0.7639, + "nll_loss": 0.772265613079071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06312255561351776, + "rewards/margins": 0.10126952826976776, + "rewards/rejected": -0.16440430283546448, + "step": 7730 + }, + { + "epoch": 0.5873425405979663, + "grad_norm": 2.842678512806397, + "learning_rate": 9.093257859606311e-07, + "log_odds_chosen": 1.505957007408142, + "log_odds_ratio": -0.39775389432907104, + "logits/chosen": -0.945507824420929, + "logits/rejected": -0.8578125238418579, + "logps/chosen": -0.62548828125, + "logps/rejected": -1.662109375, + "loss": 0.7805, + "nll_loss": 0.772753894329071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06257323920726776, + "rewards/margins": 0.10358276218175888, + "rewards/rejected": -0.16621093451976776, + "step": 7740 + }, + { + "epoch": 0.5881013810896949, + "grad_norm": 2.3845474551251695, + "learning_rate": 9.087389347953037e-07, + "log_odds_chosen": 1.724340796470642, + "log_odds_ratio": -0.38164061307907104, + "logits/chosen": -0.9351562261581421, + "logits/rejected": -0.8374999761581421, + "logps/chosen": -0.64599609375, + "logps/rejected": -1.8640625476837158, + "loss": 0.7421, + "nll_loss": 0.7303711175918579, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06458739936351776, + "rewards/margins": 0.12175750732421875, + "rewards/rejected": -0.18647460639476776, + "step": 7750 + }, + { + "epoch": 0.5888602215814236, + "grad_norm": 3.049628718243713, + "learning_rate": 9.081532183729995e-07, + "log_odds_chosen": 1.3771483898162842, + "log_odds_ratio": -0.43896484375, + "logits/chosen": -0.938281238079071, + "logits/rejected": -0.8255859613418579, + "logps/chosen": -0.696484386920929, + "logps/rejected": -1.634179711341858, + "loss": 0.7616, + "nll_loss": 0.7574218511581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06977538764476776, + "rewards/margins": 0.09376220405101776, + "rewards/rejected": -0.163330078125, + "step": 7760 + }, + { + "epoch": 0.5896190620731522, + "grad_norm": 2.9898675639163703, + "learning_rate": 9.075686330415037e-07, + "log_odds_chosen": 1.56689453125, + "log_odds_ratio": -0.36835938692092896, + "logits/chosen": -0.9673827886581421, + "logits/rejected": -0.8238281011581421, + "logps/chosen": -0.64990234375, + "logps/rejected": -1.7734375, + "loss": 0.7388, + "nll_loss": 0.656054675579071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06495361030101776, + "rewards/margins": 0.11245422065258026, + "rewards/rejected": -0.17746582627296448, + "step": 7770 + }, + { + "epoch": 0.5903779025648809, + "grad_norm": 2.711493764348975, + "learning_rate": 9.069851751650364e-07, + "log_odds_chosen": 1.5798828601837158, + "log_odds_ratio": -0.37016600370407104, + "logits/chosen": -1.006250023841858, + "logits/rejected": -0.880664050579071, + "logps/chosen": -0.5819336175918579, + "logps/rejected": -1.6472656726837158, + "loss": 0.7562, + "nll_loss": 0.6475585699081421, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.05820312350988388, + "rewards/margins": 0.106536865234375, + "rewards/rejected": -0.16489258408546448, + "step": 7780 + }, + { + "epoch": 0.5911367430566095, + "grad_norm": 3.3006891775111766, + "learning_rate": 9.064028411241582e-07, + "log_odds_chosen": 1.5465819835662842, + "log_odds_ratio": -0.3990722596645355, + "logits/chosen": -0.957812488079071, + "logits/rejected": -0.855664074420929, + "logps/chosen": -0.66064453125, + "logps/rejected": -1.7744140625, + "loss": 0.7534, + "nll_loss": 0.7235351800918579, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.0660400390625, + "rewards/margins": 0.11158446967601776, + "rewards/rejected": -0.17741699516773224, + "step": 7790 + }, + { + "epoch": 0.5918955835483382, + "grad_norm": 2.8747180760944993, + "learning_rate": 9.058216273156764e-07, + "log_odds_chosen": 1.385156273841858, + "log_odds_ratio": -0.46162110567092896, + "logits/chosen": -0.9662109613418579, + "logits/rejected": -0.890820324420929, + "logps/chosen": -0.673046886920929, + "logps/rejected": -1.666015625, + "loss": 0.7487, + "nll_loss": 0.728710949420929, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06728515774011612, + "rewards/margins": 0.09935303032398224, + "rewards/rejected": -0.16672363877296448, + "step": 7800 + }, + { + "epoch": 0.5926544240400667, + "grad_norm": 3.1985485855872056, + "learning_rate": 9.052415301525511e-07, + "log_odds_chosen": 1.815039038658142, + "log_odds_ratio": -0.3399902284145355, + "logits/chosen": -0.9580078125, + "logits/rejected": -0.891796886920929, + "logps/chosen": -0.633984386920929, + "logps/rejected": -1.9548828601837158, + "loss": 0.7619, + "nll_loss": 0.7357422113418579, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06337890774011612, + "rewards/margins": 0.132080078125, + "rewards/rejected": -0.19541016221046448, + "step": 7810 + }, + { + "epoch": 0.5934132645317954, + "grad_norm": 2.5089583144596608, + "learning_rate": 9.046625460638012e-07, + "log_odds_chosen": 1.639257788658142, + "log_odds_ratio": -0.359619140625, + "logits/chosen": -0.9945312738418579, + "logits/rejected": -0.8822265863418579, + "logps/chosen": -0.6244140863418579, + "logps/rejected": -1.753320336341858, + "loss": 0.7484, + "nll_loss": 0.73779296875, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06240234524011612, + "rewards/margins": 0.11271972954273224, + "rewards/rejected": -0.17531737685203552, + "step": 7820 + }, + { + "epoch": 0.594172105023524, + "grad_norm": 2.4805759789569914, + "learning_rate": 9.040846714944138e-07, + "log_odds_chosen": 1.455957055091858, + "log_odds_ratio": -0.36015623807907104, + "logits/chosen": -0.991406261920929, + "logits/rejected": -0.8359375, + "logps/chosen": -0.625292956829071, + "logps/rejected": -1.5916016101837158, + "loss": 0.7739, + "nll_loss": 0.70556640625, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.06253661960363388, + "rewards/margins": 0.09675903618335724, + "rewards/rejected": -0.15915527939796448, + "step": 7830 + }, + { + "epoch": 0.5949309455152527, + "grad_norm": 2.821660680248788, + "learning_rate": 9.035079029052513e-07, + "log_odds_chosen": 1.501562476158142, + "log_odds_ratio": -0.38007813692092896, + "logits/chosen": -0.9839843511581421, + "logits/rejected": -0.895703136920929, + "logps/chosen": -0.63818359375, + "logps/rejected": -1.644140601158142, + "loss": 0.7391, + "nll_loss": 0.749218761920929, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06376953423023224, + "rewards/margins": 0.10068969428539276, + "rewards/rejected": -0.1644287109375, + "step": 7840 + }, + { + "epoch": 0.5956897860069813, + "grad_norm": 2.6384636226315252, + "learning_rate": 9.029322367729605e-07, + "log_odds_chosen": 1.4099609851837158, + "log_odds_ratio": -0.41801756620407104, + "logits/chosen": -0.9906250238418579, + "logits/rejected": -0.8970702886581421, + "logps/chosen": -0.678027331829071, + "logps/rejected": -1.642187476158142, + "loss": 0.7719, + "nll_loss": 0.724414050579071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06778564304113388, + "rewards/margins": 0.09635009616613388, + "rewards/rejected": -0.1641845703125, + "step": 7850 + }, + { + "epoch": 0.59644862649871, + "grad_norm": 2.9699807727782486, + "learning_rate": 9.02357669589883e-07, + "log_odds_chosen": 1.49609375, + "log_odds_ratio": -0.40751951932907104, + "logits/chosen": -1.0207030773162842, + "logits/rejected": -0.8851562738418579, + "logps/chosen": -0.633593738079071, + "logps/rejected": -1.6726562976837158, + "loss": 0.7629, + "nll_loss": 0.765820324420929, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06331787258386612, + "rewards/margins": 0.10396728664636612, + "rewards/rejected": -0.167236328125, + "step": 7860 + }, + { + "epoch": 0.5972074669904386, + "grad_norm": 11.36507148656882, + "learning_rate": 9.017841978639643e-07, + "log_odds_chosen": 1.2036621570587158, + "log_odds_ratio": -0.46928709745407104, + "logits/chosen": -0.9251953363418579, + "logits/rejected": -0.84765625, + "logps/chosen": -0.646484375, + "logps/rejected": -1.45703125, + "loss": 0.7505, + "nll_loss": 0.7432616949081421, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06470946967601776, + "rewards/margins": 0.08107452094554901, + "rewards/rejected": -0.14573974907398224, + "step": 7870 + }, + { + "epoch": 0.5979663074821673, + "grad_norm": 2.6416555828030654, + "learning_rate": 9.012118181186658e-07, + "log_odds_chosen": 2.001953125, + "log_odds_ratio": -0.2942871153354645, + "logits/chosen": -0.9624999761581421, + "logits/rejected": -0.8707031011581421, + "logps/chosen": -0.59619140625, + "logps/rejected": -2.0082030296325684, + "loss": 0.7552, + "nll_loss": 0.7476562261581421, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.05964355543255806, + "rewards/margins": 0.14122314751148224, + "rewards/rejected": -0.2008056640625, + "step": 7880 + }, + { + "epoch": 0.5987251479738959, + "grad_norm": 3.7616505179711432, + "learning_rate": 9.00640526892875e-07, + "log_odds_chosen": 1.847265601158142, + "log_odds_ratio": -0.3622070252895355, + "logits/chosen": -0.9478515386581421, + "logits/rejected": -0.8232421875, + "logps/chosen": -0.7217773199081421, + "logps/rejected": -2.071093797683716, + "loss": 0.7506, + "nll_loss": 0.716992199420929, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.07220458984375, + "rewards/margins": 0.13499756157398224, + "rewards/rejected": -0.20712891221046448, + "step": 7890 + }, + { + "epoch": 0.5994839884656246, + "grad_norm": 2.5231640032615923, + "learning_rate": 9.000703207408191e-07, + "log_odds_chosen": 1.419921875, + "log_odds_ratio": -0.41948240995407104, + "logits/chosen": -0.9810546636581421, + "logits/rejected": -0.8177734613418579, + "logps/chosen": -0.7158203125, + "logps/rejected": -1.7648437023162842, + "loss": 0.7354, + "nll_loss": 0.777148425579071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.07145996391773224, + "rewards/margins": 0.10487060248851776, + "rewards/rejected": -0.17634277045726776, + "step": 7900 + }, + { + "epoch": 0.6002428289573531, + "grad_norm": 2.4294931927903898, + "learning_rate": 8.995011962319761e-07, + "log_odds_chosen": 1.544824242591858, + "log_odds_ratio": -0.4046630859375, + "logits/chosen": -0.958789050579071, + "logits/rejected": -0.864062488079071, + "logps/chosen": -0.6341797113418579, + "logps/rejected": -1.691015601158142, + "loss": 0.77, + "nll_loss": 0.7001953125, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06340332329273224, + "rewards/margins": 0.10576172173023224, + "rewards/rejected": -0.16911621391773224, + "step": 7910 + }, + { + "epoch": 0.6010016694490818, + "grad_norm": 2.5209477310139157, + "learning_rate": 8.989331499509894e-07, + "log_odds_chosen": 1.433203101158142, + "log_odds_ratio": -0.3963378965854645, + "logits/chosen": -0.981249988079071, + "logits/rejected": -0.8931640386581421, + "logps/chosen": -0.662792980670929, + "logps/rejected": -1.6417968273162842, + "loss": 0.7434, + "nll_loss": 0.7215820550918579, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06621094048023224, + "rewards/margins": 0.09779052436351776, + "rewards/rejected": -0.16416016221046448, + "step": 7920 + }, + { + "epoch": 0.6017605099408104, + "grad_norm": 2.7461637562871686, + "learning_rate": 8.983661784975812e-07, + "log_odds_chosen": 1.5466797351837158, + "log_odds_ratio": -0.36066895723342896, + "logits/chosen": -0.9654296636581421, + "logits/rejected": -0.8125, + "logps/chosen": -0.6265624761581421, + "logps/rejected": -1.687109351158142, + "loss": 0.7737, + "nll_loss": 0.767382800579071, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06264648586511612, + "rewards/margins": 0.10602416843175888, + "rewards/rejected": -0.16865234076976776, + "step": 7930 + }, + { + "epoch": 0.6025193504325391, + "grad_norm": 2.5384272995695034, + "learning_rate": 8.97800278486467e-07, + "log_odds_chosen": 1.696874976158142, + "log_odds_ratio": -0.33732908964157104, + "logits/chosen": -0.960742175579071, + "logits/rejected": -0.837890625, + "logps/chosen": -0.5869140625, + "logps/rejected": -1.716796875, + "loss": 0.7409, + "nll_loss": 0.6884765625, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.05874023586511612, + "rewards/margins": 0.11293945461511612, + "rewards/rejected": -0.17158202826976776, + "step": 7940 + }, + { + "epoch": 0.6032781909242677, + "grad_norm": 2.493644451819072, + "learning_rate": 8.972354465472708e-07, + "log_odds_chosen": 1.6525390148162842, + "log_odds_ratio": -0.32763671875, + "logits/chosen": -0.9765625, + "logits/rejected": -0.8423827886581421, + "logps/chosen": -0.6025390625, + "logps/rejected": -1.734765648841858, + "loss": 0.7652, + "nll_loss": 0.708300769329071, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06024169921875, + "rewards/margins": 0.11328125, + "rewards/rejected": -0.17360839247703552, + "step": 7950 + }, + { + "epoch": 0.6040370314159964, + "grad_norm": 2.8487068023888122, + "learning_rate": 8.966716793244405e-07, + "log_odds_chosen": 1.623437523841858, + "log_odds_ratio": -0.38330078125, + "logits/chosen": -0.967968761920929, + "logits/rejected": -0.8353515863418579, + "logps/chosen": -0.65625, + "logps/rejected": -1.798828125, + "loss": 0.7419, + "nll_loss": 0.6820312738418579, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06566162407398224, + "rewards/margins": 0.11431884765625, + "rewards/rejected": -0.17983397841453552, + "step": 7960 + }, + { + "epoch": 0.604795871907725, + "grad_norm": 2.5073907734682837, + "learning_rate": 8.96108973477165e-07, + "log_odds_chosen": 1.2568359375, + "log_odds_ratio": -0.4188476502895355, + "logits/chosen": -0.947070300579071, + "logits/rejected": -0.8677734136581421, + "logps/chosen": -0.6322265863418579, + "logps/rejected": -1.4666016101837158, + "loss": 0.7561, + "nll_loss": 0.697460949420929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06324462592601776, + "rewards/margins": 0.08325805515050888, + "rewards/rejected": -0.14653320610523224, + "step": 7970 + }, + { + "epoch": 0.6055547123994537, + "grad_norm": 3.2528673875615794, + "learning_rate": 8.955473256792899e-07, + "log_odds_chosen": 1.5939452648162842, + "log_odds_ratio": -0.3775878846645355, + "logits/chosen": -0.9515625238418579, + "logits/rejected": -0.838671863079071, + "logps/chosen": -0.666796863079071, + "logps/rejected": -1.766015648841858, + "loss": 0.7594, + "nll_loss": 0.7408202886581421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06668701022863388, + "rewards/margins": 0.10986328125, + "rewards/rejected": -0.17658691108226776, + "step": 7980 + }, + { + "epoch": 0.6063135528911823, + "grad_norm": 3.404393843374074, + "learning_rate": 8.949867326192358e-07, + "log_odds_chosen": 1.6707031726837158, + "log_odds_ratio": -0.35498046875, + "logits/chosen": -1.0046875476837158, + "logits/rejected": -0.8548828363418579, + "logps/chosen": -0.5908203125, + "logps/rejected": -1.740625023841858, + "loss": 0.7356, + "nll_loss": 0.6864258050918579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.05908203125, + "rewards/margins": 0.11496581882238388, + "rewards/rejected": -0.17404785752296448, + "step": 7990 + }, + { + "epoch": 0.6070723933829109, + "grad_norm": 2.582456484827545, + "learning_rate": 8.944271909999158e-07, + "log_odds_chosen": 1.7126953601837158, + "log_odds_ratio": -0.37041014432907104, + "logits/chosen": -1.015234351158142, + "logits/rejected": -0.8873046636581421, + "logps/chosen": -0.6416991949081421, + "logps/rejected": -1.874609351158142, + "loss": 0.7337, + "nll_loss": 0.674121081829071, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06411132961511612, + "rewards/margins": 0.12339477241039276, + "rewards/rejected": -0.18747559189796448, + "step": 8000 + }, + { + "epoch": 0.6078312338746396, + "grad_norm": 2.4714549139507325, + "learning_rate": 8.938686975386545e-07, + "log_odds_chosen": 1.644921898841858, + "log_odds_ratio": -0.3622070252895355, + "logits/chosen": -0.9818359613418579, + "logits/rejected": -0.8648437261581421, + "logps/chosen": -0.6283203363418579, + "logps/rejected": -1.785546898841858, + "loss": 0.7388, + "nll_loss": 0.66650390625, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06279297173023224, + "rewards/margins": 0.11582031100988388, + "rewards/rejected": -0.1787109375, + "step": 8010 + }, + { + "epoch": 0.6085900743663681, + "grad_norm": 2.492341880398822, + "learning_rate": 8.933112489671067e-07, + "log_odds_chosen": 1.431249976158142, + "log_odds_ratio": -0.40974122285842896, + "logits/chosen": -0.9873046875, + "logits/rejected": -0.8990234136581421, + "logps/chosen": -0.671093761920929, + "logps/rejected": -1.6572265625, + "loss": 0.7557, + "nll_loss": 0.71630859375, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06705322116613388, + "rewards/margins": 0.09859009087085724, + "rewards/rejected": -0.16560058295726776, + "step": 8020 + }, + { + "epoch": 0.6093489148580968, + "grad_norm": 3.057374669441468, + "learning_rate": 8.927548420311771e-07, + "log_odds_chosen": 1.4990234375, + "log_odds_ratio": -0.3926757872104645, + "logits/chosen": -1.005859375, + "logits/rejected": -0.8863281011581421, + "logps/chosen": -0.6031249761581421, + "logps/rejected": -1.5988280773162842, + "loss": 0.7314, + "nll_loss": 0.7215820550918579, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06029052659869194, + "rewards/margins": 0.09958495944738388, + "rewards/rejected": -0.1597900390625, + "step": 8030 + }, + { + "epoch": 0.6101077553498254, + "grad_norm": 2.774171691721351, + "learning_rate": 8.921994734909409e-07, + "log_odds_chosen": 1.3442871570587158, + "log_odds_ratio": -0.44404298067092896, + "logits/chosen": -0.9140625, + "logits/rejected": -0.859375, + "logps/chosen": -0.675976574420929, + "logps/rejected": -1.6130859851837158, + "loss": 0.7615, + "nll_loss": 0.731640636920929, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06757812201976776, + "rewards/margins": 0.09364929050207138, + "rewards/rejected": -0.16123047471046448, + "step": 8040 + }, + { + "epoch": 0.6108665958415541, + "grad_norm": 2.933552988268182, + "learning_rate": 8.916451401205645e-07, + "log_odds_chosen": 1.589257836341858, + "log_odds_ratio": -0.3824706971645355, + "logits/chosen": -1.013281226158142, + "logits/rejected": -0.8515625, + "logps/chosen": -0.640332043170929, + "logps/rejected": -1.775781273841858, + "loss": 0.7682, + "nll_loss": 0.7958984375, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06406249850988388, + "rewards/margins": 0.11343993991613388, + "rewards/rejected": -0.17744140326976776, + "step": 8050 + }, + { + "epoch": 0.6116254363332827, + "grad_norm": 2.857921231159211, + "learning_rate": 8.91091838708226e-07, + "log_odds_chosen": 1.3947265148162842, + "log_odds_ratio": -0.40986329317092896, + "logits/chosen": -1.0076172351837158, + "logits/rejected": -0.862500011920929, + "logps/chosen": -0.652539074420929, + "logps/rejected": -1.630859375, + "loss": 0.7746, + "nll_loss": 0.7197265625, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06525878608226776, + "rewards/margins": 0.09781493991613388, + "rewards/rejected": -0.16306152939796448, + "step": 8060 + }, + { + "epoch": 0.6123842768250114, + "grad_norm": 2.837229213150366, + "learning_rate": 8.905395660560378e-07, + "log_odds_chosen": 1.504296898841858, + "log_odds_ratio": -0.385986328125, + "logits/chosen": -0.9537109136581421, + "logits/rejected": -0.8519531488418579, + "logps/chosen": -0.6470702886581421, + "logps/rejected": -1.6808593273162842, + "loss": 0.7559, + "nll_loss": 0.692675769329071, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06475830078125, + "rewards/margins": 0.10333251953125, + "rewards/rejected": -0.16806641221046448, + "step": 8070 + }, + { + "epoch": 0.61314311731674, + "grad_norm": 2.4942308815223537, + "learning_rate": 8.899883189799695e-07, + "log_odds_chosen": 1.417578101158142, + "log_odds_ratio": -0.36015623807907104, + "logits/chosen": -0.982617199420929, + "logits/rejected": -0.87890625, + "logps/chosen": -0.6324218511581421, + "logps/rejected": -1.5691406726837158, + "loss": 0.7358, + "nll_loss": 0.7281249761581421, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.06330566108226776, + "rewards/margins": 0.09346923977136612, + "rewards/rejected": -0.1568603515625, + "step": 8080 + }, + { + "epoch": 0.6139019578084687, + "grad_norm": 2.7121767516735242, + "learning_rate": 8.894380943097694e-07, + "log_odds_chosen": 1.674414038658142, + "log_odds_ratio": -0.3631347715854645, + "logits/chosen": -0.968554675579071, + "logits/rejected": -0.856249988079071, + "logps/chosen": -0.62158203125, + "logps/rejected": -1.8341796398162842, + "loss": 0.772, + "nll_loss": 0.726855456829071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06214599683880806, + "rewards/margins": 0.12135620415210724, + "rewards/rejected": -0.18332520127296448, + "step": 8090 + }, + { + "epoch": 0.6146607983001973, + "grad_norm": 2.390556532345029, + "learning_rate": 8.888888888888888e-07, + "log_odds_chosen": 1.4236328601837158, + "log_odds_ratio": -0.4300781190395355, + "logits/chosen": -0.9994140863418579, + "logits/rejected": -0.893359363079071, + "logps/chosen": -0.627246081829071, + "logps/rejected": -1.621484398841858, + "loss": 0.742, + "nll_loss": 0.7144531011581421, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06267090141773224, + "rewards/margins": 0.09934081882238388, + "rewards/rejected": -0.16203613579273224, + "step": 8100 + }, + { + "epoch": 0.615419638791926, + "grad_norm": 2.577970320731745, + "learning_rate": 8.883406995744061e-07, + "log_odds_chosen": 1.5, + "log_odds_ratio": -0.419921875, + "logits/chosen": -0.9488281011581421, + "logits/rejected": -0.860156238079071, + "logps/chosen": -0.6480468511581421, + "logps/rejected": -1.7058594226837158, + "loss": 0.7694, + "nll_loss": 0.743359386920929, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06479492038488388, + "rewards/margins": 0.10576172173023224, + "rewards/rejected": -0.17062988877296448, + "step": 8110 + }, + { + "epoch": 0.6161784792836545, + "grad_norm": 2.447946322540896, + "learning_rate": 8.877935232369506e-07, + "log_odds_chosen": 1.5734374523162842, + "log_odds_ratio": -0.3714843690395355, + "logits/chosen": -1.0812499523162842, + "logits/rejected": -0.9462890625, + "logps/chosen": -0.61181640625, + "logps/rejected": -1.683203101158142, + "loss": 0.7284, + "nll_loss": 0.633496105670929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06105957180261612, + "rewards/margins": 0.10723876953125, + "rewards/rejected": -0.16831055283546448, + "step": 8120 + }, + { + "epoch": 0.6169373197753832, + "grad_norm": 3.306034152526167, + "learning_rate": 8.872473567606276e-07, + "log_odds_chosen": 1.40234375, + "log_odds_ratio": -0.3980956971645355, + "logits/chosen": -0.969531238079071, + "logits/rejected": -0.8519531488418579, + "logps/chosen": -0.6717773675918579, + "logps/rejected": -1.608789086341858, + "loss": 0.7549, + "nll_loss": 0.7886718511581421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06724853813648224, + "rewards/margins": 0.09385986626148224, + "rewards/rejected": -0.16091307997703552, + "step": 8130 + }, + { + "epoch": 0.6176961602671118, + "grad_norm": 3.149915089419751, + "learning_rate": 8.867021970429453e-07, + "log_odds_chosen": 1.9333984851837158, + "log_odds_ratio": -0.29780274629592896, + "logits/chosen": -0.9957031011581421, + "logits/rejected": -0.889843761920929, + "logps/chosen": -0.6250976324081421, + "logps/rejected": -1.990820288658142, + "loss": 0.737, + "nll_loss": 0.663867175579071, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.0625, + "rewards/margins": 0.13642577826976776, + "rewards/rejected": -0.19895020127296448, + "step": 8140 + }, + { + "epoch": 0.6184550007588405, + "grad_norm": 2.7878921038913322, + "learning_rate": 8.86158040994738e-07, + "log_odds_chosen": 1.447363257408142, + "log_odds_ratio": -0.3819824159145355, + "logits/chosen": -1.0300781726837158, + "logits/rejected": -0.919726550579071, + "logps/chosen": -0.6255859136581421, + "logps/rejected": -1.5720703601837158, + "loss": 0.7494, + "nll_loss": 0.727832019329071, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06258545070886612, + "rewards/margins": 0.09465332329273224, + "rewards/rejected": -0.1572265625, + "step": 8150 + }, + { + "epoch": 0.6192138412505691, + "grad_norm": 3.0476600032940997, + "learning_rate": 8.856148855400954e-07, + "log_odds_chosen": 1.740625023841858, + "log_odds_ratio": -0.3084472715854645, + "logits/chosen": -0.963085949420929, + "logits/rejected": -0.841992199420929, + "logps/chosen": -0.600292980670929, + "logps/rejected": -1.8113281726837158, + "loss": 0.7538, + "nll_loss": 0.75, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.06000976637005806, + "rewards/margins": 0.12105713039636612, + "rewards/rejected": -0.18115234375, + "step": 8160 + }, + { + "epoch": 0.6199726817422978, + "grad_norm": 2.5533530605362635, + "learning_rate": 8.850727276162873e-07, + "log_odds_chosen": 1.46875, + "log_odds_ratio": -0.4081054627895355, + "logits/chosen": -1.0302734375, + "logits/rejected": -0.8910156488418579, + "logps/chosen": -0.679882824420929, + "logps/rejected": -1.683984398841858, + "loss": 0.7257, + "nll_loss": 0.661914050579071, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06789550930261612, + "rewards/margins": 0.10047607123851776, + "rewards/rejected": -0.16835936903953552, + "step": 8170 + }, + { + "epoch": 0.6207315222340264, + "grad_norm": 3.2848117284318272, + "learning_rate": 8.845315641736929e-07, + "log_odds_chosen": 1.520605444908142, + "log_odds_ratio": -0.4037109315395355, + "logits/chosen": -0.9814453125, + "logits/rejected": -0.8617187738418579, + "logps/chosen": -0.6546875238418579, + "logps/rejected": -1.7429687976837158, + "loss": 0.7459, + "nll_loss": 0.7669922113418579, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06551513820886612, + "rewards/margins": 0.10893554985523224, + "rewards/rejected": -0.17431640625, + "step": 8180 + }, + { + "epoch": 0.6214903627257551, + "grad_norm": 2.791595021621251, + "learning_rate": 8.839913921757278e-07, + "log_odds_chosen": 1.4835937023162842, + "log_odds_ratio": -0.4066406190395355, + "logits/chosen": -0.9996093511581421, + "logits/rejected": -0.8841797113418579, + "logps/chosen": -0.615234375, + "logps/rejected": -1.641015648841858, + "loss": 0.7326, + "nll_loss": 0.6529296636581421, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06151122972369194, + "rewards/margins": 0.10253296047449112, + "rewards/rejected": -0.1640625, + "step": 8190 + }, + { + "epoch": 0.6222492032174837, + "grad_norm": 3.048746147281647, + "learning_rate": 8.834522085987722e-07, + "log_odds_chosen": 1.454199194908142, + "log_odds_ratio": -0.4039062559604645, + "logits/chosen": -1.033789038658142, + "logits/rejected": -0.9232422113418579, + "logps/chosen": -0.6869140863418579, + "logps/rejected": -1.698632836341858, + "loss": 0.7428, + "nll_loss": 0.6998046636581421, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06870117038488388, + "rewards/margins": 0.10119018703699112, + "rewards/rejected": -0.169921875, + "step": 8200 + }, + { + "epoch": 0.6230080437092124, + "grad_norm": 3.4131145719597917, + "learning_rate": 8.829140104321008e-07, + "log_odds_chosen": 1.7082030773162842, + "log_odds_ratio": -0.35747069120407104, + "logits/chosen": -0.9951171875, + "logits/rejected": -0.8929687738418579, + "logps/chosen": -0.627246081829071, + "logps/rejected": -1.8542969226837158, + "loss": 0.7528, + "nll_loss": 0.690136730670929, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06275634467601776, + "rewards/margins": 0.12246093899011612, + "rewards/rejected": -0.185302734375, + "step": 8210 + }, + { + "epoch": 0.623766884200941, + "grad_norm": 2.35008977369702, + "learning_rate": 8.82376794677811e-07, + "log_odds_chosen": 1.544824242591858, + "log_odds_ratio": -0.3555664122104645, + "logits/chosen": -1.0048828125, + "logits/rejected": -0.8949218988418579, + "logps/chosen": -0.6446288824081421, + "logps/rejected": -1.700781226158142, + "loss": 0.7389, + "nll_loss": 0.717089831829071, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06447754055261612, + "rewards/margins": 0.10557861626148224, + "rewards/rejected": -0.17014160752296448, + "step": 8220 + }, + { + "epoch": 0.6245257246926696, + "grad_norm": 2.644380570827783, + "learning_rate": 8.818405583507537e-07, + "log_odds_chosen": 1.7898437976837158, + "log_odds_ratio": -0.3233886659145355, + "logits/chosen": -0.9800781011581421, + "logits/rejected": -0.845507800579071, + "logps/chosen": -0.608691394329071, + "logps/rejected": -1.8406250476837158, + "loss": 0.7442, + "nll_loss": 0.6659179925918579, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06083984300494194, + "rewards/margins": 0.12324218451976776, + "rewards/rejected": -0.18422850966453552, + "step": 8230 + }, + { + "epoch": 0.6252845651843982, + "grad_norm": 3.0243830901341244, + "learning_rate": 8.813052984784634e-07, + "log_odds_chosen": 1.5166015625, + "log_odds_ratio": -0.40559083223342896, + "logits/chosen": -0.9458984136581421, + "logits/rejected": -0.8583984375, + "logps/chosen": -0.616894543170929, + "logps/rejected": -1.646875023841858, + "loss": 0.7313, + "nll_loss": 0.680468738079071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06174316257238388, + "rewards/margins": 0.102996826171875, + "rewards/rejected": -0.16475829482078552, + "step": 8240 + }, + { + "epoch": 0.6260434056761269, + "grad_norm": 3.2306600805265977, + "learning_rate": 8.807710121010885e-07, + "log_odds_chosen": 1.6636962890625, + "log_odds_ratio": -0.35820311307907104, + "logits/chosen": -0.9722656011581421, + "logits/rejected": -0.835156261920929, + "logps/chosen": -0.6083008050918579, + "logps/rejected": -1.7146484851837158, + "loss": 0.7438, + "nll_loss": 0.730175793170929, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06075439602136612, + "rewards/margins": 0.11063003540039062, + "rewards/rejected": -0.17136231064796448, + "step": 8250 + }, + { + "epoch": 0.6268022461678555, + "grad_norm": 2.9141471270555765, + "learning_rate": 8.802376962713231e-07, + "log_odds_chosen": 1.4658203125, + "log_odds_ratio": -0.36909180879592896, + "logits/chosen": -0.9457031488418579, + "logits/rejected": -0.824999988079071, + "logps/chosen": -0.648632824420929, + "logps/rejected": -1.61328125, + "loss": 0.7468, + "nll_loss": 0.7123047113418579, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06486816704273224, + "rewards/margins": 0.09639892727136612, + "rewards/rejected": -0.1611328125, + "step": 8260 + }, + { + "epoch": 0.6275610866595842, + "grad_norm": 3.1878050398742612, + "learning_rate": 8.797053480543386e-07, + "log_odds_chosen": 1.7580077648162842, + "log_odds_ratio": -0.3594970703125, + "logits/chosen": -0.962890625, + "logits/rejected": -0.8568359613418579, + "logps/chosen": -0.630664050579071, + "logps/rejected": -1.872460961341858, + "loss": 0.7527, + "nll_loss": 0.675000011920929, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06308593600988388, + "rewards/margins": 0.12403564155101776, + "rewards/rejected": -0.18723145127296448, + "step": 8270 + }, + { + "epoch": 0.6283199271513128, + "grad_norm": 3.0743558173968313, + "learning_rate": 8.79173964527716e-07, + "log_odds_chosen": 1.362939476966858, + "log_odds_ratio": -0.37578123807907104, + "logits/chosen": -0.9712890386581421, + "logits/rejected": -0.8773437738418579, + "logps/chosen": -0.5775390863418579, + "logps/rejected": -1.437890648841858, + "loss": 0.726, + "nll_loss": 0.6591796875, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.05767822265625, + "rewards/margins": 0.08607711642980576, + "rewards/rejected": -0.14382323622703552, + "step": 8280 + }, + { + "epoch": 0.6290787676430414, + "grad_norm": 2.651832353171311, + "learning_rate": 8.78643542781378e-07, + "log_odds_chosen": 1.3501465320587158, + "log_odds_ratio": -0.42548829317092896, + "logits/chosen": -0.982617199420929, + "logits/rejected": -0.891796886920929, + "logps/chosen": -0.633105456829071, + "logps/rejected": -1.5535156726837158, + "loss": 0.7413, + "nll_loss": 0.7281249761581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06330566108226776, + "rewards/margins": 0.09206543117761612, + "rewards/rejected": -0.15541991591453552, + "step": 8290 + }, + { + "epoch": 0.6298376081347701, + "grad_norm": 2.9512012017296425, + "learning_rate": 8.781140799175228e-07, + "log_odds_chosen": 1.624414086341858, + "log_odds_ratio": -0.3788085877895355, + "logits/chosen": -0.9400390386581421, + "logits/rejected": -0.8447265625, + "logps/chosen": -0.6796875, + "logps/rejected": -1.859765648841858, + "loss": 0.7517, + "nll_loss": 0.7349609136581421, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06798096001148224, + "rewards/margins": 0.11801757663488388, + "rewards/rejected": -0.18603515625, + "step": 8300 + }, + { + "epoch": 0.6305964486264987, + "grad_norm": 2.6873314333789105, + "learning_rate": 8.775855730505568e-07, + "log_odds_chosen": 1.347900390625, + "log_odds_ratio": -0.4040771424770355, + "logits/chosen": -0.990234375, + "logits/rejected": -0.871874988079071, + "logps/chosen": -0.6451171636581421, + "logps/rejected": -1.57421875, + "loss": 0.7653, + "nll_loss": 0.7845703363418579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06453857570886612, + "rewards/margins": 0.09277953952550888, + "rewards/rejected": -0.15729980170726776, + "step": 8310 + }, + { + "epoch": 0.6313552891182274, + "grad_norm": 2.597865177201747, + "learning_rate": 8.770580193070291e-07, + "log_odds_chosen": 1.535253882408142, + "log_odds_ratio": -0.4124999940395355, + "logits/chosen": -1.043359398841858, + "logits/rejected": -0.915234386920929, + "logps/chosen": -0.6656249761581421, + "logps/rejected": -1.754296898841858, + "loss": 0.7431, + "nll_loss": 0.694531261920929, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06660155951976776, + "rewards/margins": 0.10898437350988388, + "rewards/rejected": -0.17539063096046448, + "step": 8320 + }, + { + "epoch": 0.6321141296099559, + "grad_norm": 2.6636255315470523, + "learning_rate": 8.765314158255661e-07, + "log_odds_chosen": 1.629296898841858, + "log_odds_ratio": -0.35380858182907104, + "logits/chosen": -1.015039086341858, + "logits/rejected": -0.900390625, + "logps/chosen": -0.60302734375, + "logps/rejected": -1.716406226158142, + "loss": 0.7289, + "nll_loss": 0.688183605670929, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06025390699505806, + "rewards/margins": 0.11134032905101776, + "rewards/rejected": -0.17172852158546448, + "step": 8330 + }, + { + "epoch": 0.6328729701016846, + "grad_norm": 3.086899896839651, + "learning_rate": 8.760057597568057e-07, + "log_odds_chosen": 1.525781273841858, + "log_odds_ratio": -0.3667968809604645, + "logits/chosen": -0.9691406488418579, + "logits/rejected": -0.85546875, + "logps/chosen": -0.6664062738418579, + "logps/rejected": -1.6960937976837158, + "loss": 0.7533, + "nll_loss": 0.7625976800918579, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06668701022863388, + "rewards/margins": 0.10299072414636612, + "rewards/rejected": -0.16960449516773224, + "step": 8340 + }, + { + "epoch": 0.6336318105934132, + "grad_norm": 2.9739790332142473, + "learning_rate": 8.754810482633324e-07, + "log_odds_chosen": 1.609960913658142, + "log_odds_ratio": -0.37421876192092896, + "logits/chosen": -0.9654296636581421, + "logits/rejected": -0.863085925579071, + "logps/chosen": -0.571484386920929, + "logps/rejected": -1.644921898841858, + "loss": 0.7429, + "nll_loss": 0.6396484375, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.05717773362994194, + "rewards/margins": 0.10746459662914276, + "rewards/rejected": -0.16452637314796448, + "step": 8350 + }, + { + "epoch": 0.6343906510851419, + "grad_norm": 2.9560606544370613, + "learning_rate": 8.749572785196142e-07, + "log_odds_chosen": 1.9353516101837158, + "log_odds_ratio": -0.28803712129592896, + "logits/chosen": -1.0578124523162842, + "logits/rejected": -0.893359363079071, + "logps/chosen": -0.5663086175918579, + "logps/rejected": -1.8341796398162842, + "loss": 0.7365, + "nll_loss": 0.625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -0.05665283277630806, + "rewards/margins": 0.12680664658546448, + "rewards/rejected": -0.18339844048023224, + "step": 8360 + }, + { + "epoch": 0.6351494915768705, + "grad_norm": 2.568163559209062, + "learning_rate": 8.744344477119373e-07, + "log_odds_chosen": 1.5021483898162842, + "log_odds_ratio": -0.41474610567092896, + "logits/chosen": -0.970898449420929, + "logits/rejected": -0.892382800579071, + "logps/chosen": -0.631152331829071, + "logps/rejected": -1.679296851158142, + "loss": 0.7646, + "nll_loss": 0.6807616949081421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06303711235523224, + "rewards/margins": 0.10488434135913849, + "rewards/rejected": -0.16787108778953552, + "step": 8370 + }, + { + "epoch": 0.6359083320685992, + "grad_norm": 3.0448048864642905, + "learning_rate": 8.739125530383433e-07, + "log_odds_chosen": 1.4322998523712158, + "log_odds_ratio": -0.4263671934604645, + "logits/chosen": -0.9722656011581421, + "logits/rejected": -0.853515625, + "logps/chosen": -0.6661132574081421, + "logps/rejected": -1.634179711341858, + "loss": 0.751, + "nll_loss": 0.7247070074081421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06662597507238388, + "rewards/margins": 0.09671630710363388, + "rewards/rejected": -0.16340331733226776, + "step": 8380 + }, + { + "epoch": 0.6366671725603278, + "grad_norm": 3.1788330711321717, + "learning_rate": 8.733915917085661e-07, + "log_odds_chosen": 1.391943335533142, + "log_odds_ratio": -0.4283203184604645, + "logits/chosen": -0.9908202886581421, + "logits/rejected": -0.876757800579071, + "logps/chosen": -0.64208984375, + "logps/rejected": -1.5900390148162842, + "loss": 0.741, + "nll_loss": 0.728320300579071, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06422118842601776, + "rewards/margins": 0.09482421725988388, + "rewards/rejected": -0.15903320908546448, + "step": 8390 + }, + { + "epoch": 0.6374260130520565, + "grad_norm": 2.750893058680582, + "learning_rate": 8.728715609439695e-07, + "log_odds_chosen": 1.3787109851837158, + "log_odds_ratio": -0.4112304747104645, + "logits/chosen": -0.9349609613418579, + "logits/rejected": -0.852734386920929, + "logps/chosen": -0.600878894329071, + "logps/rejected": -1.517968773841858, + "loss": 0.7427, + "nll_loss": 0.7120116949081421, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06007080152630806, + "rewards/margins": 0.09171142429113388, + "rewards/rejected": -0.15180663764476776, + "step": 8400 + }, + { + "epoch": 0.6381848535437851, + "grad_norm": 3.0718974356666156, + "learning_rate": 8.72352457977484e-07, + "log_odds_chosen": 1.9513671398162842, + "log_odds_ratio": -0.3036254942417145, + "logits/chosen": -0.9458984136581421, + "logits/rejected": -0.851367175579071, + "logps/chosen": -0.6241210699081421, + "logps/rejected": -1.9650390148162842, + "loss": 0.7135, + "nll_loss": 0.69970703125, + "rewards/accuracies": 0.8812500238418579, + "rewards/chosen": -0.06243896484375, + "rewards/margins": 0.133941650390625, + "rewards/rejected": -0.19648437201976776, + "step": 8410 + }, + { + "epoch": 0.6389436940355138, + "grad_norm": 3.311457488876305, + "learning_rate": 8.718342800535456e-07, + "log_odds_chosen": 1.5027344226837158, + "log_odds_ratio": -0.4173828065395355, + "logits/chosen": -1.012109398841858, + "logits/rejected": -0.901562511920929, + "logps/chosen": -0.666308581829071, + "logps/rejected": -1.7080078125, + "loss": 0.7208, + "nll_loss": 0.6729491949081421, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06658935546875, + "rewards/margins": 0.10408935695886612, + "rewards/rejected": -0.170654296875, + "step": 8420 + }, + { + "epoch": 0.6397025345272424, + "grad_norm": 3.262866200217403, + "learning_rate": 8.713170244280353e-07, + "log_odds_chosen": 1.824609398841858, + "log_odds_ratio": -0.34624022245407104, + "logits/chosen": -0.997265636920929, + "logits/rejected": -0.908007800579071, + "logps/chosen": -0.6078125238418579, + "logps/rejected": -1.884374976158142, + "loss": 0.7523, + "nll_loss": 0.7369140386581421, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06077880784869194, + "rewards/margins": 0.12769775092601776, + "rewards/rejected": -0.18852539360523224, + "step": 8430 + }, + { + "epoch": 0.640461375018971, + "grad_norm": 2.8713086281177147, + "learning_rate": 8.708006883682162e-07, + "log_odds_chosen": 1.7332031726837158, + "log_odds_ratio": -0.3373046815395355, + "logits/chosen": -1.033593773841858, + "logits/rejected": -0.886914074420929, + "logps/chosen": -0.6158202886581421, + "logps/rejected": -1.820703148841858, + "loss": 0.7409, + "nll_loss": 0.629199206829071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06154785305261612, + "rewards/margins": 0.12059326469898224, + "rewards/rejected": -0.18203124403953552, + "step": 8440 + }, + { + "epoch": 0.6412202155106996, + "grad_norm": 2.9310215190540623, + "learning_rate": 8.702852691526739e-07, + "log_odds_chosen": 1.4729492664337158, + "log_odds_ratio": -0.396728515625, + "logits/chosen": -0.963085949420929, + "logits/rejected": -0.8636718988418579, + "logps/chosen": -0.599804699420929, + "logps/rejected": -1.566015601158142, + "loss": 0.7518, + "nll_loss": 0.7314453125, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.05996093899011612, + "rewards/margins": 0.09669189155101776, + "rewards/rejected": -0.15666504204273224, + "step": 8450 + }, + { + "epoch": 0.6419790560024283, + "grad_norm": 2.9271110648833347, + "learning_rate": 8.697707640712562e-07, + "log_odds_chosen": 1.4552733898162842, + "log_odds_ratio": -0.3949218690395355, + "logits/chosen": -0.966992199420929, + "logits/rejected": -0.859570324420929, + "logps/chosen": -0.6366211175918579, + "logps/rejected": -1.603124976158142, + "loss": 0.726, + "nll_loss": 0.683886706829071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06368408352136612, + "rewards/margins": 0.09665527194738388, + "rewards/rejected": -0.16030272841453552, + "step": 8460 + }, + { + "epoch": 0.6427378964941569, + "grad_norm": 3.6924975175153034, + "learning_rate": 8.692571704250135e-07, + "log_odds_chosen": 1.582373023033142, + "log_odds_ratio": -0.40437012910842896, + "logits/chosen": -0.9736328125, + "logits/rejected": -0.8726562261581421, + "logps/chosen": -0.6644531488418579, + "logps/rejected": -1.7654297351837158, + "loss": 0.7284, + "nll_loss": 0.6695312261581421, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06644286960363388, + "rewards/margins": 0.1100616455078125, + "rewards/rejected": -0.17658691108226776, + "step": 8470 + }, + { + "epoch": 0.6434967369858856, + "grad_norm": 2.677405253580464, + "learning_rate": 8.687444855261388e-07, + "log_odds_chosen": 1.5798828601837158, + "log_odds_ratio": -0.34428709745407104, + "logits/chosen": -1.045507788658142, + "logits/rejected": -0.908007800579071, + "logps/chosen": -0.67431640625, + "logps/rejected": -1.753515601158142, + "loss": 0.7573, + "nll_loss": 0.7320312261581421, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.06737060844898224, + "rewards/margins": 0.10798339545726776, + "rewards/rejected": -0.17536620795726776, + "step": 8480 + }, + { + "epoch": 0.6442555774776142, + "grad_norm": 2.8686430304565627, + "learning_rate": 8.682327066979084e-07, + "log_odds_chosen": 1.7863280773162842, + "log_odds_ratio": -0.35395509004592896, + "logits/chosen": -1.0158202648162842, + "logits/rejected": -0.909960925579071, + "logps/chosen": -0.6407226324081421, + "logps/rejected": -1.8449218273162842, + "loss": 0.7105, + "nll_loss": 0.664843738079071, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.06411132961511612, + "rewards/margins": 0.12052612006664276, + "rewards/rejected": -0.18452148139476776, + "step": 8490 + }, + { + "epoch": 0.6450144179693429, + "grad_norm": 3.535271780732417, + "learning_rate": 8.677218312746247e-07, + "log_odds_chosen": 1.3644530773162842, + "log_odds_ratio": -0.42656248807907104, + "logits/chosen": -0.934374988079071, + "logits/rejected": -0.867968738079071, + "logps/chosen": -0.6304687261581421, + "logps/rejected": -1.5177733898162842, + "loss": 0.7168, + "nll_loss": 0.693164050579071, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06307373195886612, + "rewards/margins": 0.088623046875, + "rewards/rejected": -0.15166015923023224, + "step": 8500 + }, + { + "epoch": 0.6457732584610715, + "grad_norm": 2.5067323346276043, + "learning_rate": 8.672118566015558e-07, + "log_odds_chosen": 1.441992163658142, + "log_odds_ratio": -0.384033203125, + "logits/chosen": -0.9912109375, + "logits/rejected": -0.8990234136581421, + "logps/chosen": -0.68408203125, + "logps/rejected": -1.667382836341858, + "loss": 0.7498, + "nll_loss": 0.703906238079071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06840820610523224, + "rewards/margins": 0.09831543266773224, + "rewards/rejected": -0.16667480766773224, + "step": 8510 + }, + { + "epoch": 0.6465320989528002, + "grad_norm": 2.6543785612719284, + "learning_rate": 8.667027800348789e-07, + "log_odds_chosen": 1.536718726158142, + "log_odds_ratio": -0.420166015625, + "logits/chosen": -1.006445288658142, + "logits/rejected": -0.896679699420929, + "logps/chosen": -0.6268554925918579, + "logps/rejected": -1.687109351158142, + "loss": 0.737, + "nll_loss": 0.7401367425918579, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06271972507238388, + "rewards/margins": 0.10617981106042862, + "rewards/rejected": -0.16877441108226776, + "step": 8520 + }, + { + "epoch": 0.6472909394445288, + "grad_norm": 2.762159306136293, + "learning_rate": 8.661945989416229e-07, + "log_odds_chosen": 1.3839843273162842, + "log_odds_ratio": -0.38188475370407104, + "logits/chosen": -1.0050780773162842, + "logits/rejected": -0.916210949420929, + "logps/chosen": -0.634472668170929, + "logps/rejected": -1.5574219226837158, + "loss": 0.7218, + "nll_loss": 0.675488293170929, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06340332329273224, + "rewards/margins": 0.09228515625, + "rewards/rejected": -0.15581054985523224, + "step": 8530 + }, + { + "epoch": 0.6480497799362575, + "grad_norm": 2.8951048651776854, + "learning_rate": 8.6568731069961e-07, + "log_odds_chosen": 1.61328125, + "log_odds_ratio": -0.40544432401657104, + "logits/chosen": -0.9619140625, + "logits/rejected": -0.884570300579071, + "logps/chosen": -0.647656261920929, + "logps/rejected": -1.798828125, + "loss": 0.7604, + "nll_loss": 0.7176758050918579, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06474609673023224, + "rewards/margins": 0.11499633640050888, + "rewards/rejected": -0.17973633110523224, + "step": 8540 + }, + { + "epoch": 0.648808620427986, + "grad_norm": 2.3552558261504704, + "learning_rate": 8.651809126974002e-07, + "log_odds_chosen": 1.507226586341858, + "log_odds_ratio": -0.3956542909145355, + "logits/chosen": -1.0255858898162842, + "logits/rejected": -0.929492175579071, + "logps/chosen": -0.6234375238418579, + "logps/rejected": -1.64453125, + "loss": 0.7381, + "nll_loss": 0.7227538824081421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06236572191119194, + "rewards/margins": 0.10211181640625, + "rewards/rejected": -0.1644287109375, + "step": 8550 + }, + { + "epoch": 0.6495674609197146, + "grad_norm": 3.096614392145739, + "learning_rate": 8.646754023342339e-07, + "log_odds_chosen": 1.7849609851837158, + "log_odds_ratio": -0.3517089784145355, + "logits/chosen": -1.0060546398162842, + "logits/rejected": -0.908007800579071, + "logps/chosen": -0.6517578363418579, + "logps/rejected": -1.9578125476837158, + "loss": 0.731, + "nll_loss": 0.7100585699081421, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06523437798023224, + "rewards/margins": 0.13050536811351776, + "rewards/rejected": -0.19589844346046448, + "step": 8560 + }, + { + "epoch": 0.6503263014114433, + "grad_norm": 3.16796873792443, + "learning_rate": 8.64170777019976e-07, + "log_odds_chosen": 1.470434546470642, + "log_odds_ratio": -0.419921875, + "logits/chosen": -0.982226550579071, + "logits/rejected": -0.8779296875, + "logps/chosen": -0.679980456829071, + "logps/rejected": -1.7218749523162842, + "loss": 0.7338, + "nll_loss": 0.739062488079071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06800536811351776, + "rewards/margins": 0.10419921576976776, + "rewards/rejected": -0.17219237983226776, + "step": 8570 + }, + { + "epoch": 0.6510851419031719, + "grad_norm": 2.6695115140682044, + "learning_rate": 8.636670341750609e-07, + "log_odds_chosen": 1.385839819908142, + "log_odds_ratio": -0.4066406190395355, + "logits/chosen": -1.0009765625, + "logits/rejected": -0.8802734613418579, + "logps/chosen": -0.6244140863418579, + "logps/rejected": -1.5412108898162842, + "loss": 0.7031, + "nll_loss": 0.6615234613418579, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06245117262005806, + "rewards/margins": 0.0916748046875, + "rewards/rejected": -0.15415039658546448, + "step": 8580 + }, + { + "epoch": 0.6518439823949006, + "grad_norm": 3.1696301000680327, + "learning_rate": 8.631641712304359e-07, + "log_odds_chosen": 1.8359375, + "log_odds_ratio": -0.33056640625, + "logits/chosen": -1.0232422351837158, + "logits/rejected": -0.9052734375, + "logps/chosen": -0.6185547113418579, + "logps/rejected": -1.9460937976837158, + "loss": 0.7555, + "nll_loss": 0.686230480670929, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06185302883386612, + "rewards/margins": 0.13284912705421448, + "rewards/rejected": -0.19472655653953552, + "step": 8590 + }, + { + "epoch": 0.6526028228866292, + "grad_norm": 2.8968920212170555, + "learning_rate": 8.626621856275073e-07, + "log_odds_chosen": 1.299218773841858, + "log_odds_ratio": -0.4683593809604645, + "logits/chosen": -0.9886718988418579, + "logits/rejected": -0.884765625, + "logps/chosen": -0.6806640625, + "logps/rejected": -1.560156226158142, + "loss": 0.7466, + "nll_loss": 0.73974609375, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06805419921875, + "rewards/margins": 0.08784179389476776, + "rewards/rejected": -0.15593262016773224, + "step": 8600 + }, + { + "epoch": 0.6533616633783579, + "grad_norm": 2.552751142554736, + "learning_rate": 8.621610748180847e-07, + "log_odds_chosen": 1.638769507408142, + "log_odds_ratio": -0.3624511659145355, + "logits/chosen": -1.0068359375, + "logits/rejected": -0.897656261920929, + "logps/chosen": -0.631152331829071, + "logps/rejected": -1.762109398841858, + "loss": 0.734, + "nll_loss": 0.666210949420929, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06315918266773224, + "rewards/margins": 0.11306152492761612, + "rewards/rejected": -0.17631836235523224, + "step": 8610 + }, + { + "epoch": 0.6541205038700865, + "grad_norm": 2.753626232114711, + "learning_rate": 8.616608362643274e-07, + "log_odds_chosen": 1.6326172351837158, + "log_odds_ratio": -0.3794189393520355, + "logits/chosen": -1.000585913658142, + "logits/rejected": -0.8974609375, + "logps/chosen": -0.686328113079071, + "logps/rejected": -1.773046851158142, + "loss": 0.7381, + "nll_loss": 0.7484375238418579, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06859131157398224, + "rewards/margins": 0.10865478217601776, + "rewards/rejected": -0.17722168564796448, + "step": 8620 + }, + { + "epoch": 0.6548793443618152, + "grad_norm": 2.689379678521682, + "learning_rate": 8.611614674386904e-07, + "log_odds_chosen": 1.426660180091858, + "log_odds_ratio": -0.38823240995407104, + "logits/chosen": -0.972460925579071, + "logits/rejected": -0.898242175579071, + "logps/chosen": -0.6400390863418579, + "logps/rejected": -1.603124976158142, + "loss": 0.7334, + "nll_loss": 0.6958984136581421, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06398925930261612, + "rewards/margins": 0.09627075493335724, + "rewards/rejected": -0.16020508110523224, + "step": 8630 + }, + { + "epoch": 0.6556381848535437, + "grad_norm": 2.677306624529923, + "learning_rate": 8.606629658238703e-07, + "log_odds_chosen": 1.325585961341858, + "log_odds_ratio": -0.40888673067092896, + "logits/chosen": -0.9624999761581421, + "logits/rejected": -0.8707031011581421, + "logps/chosen": -0.620410144329071, + "logps/rejected": -1.503515601158142, + "loss": 0.7296, + "nll_loss": 0.6761718988418579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06203613430261612, + "rewards/margins": 0.08827514946460724, + "rewards/rejected": -0.15034179389476776, + "step": 8640 + }, + { + "epoch": 0.6563970253452724, + "grad_norm": 2.895149978178067, + "learning_rate": 8.601653289127525e-07, + "log_odds_chosen": 1.5421874523162842, + "log_odds_ratio": -0.3878417909145355, + "logits/chosen": -1.0207030773162842, + "logits/rejected": -0.9361327886581421, + "logps/chosen": -0.608203113079071, + "logps/rejected": -1.676367163658142, + "loss": 0.7097, + "nll_loss": 0.677734375, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06082763522863388, + "rewards/margins": 0.10679320991039276, + "rewards/rejected": -0.16767577826976776, + "step": 8650 + }, + { + "epoch": 0.657155865837001, + "grad_norm": 2.753880165644991, + "learning_rate": 8.596685542083577e-07, + "log_odds_chosen": 1.6876952648162842, + "log_odds_ratio": -0.3546386659145355, + "logits/chosen": -1.0119140148162842, + "logits/rejected": -0.9166015386581421, + "logps/chosen": -0.5708984136581421, + "logps/rejected": -1.743749976158142, + "loss": 0.7311, + "nll_loss": 0.6666015386581421, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.05710449069738388, + "rewards/margins": 0.11738280951976776, + "rewards/rejected": -0.17441406846046448, + "step": 8660 + }, + { + "epoch": 0.6579147063287297, + "grad_norm": 2.9148157983710514, + "learning_rate": 8.591726392237899e-07, + "log_odds_chosen": 1.8271484375, + "log_odds_ratio": -0.36079102754592896, + "logits/chosen": -0.9859374761581421, + "logits/rejected": -0.8843749761581421, + "logps/chosen": -0.6114257574081421, + "logps/rejected": -1.8933594226837158, + "loss": 0.742, + "nll_loss": 0.689746081829071, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.06110839918255806, + "rewards/margins": 0.12818603217601776, + "rewards/rejected": -0.18935546278953552, + "step": 8670 + }, + { + "epoch": 0.6586735468204583, + "grad_norm": 2.7111935790151516, + "learning_rate": 8.586775814821837e-07, + "log_odds_chosen": 1.3165161609649658, + "log_odds_ratio": -0.46293944120407104, + "logits/chosen": -0.99609375, + "logits/rejected": -0.8892577886581421, + "logps/chosen": -0.6864258050918579, + "logps/rejected": -1.635156273841858, + "loss": 0.7611, + "nll_loss": 0.7523437738418579, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.06866455078125, + "rewards/margins": 0.09486083686351776, + "rewards/rejected": -0.163330078125, + "step": 8680 + }, + { + "epoch": 0.659432387312187, + "grad_norm": 2.6280862213836933, + "learning_rate": 8.58183378516652e-07, + "log_odds_chosen": 1.5119140148162842, + "log_odds_ratio": -0.4378906190395355, + "logits/chosen": -0.971875011920929, + "logits/rejected": -0.873046875, + "logps/chosen": -0.630175769329071, + "logps/rejected": -1.699609398841858, + "loss": 0.7368, + "nll_loss": 0.674023449420929, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06298828125, + "rewards/margins": 0.107025146484375, + "rewards/rejected": -0.17001953721046448, + "step": 8690 + }, + { + "epoch": 0.6601912278039156, + "grad_norm": 2.720790730426448, + "learning_rate": 8.576900278702358e-07, + "log_odds_chosen": 1.4738280773162842, + "log_odds_ratio": -0.39497071504592896, + "logits/chosen": -1.0126953125, + "logits/rejected": -0.9164062738418579, + "logps/chosen": -0.6498047113418579, + "logps/rejected": -1.675390601158142, + "loss": 0.6992, + "nll_loss": 0.6756836175918579, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06499023735523224, + "rewards/margins": 0.10251464694738388, + "rewards/rejected": -0.16745606064796448, + "step": 8700 + }, + { + "epoch": 0.6609500682956443, + "grad_norm": 2.5856309339344405, + "learning_rate": 8.57197527095851e-07, + "log_odds_chosen": 1.5299804210662842, + "log_odds_ratio": -0.39042967557907104, + "logits/chosen": -0.9970703125, + "logits/rejected": -0.8843749761581421, + "logps/chosen": -0.611523449420929, + "logps/rejected": -1.680078148841858, + "loss": 0.727, + "nll_loss": 0.6763671636581421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06114501878619194, + "rewards/margins": 0.10677490383386612, + "rewards/rejected": -0.16816405951976776, + "step": 8710 + }, + { + "epoch": 0.6617089087873729, + "grad_norm": 2.64453440804158, + "learning_rate": 8.567058737562385e-07, + "log_odds_chosen": 1.338964819908142, + "log_odds_ratio": -0.45068359375, + "logits/chosen": -1.004492163658142, + "logits/rejected": -0.904296875, + "logps/chosen": -0.6468750238418579, + "logps/rejected": -1.526757836341858, + "loss": 0.7073, + "nll_loss": 0.6474609375, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06473388522863388, + "rewards/margins": 0.08798141777515411, + "rewards/rejected": -0.15273436903953552, + "step": 8720 + }, + { + "epoch": 0.6624677492791016, + "grad_norm": 2.405005952527545, + "learning_rate": 8.562150654239141e-07, + "log_odds_chosen": 1.528906226158142, + "log_odds_ratio": -0.399658203125, + "logits/chosen": -1.008398413658142, + "logits/rejected": -0.908203125, + "logps/chosen": -0.65576171875, + "logps/rejected": -1.742578148841858, + "loss": 0.7386, + "nll_loss": 0.7059570550918579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06556396186351776, + "rewards/margins": 0.10859374701976776, + "rewards/rejected": -0.17434081435203552, + "step": 8730 + }, + { + "epoch": 0.6632265897708302, + "grad_norm": 2.6175396641657516, + "learning_rate": 8.55725099681116e-07, + "log_odds_chosen": 1.3904297351837158, + "log_odds_ratio": -0.4146484434604645, + "logits/chosen": -1.0255858898162842, + "logits/rejected": -0.9332031011581421, + "logps/chosen": -0.654492199420929, + "logps/rejected": -1.6095702648162842, + "loss": 0.7259, + "nll_loss": 0.6895507574081421, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06549072265625, + "rewards/margins": 0.09559936821460724, + "rewards/rejected": -0.16103515028953552, + "step": 8740 + }, + { + "epoch": 0.6639854302625589, + "grad_norm": 2.6906800972208758, + "learning_rate": 8.552359741197579e-07, + "log_odds_chosen": 1.6115233898162842, + "log_odds_ratio": -0.38212889432907104, + "logits/chosen": -1.016015648841858, + "logits/rejected": -0.901562511920929, + "logps/chosen": -0.61474609375, + "logps/rejected": -1.6902344226837158, + "loss": 0.7212, + "nll_loss": 0.647656261920929, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06151122972369194, + "rewards/margins": 0.10777588188648224, + "rewards/rejected": -0.16923828423023224, + "step": 8750 + }, + { + "epoch": 0.6647442707542874, + "grad_norm": 3.1965397296377285, + "learning_rate": 8.547476863413765e-07, + "log_odds_chosen": 1.517968773841858, + "log_odds_ratio": -0.375732421875, + "logits/chosen": -1.025781273841858, + "logits/rejected": -0.910351574420929, + "logps/chosen": -0.626953125, + "logps/rejected": -1.685156226158142, + "loss": 0.721, + "nll_loss": 0.61767578125, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06269530951976776, + "rewards/margins": 0.10578612983226776, + "rewards/rejected": -0.16845703125, + "step": 8760 + }, + { + "epoch": 0.6655031112460161, + "grad_norm": 3.1745774121709993, + "learning_rate": 8.54260233957083e-07, + "log_odds_chosen": 1.410253882408142, + "log_odds_ratio": -0.41484373807907104, + "logits/chosen": -1.0271484851837158, + "logits/rejected": -0.905078113079071, + "logps/chosen": -0.639941394329071, + "logps/rejected": -1.6300780773162842, + "loss": 0.7232, + "nll_loss": 0.722460925579071, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06402587890625, + "rewards/margins": 0.09885253757238388, + "rewards/rejected": -0.16291503608226776, + "step": 8770 + }, + { + "epoch": 0.6662619517377447, + "grad_norm": 3.3473736434386168, + "learning_rate": 8.537736145875154e-07, + "log_odds_chosen": 1.4734375476837158, + "log_odds_ratio": -0.4004882872104645, + "logits/chosen": -1.0300781726837158, + "logits/rejected": -0.8910156488418579, + "logps/chosen": -0.65380859375, + "logps/rejected": -1.6845703125, + "loss": 0.7195, + "nll_loss": 0.731640636920929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06539306789636612, + "rewards/margins": 0.10313721001148224, + "rewards/rejected": -0.16843262314796448, + "step": 8780 + }, + { + "epoch": 0.6670207922294734, + "grad_norm": 2.7101377956102914, + "learning_rate": 8.532878258627874e-07, + "log_odds_chosen": 1.5070312023162842, + "log_odds_ratio": -0.39033204317092896, + "logits/chosen": -1.0041015148162842, + "logits/rejected": -0.8978515863418579, + "logps/chosen": -0.6019531488418579, + "logps/rejected": -1.6140625476837158, + "loss": 0.7291, + "nll_loss": 0.70166015625, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06022949144244194, + "rewards/margins": 0.10114135593175888, + "rewards/rejected": -0.161376953125, + "step": 8790 + }, + { + "epoch": 0.667779632721202, + "grad_norm": 3.1457287417296307, + "learning_rate": 8.528028654224416e-07, + "log_odds_chosen": 1.606835961341858, + "log_odds_ratio": -0.35566407442092896, + "logits/chosen": -1.040624976158142, + "logits/rejected": -0.895703136920929, + "logps/chosen": -0.637988269329071, + "logps/rejected": -1.7429687976837158, + "loss": 0.7173, + "nll_loss": 0.7562500238418579, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06373290717601776, + "rewards/margins": 0.11043701320886612, + "rewards/rejected": -0.1741943359375, + "step": 8800 + }, + { + "epoch": 0.6685384732129307, + "grad_norm": 2.4979785314087897, + "learning_rate": 8.523187309154008e-07, + "log_odds_chosen": 1.6749999523162842, + "log_odds_ratio": -0.3563476502895355, + "logits/chosen": -1.016992211341858, + "logits/rejected": -0.9253906011581421, + "logps/chosen": -0.6712890863418579, + "logps/rejected": -1.8273437023162842, + "loss": 0.7068, + "nll_loss": 0.613574206829071, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06718750298023224, + "rewards/margins": 0.11564941704273224, + "rewards/rejected": -0.18283692002296448, + "step": 8810 + }, + { + "epoch": 0.6692973137046593, + "grad_norm": 2.89145165275318, + "learning_rate": 8.518354199999198e-07, + "log_odds_chosen": 1.4490234851837158, + "log_odds_ratio": -0.36884766817092896, + "logits/chosen": -1.058007836341858, + "logits/rejected": -0.927734375, + "logps/chosen": -0.631542980670929, + "logps/rejected": -1.5812499523162842, + "loss": 0.7171, + "nll_loss": 0.643359363079071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06312255561351776, + "rewards/margins": 0.09495849907398224, + "rewards/rejected": -0.15803222358226776, + "step": 8820 + }, + { + "epoch": 0.6700561541963879, + "grad_norm": 2.7949277153698695, + "learning_rate": 8.513529303435386e-07, + "log_odds_chosen": 1.6535155773162842, + "log_odds_ratio": -0.3829101622104645, + "logits/chosen": -1.033203125, + "logits/rejected": -0.938281238079071, + "logps/chosen": -0.6231445074081421, + "logps/rejected": -1.771484375, + "loss": 0.7095, + "nll_loss": 0.680371105670929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06230468675494194, + "rewards/margins": 0.11488189548254013, + "rewards/rejected": -0.17722168564796448, + "step": 8830 + }, + { + "epoch": 0.6708149946881166, + "grad_norm": 2.6022467229642148, + "learning_rate": 8.50871259623034e-07, + "log_odds_chosen": 1.805273413658142, + "log_odds_ratio": -0.3316406309604645, + "logits/chosen": -0.982421875, + "logits/rejected": -0.8453124761581421, + "logps/chosen": -0.578906238079071, + "logps/rejected": -1.7998046875, + "loss": 0.717, + "nll_loss": 0.693066418170929, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -0.05787353590130806, + "rewards/margins": 0.12221679836511612, + "rewards/rejected": -0.18012695014476776, + "step": 8840 + }, + { + "epoch": 0.6715738351798451, + "grad_norm": 2.5418111643811905, + "learning_rate": 8.503904055243742e-07, + "log_odds_chosen": 1.670019507408142, + "log_odds_ratio": -0.365234375, + "logits/chosen": -0.9320312738418579, + "logits/rejected": -0.8900390863418579, + "logps/chosen": -0.57763671875, + "logps/rejected": -1.7355468273162842, + "loss": 0.7158, + "nll_loss": 0.622363269329071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.0577392578125, + "rewards/margins": 0.11576537787914276, + "rewards/rejected": -0.17355957627296448, + "step": 8850 + }, + { + "epoch": 0.6723326756715738, + "grad_norm": 3.052844854817002, + "learning_rate": 8.499103657426704e-07, + "log_odds_chosen": 1.416601538658142, + "log_odds_ratio": -0.42431640625, + "logits/chosen": -1.056054711341858, + "logits/rejected": -0.9154297113418579, + "logps/chosen": -0.685742199420929, + "logps/rejected": -1.661523461341858, + "loss": 0.7174, + "nll_loss": 0.71533203125, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06862793117761612, + "rewards/margins": 0.09736938774585724, + "rewards/rejected": -0.16611328721046448, + "step": 8860 + }, + { + "epoch": 0.6730915161633024, + "grad_norm": 2.511251534832671, + "learning_rate": 8.494311379821314e-07, + "log_odds_chosen": 1.158789038658142, + "log_odds_ratio": -0.44999998807907104, + "logits/chosen": -0.9912109375, + "logits/rejected": -0.880859375, + "logps/chosen": -0.630566418170929, + "logps/rejected": -1.350000023841858, + "loss": 0.7238, + "nll_loss": 0.6845703125, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06308593600988388, + "rewards/margins": 0.07196655124425888, + "rewards/rejected": -0.13498535752296448, + "step": 8870 + }, + { + "epoch": 0.6738503566550311, + "grad_norm": 2.412780939469374, + "learning_rate": 8.489527199560178e-07, + "log_odds_chosen": 1.5443847179412842, + "log_odds_ratio": -0.4092773497104645, + "logits/chosen": -1.009179711341858, + "logits/rejected": -0.914257824420929, + "logps/chosen": -0.613574206829071, + "logps/rejected": -1.6824219226837158, + "loss": 0.7251, + "nll_loss": 0.674023449420929, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06130371242761612, + "rewards/margins": 0.10692749172449112, + "rewards/rejected": -0.16813965141773224, + "step": 8880 + }, + { + "epoch": 0.6746091971467597, + "grad_norm": 3.0254601092370574, + "learning_rate": 8.484751093865948e-07, + "log_odds_chosen": 1.3079102039337158, + "log_odds_ratio": -0.43115234375, + "logits/chosen": -1.008398413658142, + "logits/rejected": -0.904296875, + "logps/chosen": -0.6478515863418579, + "logps/rejected": -1.4953124523162842, + "loss": 0.7184, + "nll_loss": 0.7030273675918579, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06484375149011612, + "rewards/margins": 0.08479003608226776, + "rewards/rejected": -0.14960937201976776, + "step": 8890 + }, + { + "epoch": 0.6753680376384884, + "grad_norm": 3.9457403315097856, + "learning_rate": 8.47998304005088e-07, + "log_odds_chosen": 1.51513671875, + "log_odds_ratio": -0.4247070252895355, + "logits/chosen": -0.949023425579071, + "logits/rejected": -0.873046875, + "logps/chosen": -0.719921886920929, + "logps/rejected": -1.7775390148162842, + "loss": 0.7391, + "nll_loss": 0.7147461175918579, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.07200928032398224, + "rewards/margins": 0.10576172173023224, + "rewards/rejected": -0.17766113579273224, + "step": 8900 + }, + { + "epoch": 0.676126878130217, + "grad_norm": 3.1192413134893493, + "learning_rate": 8.475223015516377e-07, + "log_odds_chosen": 1.552343726158142, + "log_odds_ratio": -0.3772949278354645, + "logits/chosen": -0.994335949420929, + "logits/rejected": -0.8929687738418579, + "logps/chosen": -0.620800793170929, + "logps/rejected": -1.673828125, + "loss": 0.7235, + "nll_loss": 0.69287109375, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06203613430261612, + "rewards/margins": 0.10540771484375, + "rewards/rejected": -0.16730956733226776, + "step": 8910 + }, + { + "epoch": 0.6768857186219457, + "grad_norm": 2.6214655270353266, + "learning_rate": 8.470470997752534e-07, + "log_odds_chosen": 1.460546851158142, + "log_odds_ratio": -0.40434569120407104, + "logits/chosen": -1.045312523841858, + "logits/rejected": -0.9146484136581421, + "logps/chosen": -0.6294921636581421, + "logps/rejected": -1.6281249523162842, + "loss": 0.7257, + "nll_loss": 0.6561523675918579, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06297607719898224, + "rewards/margins": 0.09989013522863388, + "rewards/rejected": -0.16279296576976776, + "step": 8920 + }, + { + "epoch": 0.6776445591136743, + "grad_norm": 2.87762511585744, + "learning_rate": 8.465726964337702e-07, + "log_odds_chosen": 1.945703148841858, + "log_odds_ratio": -0.3311523497104645, + "logits/chosen": -1.008203148841858, + "logits/rejected": -0.8978515863418579, + "logps/chosen": -0.610546886920929, + "logps/rejected": -2.012500047683716, + "loss": 0.7284, + "nll_loss": 0.7041015625, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06104736402630806, + "rewards/margins": 0.14020995795726776, + "rewards/rejected": -0.20107421278953552, + "step": 8930 + }, + { + "epoch": 0.678403399605403, + "grad_norm": 2.683763592444103, + "learning_rate": 8.460990892938031e-07, + "log_odds_chosen": 1.658789038658142, + "log_odds_ratio": -0.37397462129592896, + "logits/chosen": -1.0085937976837158, + "logits/rejected": -0.882617175579071, + "logps/chosen": -0.609179675579071, + "logps/rejected": -1.791015625, + "loss": 0.7071, + "nll_loss": 0.689257800579071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06092529371380806, + "rewards/margins": 0.11809692531824112, + "rewards/rejected": -0.1790771484375, + "step": 8940 + }, + { + "epoch": 0.6791622400971316, + "grad_norm": 2.449182932081057, + "learning_rate": 8.456262761307038e-07, + "log_odds_chosen": 1.5183594226837158, + "log_odds_ratio": -0.38395994901657104, + "logits/chosen": -0.984375, + "logits/rejected": -0.885546863079071, + "logps/chosen": -0.669726550579071, + "logps/rejected": -1.7228515148162842, + "loss": 0.7217, + "nll_loss": 0.712109386920929, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06695556640625, + "rewards/margins": 0.10532226413488388, + "rewards/rejected": -0.1722412109375, + "step": 8950 + }, + { + "epoch": 0.6799210805888602, + "grad_norm": 2.3708588018010195, + "learning_rate": 8.451542547285166e-07, + "log_odds_chosen": 1.4533202648162842, + "log_odds_ratio": -0.41425782442092896, + "logits/chosen": -1.015234351158142, + "logits/rejected": -0.8861328363418579, + "logps/chosen": -0.63037109375, + "logps/rejected": -1.6416015625, + "loss": 0.7285, + "nll_loss": 0.674511730670929, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06312255561351776, + "rewards/margins": 0.10125732421875, + "rewards/rejected": -0.1641845703125, + "step": 8960 + }, + { + "epoch": 0.6806799210805888, + "grad_norm": 3.5430099084984334, + "learning_rate": 8.44683022879934e-07, + "log_odds_chosen": 1.6541016101837158, + "log_odds_ratio": -0.3408203125, + "logits/chosen": -1.025976538658142, + "logits/rejected": -0.925000011920929, + "logps/chosen": -0.60009765625, + "logps/rejected": -1.7423827648162842, + "loss": 0.736, + "nll_loss": 0.6495116949081421, + "rewards/accuracies": 0.8812500238418579, + "rewards/chosen": -0.05999755859375, + "rewards/margins": 0.11427001655101776, + "rewards/rejected": -0.17426757514476776, + "step": 8970 + }, + { + "epoch": 0.6814387615723175, + "grad_norm": 3.102706773563467, + "learning_rate": 8.442125783862544e-07, + "log_odds_chosen": 1.7042968273162842, + "log_odds_ratio": -0.35102540254592896, + "logits/chosen": -1.0712890625, + "logits/rejected": -0.9183593988418579, + "logps/chosen": -0.6201171875, + "logps/rejected": -1.8410155773162842, + "loss": 0.7214, + "nll_loss": 0.669628918170929, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.06193847581744194, + "rewards/margins": 0.12209472805261612, + "rewards/rejected": -0.18408203125, + "step": 8980 + }, + { + "epoch": 0.6821976020640461, + "grad_norm": 2.9053453555705984, + "learning_rate": 8.437429190573388e-07, + "log_odds_chosen": 1.7033202648162842, + "log_odds_ratio": -0.3802246153354645, + "logits/chosen": -1.0107421875, + "logits/rejected": -0.9296875, + "logps/chosen": -0.61279296875, + "logps/rejected": -1.8103516101837158, + "loss": 0.7214, + "nll_loss": 0.6630859375, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06125488132238388, + "rewards/margins": 0.11966552585363388, + "rewards/rejected": -0.18088379502296448, + "step": 8990 + }, + { + "epoch": 0.6829564425557748, + "grad_norm": 2.490135537579069, + "learning_rate": 8.432740427115678e-07, + "log_odds_chosen": 1.4577147960662842, + "log_odds_ratio": -0.39594727754592896, + "logits/chosen": -0.9605468511581421, + "logits/rejected": -0.8550781011581421, + "logps/chosen": -0.6131836175918579, + "logps/rejected": -1.596093773841858, + "loss": 0.7053, + "nll_loss": 0.696972668170929, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.061279296875, + "rewards/margins": 0.09818573296070099, + "rewards/rejected": -0.15944823622703552, + "step": 9000 + }, + { + "epoch": 0.6837152830475034, + "grad_norm": 3.8422780431570813, + "learning_rate": 8.428059471757984e-07, + "log_odds_chosen": 1.509765625, + "log_odds_ratio": -0.3612060546875, + "logits/chosen": -0.9937499761581421, + "logits/rejected": -0.902148425579071, + "logps/chosen": -0.6751953363418579, + "logps/rejected": -1.7332031726837158, + "loss": 0.7349, + "nll_loss": 0.7255859375, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06752929836511612, + "rewards/margins": 0.10584716498851776, + "rewards/rejected": -0.17343750596046448, + "step": 9010 + }, + { + "epoch": 0.6844741235392321, + "grad_norm": 3.8217632772985533, + "learning_rate": 8.423386302853226e-07, + "log_odds_chosen": 1.5412108898162842, + "log_odds_ratio": -0.3634277284145355, + "logits/chosen": -1.0535156726837158, + "logits/rejected": -0.940234363079071, + "logps/chosen": -0.604785144329071, + "logps/rejected": -1.644140601158142, + "loss": 0.7039, + "nll_loss": 0.687304675579071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06046142429113388, + "rewards/margins": 0.10375366359949112, + "rewards/rejected": -0.16440430283546448, + "step": 9020 + }, + { + "epoch": 0.6852329640309607, + "grad_norm": 2.868969089731345, + "learning_rate": 8.418720898838254e-07, + "log_odds_chosen": 1.5802733898162842, + "log_odds_ratio": -0.3963378965854645, + "logits/chosen": -0.9775390625, + "logits/rejected": -0.896484375, + "logps/chosen": -0.645800769329071, + "logps/rejected": -1.7529296875, + "loss": 0.7625, + "nll_loss": 0.67333984375, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06461181491613388, + "rewards/margins": 0.11072997748851776, + "rewards/rejected": -0.17524413764476776, + "step": 9030 + }, + { + "epoch": 0.6859918045226894, + "grad_norm": 2.9932422667148457, + "learning_rate": 8.414063238233425e-07, + "log_odds_chosen": 1.869140625, + "log_odds_ratio": -0.29838865995407104, + "logits/chosen": -0.987500011920929, + "logits/rejected": -0.8609374761581421, + "logps/chosen": -0.6407226324081421, + "logps/rejected": -1.9460937976837158, + "loss": 0.72, + "nll_loss": 0.7079101800918579, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.0640869140625, + "rewards/margins": 0.13059082627296448, + "rewards/rejected": -0.194580078125, + "step": 9040 + }, + { + "epoch": 0.686750645014418, + "grad_norm": 3.080579444219386, + "learning_rate": 8.409413299642188e-07, + "log_odds_chosen": 1.582617163658142, + "log_odds_ratio": -0.3634277284145355, + "logits/chosen": -1.0291016101837158, + "logits/rejected": -0.892773449420929, + "logps/chosen": -0.6392577886581421, + "logps/rejected": -1.748632788658142, + "loss": 0.7379, + "nll_loss": 0.687207043170929, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06386718899011612, + "rewards/margins": 0.11099853366613388, + "rewards/rejected": -0.17478027939796448, + "step": 9050 + }, + { + "epoch": 0.6875094855061467, + "grad_norm": 2.48095153866512, + "learning_rate": 8.404771061750672e-07, + "log_odds_chosen": 1.6251952648162842, + "log_odds_ratio": -0.3843750059604645, + "logits/chosen": -0.9576171636581421, + "logits/rejected": -0.845703125, + "logps/chosen": -0.6758788824081421, + "logps/rejected": -1.826757788658142, + "loss": 0.7342, + "nll_loss": 0.716992199420929, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06765136867761612, + "rewards/margins": 0.11514892429113388, + "rewards/rejected": -0.18276366591453552, + "step": 9060 + }, + { + "epoch": 0.6882683259978752, + "grad_norm": 3.138066548353663, + "learning_rate": 8.400136503327277e-07, + "log_odds_chosen": 1.494531273841858, + "log_odds_ratio": -0.4088134765625, + "logits/chosen": -1.018164038658142, + "logits/rejected": -0.9740234613418579, + "logps/chosen": -0.651074230670929, + "logps/rejected": -1.631250023841858, + "loss": 0.7057, + "nll_loss": 0.5933593511581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06508789211511612, + "rewards/margins": 0.09808349609375, + "rewards/rejected": -0.16330567002296448, + "step": 9070 + }, + { + "epoch": 0.6890271664896039, + "grad_norm": 2.6656921736577415, + "learning_rate": 8.395509603222271e-07, + "log_odds_chosen": 1.689062476158142, + "log_odds_ratio": -0.3423095643520355, + "logits/chosen": -1.023828148841858, + "logits/rejected": -0.921875, + "logps/chosen": -0.623339831829071, + "logps/rejected": -1.80078125, + "loss": 0.7244, + "nll_loss": 0.718066394329071, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06226806715130806, + "rewards/margins": 0.11767578125, + "rewards/rejected": -0.1800537109375, + "step": 9080 + }, + { + "epoch": 0.6897860069813325, + "grad_norm": 2.818770206694993, + "learning_rate": 8.390890340367368e-07, + "log_odds_chosen": 1.294335961341858, + "log_odds_ratio": -0.39106446504592896, + "logits/chosen": -1.0060546398162842, + "logits/rejected": -0.911328136920929, + "logps/chosen": -0.6317383050918579, + "logps/rejected": -1.4919922351837158, + "loss": 0.7159, + "nll_loss": 0.7132812738418579, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06312255561351776, + "rewards/margins": 0.086029052734375, + "rewards/rejected": -0.14924316108226776, + "step": 9090 + }, + { + "epoch": 0.6905448474730611, + "grad_norm": 2.7946045339632963, + "learning_rate": 8.386278693775346e-07, + "log_odds_chosen": 1.460546851158142, + "log_odds_ratio": -0.38300782442092896, + "logits/chosen": -0.9693359136581421, + "logits/rejected": -0.891406238079071, + "logps/chosen": -0.6612304449081421, + "logps/rejected": -1.6355469226837158, + "loss": 0.7402, + "nll_loss": 0.6854492425918579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06613769382238388, + "rewards/margins": 0.09750976413488388, + "rewards/rejected": -0.16352538764476776, + "step": 9100 + }, + { + "epoch": 0.6913036879647898, + "grad_norm": 3.4993736100208173, + "learning_rate": 8.381674642539632e-07, + "log_odds_chosen": 1.458398461341858, + "log_odds_ratio": -0.4149414002895355, + "logits/chosen": -1.0343749523162842, + "logits/rejected": -0.9302734136581421, + "logps/chosen": -0.636914074420929, + "logps/rejected": -1.6232421398162842, + "loss": 0.7265, + "nll_loss": 0.678906261920929, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06370849907398224, + "rewards/margins": 0.09853820502758026, + "rewards/rejected": -0.16232910752296448, + "step": 9110 + }, + { + "epoch": 0.6920625284565184, + "grad_norm": 2.87806956546888, + "learning_rate": 8.37707816583391e-07, + "log_odds_chosen": 1.818750023841858, + "log_odds_ratio": -0.3299316465854645, + "logits/chosen": -1.082617163658142, + "logits/rejected": -0.9185546636581421, + "logps/chosen": -0.662890613079071, + "logps/rejected": -1.958593726158142, + "loss": 0.7081, + "nll_loss": 0.699999988079071, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06630859524011612, + "rewards/margins": 0.12961426377296448, + "rewards/rejected": -0.19589844346046448, + "step": 9120 + }, + { + "epoch": 0.6928213689482471, + "grad_norm": 2.976141220633325, + "learning_rate": 8.372489242911724e-07, + "log_odds_chosen": 1.623632788658142, + "log_odds_ratio": -0.3701171875, + "logits/chosen": -1.049218773841858, + "logits/rejected": -0.901171863079071, + "logps/chosen": -0.587109386920929, + "logps/rejected": -1.689453125, + "loss": 0.7078, + "nll_loss": 0.5948241949081421, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.05867920070886612, + "rewards/margins": 0.11011962592601776, + "rewards/rejected": -0.16887207329273224, + "step": 9130 + }, + { + "epoch": 0.6935802094399757, + "grad_norm": 3.495959990506981, + "learning_rate": 8.367907853106078e-07, + "log_odds_chosen": 1.5183594226837158, + "log_odds_ratio": -0.3849121034145355, + "logits/chosen": -1.039453148841858, + "logits/rejected": -0.9058593511581421, + "logps/chosen": -0.6591796875, + "logps/rejected": -1.702539086341858, + "loss": 0.7543, + "nll_loss": 0.73583984375, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06594238430261612, + "rewards/margins": 0.10428466647863388, + "rewards/rejected": -0.17019042372703552, + "step": 9140 + }, + { + "epoch": 0.6943390499317044, + "grad_norm": 3.15168909268087, + "learning_rate": 8.363333975829066e-07, + "log_odds_chosen": 1.76953125, + "log_odds_ratio": -0.3524414002895355, + "logits/chosen": -1.012109398841858, + "logits/rejected": -0.8880859613418579, + "logps/chosen": -0.624804675579071, + "logps/rejected": -1.8757812976837158, + "loss": 0.7177, + "nll_loss": 0.6651366949081421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06248779222369194, + "rewards/margins": 0.12497558444738388, + "rewards/rejected": -0.1875, + "step": 9150 + }, + { + "epoch": 0.695097890423433, + "grad_norm": 3.0190535757643326, + "learning_rate": 8.358767590571457e-07, + "log_odds_chosen": 1.661523461341858, + "log_odds_ratio": -0.355224609375, + "logits/chosen": -1.0380859375, + "logits/rejected": -0.912109375, + "logps/chosen": -0.5850585699081421, + "logps/rejected": -1.651953101158142, + "loss": 0.6982, + "nll_loss": 0.642285168170929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.05849609524011612, + "rewards/margins": 0.10675048828125, + "rewards/rejected": -0.16518554091453552, + "step": 9160 + }, + { + "epoch": 0.6958567309151616, + "grad_norm": 2.6297503660881665, + "learning_rate": 8.354208676902326e-07, + "log_odds_chosen": 1.6213867664337158, + "log_odds_ratio": -0.3669677674770355, + "logits/chosen": -1.0037109851837158, + "logits/rejected": -0.890625, + "logps/chosen": -0.5731445550918579, + "logps/rejected": -1.680078148841858, + "loss": 0.7222, + "nll_loss": 0.642285168170929, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.05728759616613388, + "rewards/margins": 0.11073608696460724, + "rewards/rejected": -0.16796875, + "step": 9170 + }, + { + "epoch": 0.6966155714068902, + "grad_norm": 3.1619887361778525, + "learning_rate": 8.349657214468659e-07, + "log_odds_chosen": 1.474023461341858, + "log_odds_ratio": -0.3697753846645355, + "logits/chosen": -1.0623047351837158, + "logits/rejected": -0.9458984136581421, + "logps/chosen": -0.5767577886581421, + "logps/rejected": -1.539453148841858, + "loss": 0.7018, + "nll_loss": 0.640332043170929, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.05766601487994194, + "rewards/margins": 0.09627685695886612, + "rewards/rejected": -0.1539306640625, + "step": 9180 + }, + { + "epoch": 0.6973744118986189, + "grad_norm": 2.9357100092718236, + "learning_rate": 8.345113182994988e-07, + "log_odds_chosen": 1.493749976158142, + "log_odds_ratio": -0.3768066465854645, + "logits/chosen": -1.0568358898162842, + "logits/rejected": -0.921679675579071, + "logps/chosen": -0.5972656011581421, + "logps/rejected": -1.613671898841858, + "loss": 0.6904, + "nll_loss": 0.6214843988418579, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.05975341796875, + "rewards/margins": 0.10163573920726776, + "rewards/rejected": -0.16145019233226776, + "step": 9190 + }, + { + "epoch": 0.6981332523903475, + "grad_norm": 2.604420840193982, + "learning_rate": 8.34057656228299e-07, + "log_odds_chosen": 1.535742163658142, + "log_odds_ratio": -0.4088378846645355, + "logits/chosen": -1.037109375, + "logits/rejected": -0.9345703125, + "logps/chosen": -0.64697265625, + "logps/rejected": -1.7136719226837158, + "loss": 0.7144, + "nll_loss": 0.651562511920929, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06472168117761612, + "rewards/margins": 0.10678710788488388, + "rewards/rejected": -0.17136231064796448, + "step": 9200 + }, + { + "epoch": 0.6988920928820762, + "grad_norm": 3.200157985640524, + "learning_rate": 8.336047332211128e-07, + "log_odds_chosen": 1.6169922351837158, + "log_odds_ratio": -0.34624022245407104, + "logits/chosen": -1.0525391101837158, + "logits/rejected": -0.942187488079071, + "logps/chosen": -0.62646484375, + "logps/rejected": -1.7042968273162842, + "loss": 0.71, + "nll_loss": 0.6533203125, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06267090141773224, + "rewards/margins": 0.10789795219898224, + "rewards/rejected": -0.17041015625, + "step": 9210 + }, + { + "epoch": 0.6996509333738048, + "grad_norm": 3.2156469672490178, + "learning_rate": 8.331525472734267e-07, + "log_odds_chosen": 1.89453125, + "log_odds_ratio": -0.33203125, + "logits/chosen": -1.049218773841858, + "logits/rejected": -0.9134765863418579, + "logps/chosen": -0.6415039300918579, + "logps/rejected": -2.007031202316284, + "loss": 0.7091, + "nll_loss": 0.67333984375, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06411132961511612, + "rewards/margins": 0.136566162109375, + "rewards/rejected": -0.20078125596046448, + "step": 9220 + }, + { + "epoch": 0.7004097738655335, + "grad_norm": 3.4145794329670482, + "learning_rate": 8.327010963883302e-07, + "log_odds_chosen": 1.4153320789337158, + "log_odds_ratio": -0.4208007752895355, + "logits/chosen": -1.0205078125, + "logits/rejected": -0.932421863079071, + "logps/chosen": -0.61962890625, + "logps/rejected": -1.5984375476837158, + "loss": 0.7182, + "nll_loss": 0.70166015625, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06198730319738388, + "rewards/margins": 0.097930908203125, + "rewards/rejected": -0.15986327826976776, + "step": 9230 + }, + { + "epoch": 0.7011686143572621, + "grad_norm": 3.1446396732274375, + "learning_rate": 8.322503785764789e-07, + "log_odds_chosen": 1.369726538658142, + "log_odds_ratio": -0.4117675721645355, + "logits/chosen": -0.993359386920929, + "logits/rejected": -0.887499988079071, + "logps/chosen": -0.6258789300918579, + "logps/rejected": -1.572265625, + "loss": 0.7209, + "nll_loss": 0.689648449420929, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06258545070886612, + "rewards/margins": 0.094696044921875, + "rewards/rejected": -0.15737304091453552, + "step": 9240 + }, + { + "epoch": 0.7019274548489908, + "grad_norm": 2.7808709633900044, + "learning_rate": 8.318003918560583e-07, + "log_odds_chosen": 1.7595703601837158, + "log_odds_ratio": -0.3587402403354645, + "logits/chosen": -0.9945312738418579, + "logits/rejected": -0.9126952886581421, + "logps/chosen": -0.629687488079071, + "logps/rejected": -1.862695336341858, + "loss": 0.7204, + "nll_loss": 0.708691418170929, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06302490085363388, + "rewards/margins": 0.12322998046875, + "rewards/rejected": -0.18605956435203552, + "step": 9250 + }, + { + "epoch": 0.7026862953407194, + "grad_norm": 3.199962352835074, + "learning_rate": 8.313511342527453e-07, + "log_odds_chosen": 1.432519555091858, + "log_odds_ratio": -0.45488280057907104, + "logits/chosen": -1.012304663658142, + "logits/rejected": -0.9175781011581421, + "logps/chosen": -0.640429675579071, + "logps/rejected": -1.6814453601837158, + "loss": 0.7062, + "nll_loss": 0.6934570074081421, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.06409911811351776, + "rewards/margins": 0.10413818061351776, + "rewards/rejected": -0.16818848252296448, + "step": 9260 + }, + { + "epoch": 0.703445135832448, + "grad_norm": 3.0159947288223945, + "learning_rate": 8.309026037996745e-07, + "log_odds_chosen": 1.75146484375, + "log_odds_ratio": -0.35882568359375, + "logits/chosen": -1.0076172351837158, + "logits/rejected": -0.9087890386581421, + "logps/chosen": -0.5914062261581421, + "logps/rejected": -1.811914086341858, + "loss": 0.7019, + "nll_loss": 0.6719726324081421, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.05911865085363388, + "rewards/margins": 0.12203369289636612, + "rewards/rejected": -0.18107910454273224, + "step": 9270 + }, + { + "epoch": 0.7042039763241766, + "grad_norm": 3.052470283610497, + "learning_rate": 8.304547985373996e-07, + "log_odds_chosen": 1.763574242591858, + "log_odds_ratio": -0.3483032286167145, + "logits/chosen": -1.0441405773162842, + "logits/rejected": -0.939453125, + "logps/chosen": -0.6279296875, + "logps/rejected": -1.82421875, + "loss": 0.685, + "nll_loss": 0.633007824420929, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.06272582709789276, + "rewards/margins": 0.11971130222082138, + "rewards/rejected": -0.1824951171875, + "step": 9280 + }, + { + "epoch": 0.7049628168159053, + "grad_norm": 3.4082513013731766, + "learning_rate": 8.300077165138592e-07, + "log_odds_chosen": 1.7136719226837158, + "log_odds_ratio": -0.3604492247104645, + "logits/chosen": -1.038476586341858, + "logits/rejected": -0.924609363079071, + "logps/chosen": -0.620410144329071, + "logps/rejected": -1.804296851158142, + "loss": 0.6991, + "nll_loss": 0.687304675579071, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06206054612994194, + "rewards/margins": 0.11837158352136612, + "rewards/rejected": -0.18034668266773224, + "step": 9290 + }, + { + "epoch": 0.7057216573076339, + "grad_norm": 2.9015258805794555, + "learning_rate": 8.295613557843402e-07, + "log_odds_chosen": 1.6443359851837158, + "log_odds_ratio": -0.3910156190395355, + "logits/chosen": -0.9951171875, + "logits/rejected": -0.9296875, + "logps/chosen": -0.67236328125, + "logps/rejected": -1.8214843273162842, + "loss": 0.7376, + "nll_loss": 0.7109375, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06719970703125, + "rewards/margins": 0.11496581882238388, + "rewards/rejected": -0.1822509765625, + "step": 9300 + }, + { + "epoch": 0.7064804977993626, + "grad_norm": 2.939785147512604, + "learning_rate": 8.291157144114419e-07, + "log_odds_chosen": 1.4240233898162842, + "log_odds_ratio": -0.4503173828125, + "logits/chosen": -0.9984375238418579, + "logits/rejected": -0.904296875, + "logps/chosen": -0.712109386920929, + "logps/rejected": -1.7039062976837158, + "loss": 0.7093, + "nll_loss": 0.709179699420929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.07125244289636612, + "rewards/margins": 0.09906616061925888, + "rewards/rejected": -0.1702880859375, + "step": 9310 + }, + { + "epoch": 0.7072393382910912, + "grad_norm": 2.600650906164271, + "learning_rate": 8.286707904650417e-07, + "log_odds_chosen": 1.435156226158142, + "log_odds_ratio": -0.41533201932907104, + "logits/chosen": -1.022070288658142, + "logits/rejected": -0.9398437738418579, + "logps/chosen": -0.6942383050918579, + "logps/rejected": -1.678125023841858, + "loss": 0.7282, + "nll_loss": 0.76171875, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06944580376148224, + "rewards/margins": 0.09825439751148224, + "rewards/rejected": -0.16779784858226776, + "step": 9320 + }, + { + "epoch": 0.7079981787828199, + "grad_norm": 3.2070573741611685, + "learning_rate": 8.282265820222593e-07, + "log_odds_chosen": 1.4519531726837158, + "log_odds_ratio": -0.402587890625, + "logits/chosen": -0.99609375, + "logits/rejected": -0.8941406011581421, + "logps/chosen": -0.6220703125, + "logps/rejected": -1.6042969226837158, + "loss": 0.7007, + "nll_loss": 0.659472644329071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06219482421875, + "rewards/margins": 0.09814453125, + "rewards/rejected": -0.16032715141773224, + "step": 9330 + }, + { + "epoch": 0.7087570192745485, + "grad_norm": 2.8976755408991886, + "learning_rate": 8.277830871674222e-07, + "log_odds_chosen": 1.535058617591858, + "log_odds_ratio": -0.38789063692092896, + "logits/chosen": -0.9917968511581421, + "logits/rejected": -0.8589843511581421, + "logps/chosen": -0.6283203363418579, + "logps/rejected": -1.6736328601837158, + "loss": 0.7335, + "nll_loss": 0.61328125, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06282959133386612, + "rewards/margins": 0.10449524223804474, + "rewards/rejected": -0.16738280653953552, + "step": 9340 + }, + { + "epoch": 0.7095158597662772, + "grad_norm": 3.3987506799493086, + "learning_rate": 8.273403039920306e-07, + "log_odds_chosen": 1.495996117591858, + "log_odds_ratio": -0.37250977754592896, + "logits/chosen": -1.0451171398162842, + "logits/rejected": -0.9439452886581421, + "logps/chosen": -0.6737304925918579, + "logps/rejected": -1.664453148841858, + "loss": 0.7109, + "nll_loss": 0.725781261920929, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06737060844898224, + "rewards/margins": 0.09909667819738388, + "rewards/rejected": -0.16655273735523224, + "step": 9350 + }, + { + "epoch": 0.7102747002580058, + "grad_norm": 3.7816459933046587, + "learning_rate": 8.268982305947231e-07, + "log_odds_chosen": 1.4901854991912842, + "log_odds_ratio": -0.39619141817092896, + "logits/chosen": -1.0166015625, + "logits/rejected": -0.8984375, + "logps/chosen": -0.6463867425918579, + "logps/rejected": -1.638085961341858, + "loss": 0.7177, + "nll_loss": 0.7064453363418579, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06459961086511612, + "rewards/margins": 0.09928588569164276, + "rewards/rejected": -0.16384276747703552, + "step": 9360 + }, + { + "epoch": 0.7110335407497345, + "grad_norm": 2.6035328018623867, + "learning_rate": 8.264568650812423e-07, + "log_odds_chosen": 1.539648413658142, + "log_odds_ratio": -0.3548828065395355, + "logits/chosen": -1.0623047351837158, + "logits/rejected": -0.9449218511581421, + "logps/chosen": -0.638964831829071, + "logps/rejected": -1.673828125, + "loss": 0.7036, + "nll_loss": 0.6361328363418579, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06395263969898224, + "rewards/margins": 0.10349731147289276, + "rewards/rejected": -0.1673583984375, + "step": 9370 + }, + { + "epoch": 0.711792381241463, + "grad_norm": 3.1513163902945993, + "learning_rate": 8.26016205564401e-07, + "log_odds_chosen": 1.645898461341858, + "log_odds_ratio": -0.36396485567092896, + "logits/chosen": -1.0388672351837158, + "logits/rejected": -0.927929699420929, + "logps/chosen": -0.639355480670929, + "logps/rejected": -1.76953125, + "loss": 0.6799, + "nll_loss": 0.664843738079071, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06398925930261612, + "rewards/margins": 0.11285400390625, + "rewards/rejected": -0.17683105170726776, + "step": 9380 + }, + { + "epoch": 0.7125512217331916, + "grad_norm": 2.9624290128543147, + "learning_rate": 8.25576250164048e-07, + "log_odds_chosen": 1.588281273841858, + "log_odds_ratio": -0.3607177734375, + "logits/chosen": -1.052148461341858, + "logits/rejected": -0.9164062738418579, + "logps/chosen": -0.6328125, + "logps/rejected": -1.719140648841858, + "loss": 0.7075, + "nll_loss": 0.6480468511581421, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06333007663488388, + "rewards/margins": 0.10847778618335724, + "rewards/rejected": -0.17189940810203552, + "step": 9390 + }, + { + "epoch": 0.7133100622249203, + "grad_norm": 3.098491141706027, + "learning_rate": 8.251369970070346e-07, + "log_odds_chosen": 1.573632836341858, + "log_odds_ratio": -0.404296875, + "logits/chosen": -1.037695288658142, + "logits/rejected": -0.8955078125, + "logps/chosen": -0.636425793170929, + "logps/rejected": -1.751953125, + "loss": 0.7252, + "nll_loss": 0.6932617425918579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06368408352136612, + "rewards/margins": 0.11151123046875, + "rewards/rejected": -0.17529296875, + "step": 9400 + }, + { + "epoch": 0.7140689027166489, + "grad_norm": 3.4785135599316175, + "learning_rate": 8.246984442271813e-07, + "log_odds_chosen": 1.3253905773162842, + "log_odds_ratio": -0.4205566346645355, + "logits/chosen": -1.0634765625, + "logits/rejected": -0.9535156488418579, + "logps/chosen": -0.626660168170929, + "logps/rejected": -1.505273461341858, + "loss": 0.7136, + "nll_loss": 0.676464855670929, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06265868991613388, + "rewards/margins": 0.08788452297449112, + "rewards/rejected": -0.1505126953125, + "step": 9410 + }, + { + "epoch": 0.7148277432083776, + "grad_norm": 4.066118106757687, + "learning_rate": 8.242605899652435e-07, + "log_odds_chosen": 1.559960961341858, + "log_odds_ratio": -0.3536376953125, + "logits/chosen": -1.0441405773162842, + "logits/rejected": -0.917187511920929, + "logps/chosen": -0.595507800579071, + "logps/rejected": -1.647070288658142, + "loss": 0.7127, + "nll_loss": 0.6332031488418579, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.0595703125, + "rewards/margins": 0.10518188774585724, + "rewards/rejected": -0.16462402045726776, + "step": 9420 + }, + { + "epoch": 0.7155865837001062, + "grad_norm": 2.778220956023162, + "learning_rate": 8.238234323688798e-07, + "log_odds_chosen": 1.290380835533142, + "log_odds_ratio": -0.43852537870407104, + "logits/chosen": -0.970507800579071, + "logits/rejected": -0.879687488079071, + "logps/chosen": -0.62255859375, + "logps/rejected": -1.532812476158142, + "loss": 0.7116, + "nll_loss": 0.7496093511581421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06223144382238388, + "rewards/margins": 0.09091796725988388, + "rewards/rejected": -0.15324707329273224, + "step": 9430 + }, + { + "epoch": 0.7163454241918349, + "grad_norm": 3.0909960504122536, + "learning_rate": 8.233869695926182e-07, + "log_odds_chosen": 1.1931641101837158, + "log_odds_ratio": -0.4408203065395355, + "logits/chosen": -1.0041015148162842, + "logits/rejected": -0.9169921875, + "logps/chosen": -0.65625, + "logps/rejected": -1.42578125, + "loss": 0.7176, + "nll_loss": 0.7022460699081421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06563720852136612, + "rewards/margins": 0.07687988132238388, + "rewards/rejected": -0.14260253310203552, + "step": 9440 + }, + { + "epoch": 0.7171042646835635, + "grad_norm": 2.8911447642832693, + "learning_rate": 8.229511997978235e-07, + "log_odds_chosen": 1.635351538658142, + "log_odds_ratio": -0.340087890625, + "logits/chosen": -1.026953101158142, + "logits/rejected": -0.8980468511581421, + "logps/chosen": -0.627734363079071, + "logps/rejected": -1.747656226158142, + "loss": 0.7057, + "nll_loss": 0.7240234613418579, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.06282959133386612, + "rewards/margins": 0.11191406100988388, + "rewards/rejected": -0.1746826171875, + "step": 9450 + }, + { + "epoch": 0.7178631051752922, + "grad_norm": 2.8737807569712803, + "learning_rate": 8.22516121152665e-07, + "log_odds_chosen": 1.576269507408142, + "log_odds_ratio": -0.3546386659145355, + "logits/chosen": -1.055078148841858, + "logits/rejected": -0.9457031488418579, + "logps/chosen": -0.65966796875, + "logps/rejected": -1.761328101158142, + "loss": 0.688, + "nll_loss": 0.6517578363418579, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06590576469898224, + "rewards/margins": 0.11024169623851776, + "rewards/rejected": -0.17629393935203552, + "step": 9460 + }, + { + "epoch": 0.7186219456670208, + "grad_norm": 2.5498810925172175, + "learning_rate": 8.220817318320836e-07, + "log_odds_chosen": 1.574804663658142, + "log_odds_ratio": -0.41093748807907104, + "logits/chosen": -1.0486328601837158, + "logits/rejected": -0.9603515863418579, + "logps/chosen": -0.638378918170929, + "logps/rejected": -1.734765648841858, + "loss": 0.7249, + "nll_loss": 0.6749023199081421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.0638427734375, + "rewards/margins": 0.10963134467601776, + "rewards/rejected": -0.17338867485523224, + "step": 9470 + }, + { + "epoch": 0.7193807861587495, + "grad_norm": 2.5597559134110615, + "learning_rate": 8.216480300177611e-07, + "log_odds_chosen": 1.4949219226837158, + "log_odds_ratio": -0.4126953184604645, + "logits/chosen": -0.9994140863418579, + "logits/rejected": -0.9130859375, + "logps/chosen": -0.6241210699081421, + "logps/rejected": -1.637109398841858, + "loss": 0.7042, + "nll_loss": 0.671679675579071, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.06241454929113388, + "rewards/margins": 0.10139159858226776, + "rewards/rejected": -0.1636962890625, + "step": 9480 + }, + { + "epoch": 0.720139626650478, + "grad_norm": 2.942311840429656, + "learning_rate": 8.212150138980857e-07, + "log_odds_chosen": 1.49365234375, + "log_odds_ratio": -0.39448243379592896, + "logits/chosen": -1.0380859375, + "logits/rejected": -0.9300781488418579, + "logps/chosen": -0.6373046636581421, + "logps/rejected": -1.6593749523162842, + "loss": 0.6918, + "nll_loss": 0.686328113079071, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06378173828125, + "rewards/margins": 0.10233154147863388, + "rewards/rejected": -0.166015625, + "step": 9490 + }, + { + "epoch": 0.7208984671422067, + "grad_norm": 2.745660460923844, + "learning_rate": 8.207826816681233e-07, + "log_odds_chosen": 1.2062499523162842, + "log_odds_ratio": -0.46992188692092896, + "logits/chosen": -1.0085937976837158, + "logits/rejected": -0.9095703363418579, + "logps/chosen": -0.667773425579071, + "logps/rejected": -1.4835937023162842, + "loss": 0.7297, + "nll_loss": 0.725781261920929, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06674804538488388, + "rewards/margins": 0.08168945461511612, + "rewards/rejected": -0.14841309189796448, + "step": 9500 + }, + { + "epoch": 0.7216573076339353, + "grad_norm": 4.0009419229614, + "learning_rate": 8.203510315295829e-07, + "log_odds_chosen": 1.691796898841858, + "log_odds_ratio": -0.31611329317092896, + "logits/chosen": -1.0703125, + "logits/rejected": -0.8960937261581421, + "logps/chosen": -0.6041015386581421, + "logps/rejected": -1.7589843273162842, + "loss": 0.7152, + "nll_loss": 0.687792956829071, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06040038913488388, + "rewards/margins": 0.11550293117761612, + "rewards/rejected": -0.17592772841453552, + "step": 9510 + }, + { + "epoch": 0.722416148125664, + "grad_norm": 3.1941615556516125, + "learning_rate": 8.199200616907878e-07, + "log_odds_chosen": 1.6658203601837158, + "log_odds_ratio": -0.35419923067092896, + "logits/chosen": -1.046289086341858, + "logits/rejected": -0.9287109375, + "logps/chosen": -0.6255859136581421, + "logps/rejected": -1.803125023841858, + "loss": 0.7029, + "nll_loss": 0.6260741949081421, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06258545070886612, + "rewards/margins": 0.11777343600988388, + "rewards/rejected": -0.18037109076976776, + "step": 9520 + }, + { + "epoch": 0.7231749886173926, + "grad_norm": 3.049993025984763, + "learning_rate": 8.194897703666421e-07, + "log_odds_chosen": 1.5374023914337158, + "log_odds_ratio": -0.35200196504592896, + "logits/chosen": -1.004296898841858, + "logits/rejected": -0.8941406011581421, + "logps/chosen": -0.619335949420929, + "logps/rejected": -1.7029297351837158, + "loss": 0.6998, + "nll_loss": 0.6654297113418579, + "rewards/accuracies": 0.856249988079071, + "rewards/chosen": -0.06191406399011612, + "rewards/margins": 0.10847778618335724, + "rewards/rejected": -0.17036132514476776, + "step": 9530 + }, + { + "epoch": 0.7239338291091213, + "grad_norm": 2.871771558134182, + "learning_rate": 8.190601557786015e-07, + "log_odds_chosen": 1.481347680091858, + "log_odds_ratio": -0.40869140625, + "logits/chosen": -1.020898461341858, + "logits/rejected": -0.909960925579071, + "logps/chosen": -0.626953125, + "logps/rejected": -1.6613280773162842, + "loss": 0.7111, + "nll_loss": 0.688281238079071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06268310546875, + "rewards/margins": 0.10338135063648224, + "rewards/rejected": -0.16608886420726776, + "step": 9540 + }, + { + "epoch": 0.7246926696008499, + "grad_norm": 2.9624145394028956, + "learning_rate": 8.186312161546413e-07, + "log_odds_chosen": 1.67919921875, + "log_odds_ratio": -0.3866210877895355, + "logits/chosen": -1.064062476158142, + "logits/rejected": -0.9556640386581421, + "logps/chosen": -0.6200195550918579, + "logps/rejected": -1.801171898841858, + "loss": 0.6751, + "nll_loss": 0.6309570074081421, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06196289137005806, + "rewards/margins": 0.11810302734375, + "rewards/rejected": -0.1800537109375, + "step": 9550 + }, + { + "epoch": 0.7254515100925786, + "grad_norm": 2.5334843972987664, + "learning_rate": 8.182029497292262e-07, + "log_odds_chosen": 1.605371117591858, + "log_odds_ratio": -0.4095703065395355, + "logits/chosen": -1.086523413658142, + "logits/rejected": -0.967578113079071, + "logps/chosen": -0.695507824420929, + "logps/rejected": -1.8291015625, + "loss": 0.7053, + "nll_loss": 0.693652331829071, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06959228217601776, + "rewards/margins": 0.11314697563648224, + "rewards/rejected": -0.18266601860523224, + "step": 9560 + }, + { + "epoch": 0.7262103505843072, + "grad_norm": 4.016065116443712, + "learning_rate": 8.177753547432792e-07, + "log_odds_chosen": 1.692968726158142, + "log_odds_ratio": -0.35295408964157104, + "logits/chosen": -1.01953125, + "logits/rejected": -0.9164062738418579, + "logps/chosen": -0.6377929449081421, + "logps/rejected": -1.80859375, + "loss": 0.7054, + "nll_loss": 0.682910144329071, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.06374511867761612, + "rewards/margins": 0.11716308444738388, + "rewards/rejected": -0.18085937201976776, + "step": 9570 + }, + { + "epoch": 0.7269691910760359, + "grad_norm": 4.45337210149427, + "learning_rate": 8.173484294441524e-07, + "log_odds_chosen": 1.3463866710662842, + "log_odds_ratio": -0.4344726502895355, + "logits/chosen": -1.000585913658142, + "logits/rejected": -0.912109375, + "logps/chosen": -0.6314452886581421, + "logps/rejected": -1.586523413658142, + "loss": 0.7228, + "nll_loss": 0.7105468511581421, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06313476711511612, + "rewards/margins": 0.09547729790210724, + "rewards/rejected": -0.15849609673023224, + "step": 9580 + }, + { + "epoch": 0.7277280315677644, + "grad_norm": 3.093014359022252, + "learning_rate": 8.169221720855952e-07, + "log_odds_chosen": 1.443212866783142, + "log_odds_ratio": -0.37128907442092896, + "logits/chosen": -1.045507788658142, + "logits/rejected": -0.950976550579071, + "logps/chosen": -0.588085949420929, + "logps/rejected": -1.5234375, + "loss": 0.7095, + "nll_loss": 0.6708008050918579, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.05874023586511612, + "rewards/margins": 0.09354247897863388, + "rewards/rejected": -0.15234375, + "step": 9590 + }, + { + "epoch": 0.7284868720594931, + "grad_norm": 2.8045353815731273, + "learning_rate": 8.164965809277261e-07, + "log_odds_chosen": 1.424414038658142, + "log_odds_ratio": -0.37666016817092896, + "logits/chosen": -1.0439453125, + "logits/rejected": -0.9107421636581421, + "logps/chosen": -0.6259765625, + "logps/rejected": -1.5779297351837158, + "loss": 0.6969, + "nll_loss": 0.6421874761581421, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06264648586511612, + "rewards/margins": 0.09520263969898224, + "rewards/rejected": -0.15773925185203552, + "step": 9600 + }, + { + "epoch": 0.7292457125512217, + "grad_norm": 2.761496333392702, + "learning_rate": 8.160716542370011e-07, + "log_odds_chosen": 1.6564452648162842, + "log_odds_ratio": -0.3475585877895355, + "logits/chosen": -1.098046898841858, + "logits/rejected": -0.9603515863418579, + "logps/chosen": -0.6324218511581421, + "logps/rejected": -1.783593773841858, + "loss": 0.7157, + "nll_loss": 0.6709960699081421, + "rewards/accuracies": 0.8687499761581421, + "rewards/chosen": -0.06326904147863388, + "rewards/margins": 0.11512450873851776, + "rewards/rejected": -0.17827148735523224, + "step": 9610 + }, + { + "epoch": 0.7300045530429504, + "grad_norm": 3.015306363526461, + "learning_rate": 8.156473902861856e-07, + "log_odds_chosen": 1.65771484375, + "log_odds_ratio": -0.3919921815395355, + "logits/chosen": -1.0265624523162842, + "logits/rejected": -0.9195312261581421, + "logps/chosen": -0.6045898199081421, + "logps/rejected": -1.729882836341858, + "loss": 0.6856, + "nll_loss": 0.664746105670929, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06038818508386612, + "rewards/margins": 0.11270751804113388, + "rewards/rejected": -0.17307129502296448, + "step": 9620 + }, + { + "epoch": 0.730763393534679, + "grad_norm": 3.3445443161524615, + "learning_rate": 8.152237873543241e-07, + "log_odds_chosen": 1.431640625, + "log_odds_ratio": -0.4449218809604645, + "logits/chosen": -0.990039050579071, + "logits/rejected": -0.8568359613418579, + "logps/chosen": -0.6485351324081421, + "logps/rejected": -1.675390601158142, + "loss": 0.7148, + "nll_loss": 0.7191406488418579, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06489257514476776, + "rewards/margins": 0.10255126655101776, + "rewards/rejected": -0.16757813096046448, + "step": 9630 + }, + { + "epoch": 0.7315222340264077, + "grad_norm": 3.750217860575627, + "learning_rate": 8.148008437267104e-07, + "log_odds_chosen": 1.6515624523162842, + "log_odds_ratio": -0.3653808534145355, + "logits/chosen": -1.0380859375, + "logits/rejected": -0.95703125, + "logps/chosen": -0.6327148675918579, + "logps/rejected": -1.7820312976837158, + "loss": 0.7153, + "nll_loss": 0.673828125, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06325683742761612, + "rewards/margins": 0.11501464992761612, + "rewards/rejected": -0.17824706435203552, + "step": 9640 + }, + { + "epoch": 0.7322810745181363, + "grad_norm": 3.7236221175889663, + "learning_rate": 8.143785576948602e-07, + "log_odds_chosen": 1.3380858898162842, + "log_odds_ratio": -0.38300782442092896, + "logits/chosen": -0.994140625, + "logits/rejected": -0.874804675579071, + "logps/chosen": -0.5947265625, + "logps/rejected": -1.4792969226837158, + "loss": 0.7217, + "nll_loss": 0.673535168170929, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.05947265774011612, + "rewards/margins": 0.08846435695886612, + "rewards/rejected": -0.14794921875, + "step": 9650 + }, + { + "epoch": 0.7330399150098649, + "grad_norm": 3.6288611870242407, + "learning_rate": 8.139569275564796e-07, + "log_odds_chosen": 1.61181640625, + "log_odds_ratio": -0.3910156190395355, + "logits/chosen": -0.9994140863418579, + "logits/rejected": -0.878710925579071, + "logps/chosen": -0.719531238079071, + "logps/rejected": -1.8894531726837158, + "loss": 0.7069, + "nll_loss": 0.728515625, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.072021484375, + "rewards/margins": 0.11671753227710724, + "rewards/rejected": -0.188720703125, + "step": 9660 + }, + { + "epoch": 0.7337987555015936, + "grad_norm": 2.950044313068542, + "learning_rate": 8.135359516154388e-07, + "log_odds_chosen": 1.6580078601837158, + "log_odds_ratio": -0.33281248807907104, + "logits/chosen": -1.054101586341858, + "logits/rejected": -0.9556640386581421, + "logps/chosen": -0.6055663824081421, + "logps/rejected": -1.7160155773162842, + "loss": 0.7187, + "nll_loss": 0.6265624761581421, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.06060791015625, + "rewards/margins": 0.11098632961511612, + "rewards/rejected": -0.1715087890625, + "step": 9670 + }, + { + "epoch": 0.7345575959933222, + "grad_norm": 3.0822414004750263, + "learning_rate": 8.131156281817418e-07, + "log_odds_chosen": 1.7685546875, + "log_odds_ratio": -0.32861328125, + "logits/chosen": -1.0246093273162842, + "logits/rejected": -0.902148425579071, + "logps/chosen": -0.5926758050918579, + "logps/rejected": -1.786718726158142, + "loss": 0.7261, + "nll_loss": 0.655468761920929, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.05927734449505806, + "rewards/margins": 0.11955566704273224, + "rewards/rejected": -0.17875976860523224, + "step": 9680 + }, + { + "epoch": 0.7353164364850509, + "grad_norm": 2.615596741963579, + "learning_rate": 8.126959555714979e-07, + "log_odds_chosen": 1.173095703125, + "log_odds_ratio": -0.4927734434604645, + "logits/chosen": -1.0300781726837158, + "logits/rejected": -0.912109375, + "logps/chosen": -0.6563476324081421, + "logps/rejected": -1.4714844226837158, + "loss": 0.7003, + "nll_loss": 0.63525390625, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.06563720852136612, + "rewards/margins": 0.0815223678946495, + "rewards/rejected": -0.14711913466453552, + "step": 9690 + }, + { + "epoch": 0.7360752769767794, + "grad_norm": 3.360149813748261, + "learning_rate": 8.122769321068952e-07, + "log_odds_chosen": 1.746484398841858, + "log_odds_ratio": -0.3385253846645355, + "logits/chosen": -0.9775390625, + "logits/rejected": -0.888671875, + "logps/chosen": -0.6509765386581421, + "logps/rejected": -1.8640625476837158, + "loss": 0.6869, + "nll_loss": 0.673828125, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06505737453699112, + "rewards/margins": 0.12130127102136612, + "rewards/rejected": -0.18649902939796448, + "step": 9700 + }, + { + "epoch": 0.7368341174685081, + "grad_norm": 2.909685109657297, + "learning_rate": 8.118585561161698e-07, + "log_odds_chosen": 1.688574194908142, + "log_odds_ratio": -0.3746093809604645, + "logits/chosen": -1.053320288658142, + "logits/rejected": -0.9369140863418579, + "logps/chosen": -0.5865234136581421, + "logps/rejected": -1.7595703601837158, + "loss": 0.7144, + "nll_loss": 0.708691418170929, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.05860595777630806, + "rewards/margins": 0.11747436225414276, + "rewards/rejected": -0.176025390625, + "step": 9710 + }, + { + "epoch": 0.7375929579602367, + "grad_norm": 3.189014729667494, + "learning_rate": 8.114408259335793e-07, + "log_odds_chosen": 1.705468773841858, + "log_odds_ratio": -0.35576170682907104, + "logits/chosen": -1.022070288658142, + "logits/rejected": -0.9437500238418579, + "logps/chosen": -0.5956054925918579, + "logps/rejected": -1.735937476158142, + "loss": 0.6708, + "nll_loss": 0.6220703125, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.05959472805261612, + "rewards/margins": 0.11409912258386612, + "rewards/rejected": -0.17365722358226776, + "step": 9720 + }, + { + "epoch": 0.7383517984519654, + "grad_norm": 3.077063349879149, + "learning_rate": 8.110237398993754e-07, + "log_odds_chosen": 1.367919921875, + "log_odds_ratio": -0.43510740995407104, + "logits/chosen": -1.018164038658142, + "logits/rejected": -0.95703125, + "logps/chosen": -0.6788085699081421, + "logps/rejected": -1.609960913658142, + "loss": 0.7031, + "nll_loss": 0.65380859375, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06787109375, + "rewards/margins": 0.09309692680835724, + "rewards/rejected": -0.1611328125, + "step": 9730 + }, + { + "epoch": 0.739110638943694, + "grad_norm": 2.748602718265524, + "learning_rate": 8.106072963597751e-07, + "log_odds_chosen": 1.61865234375, + "log_odds_ratio": -0.3550781309604645, + "logits/chosen": -1.0451171398162842, + "logits/rejected": -0.933789074420929, + "logps/chosen": -0.582714855670929, + "logps/rejected": -1.6845703125, + "loss": 0.7167, + "nll_loss": 0.63525390625, + "rewards/accuracies": 0.8187500238418579, + "rewards/chosen": -0.05832519382238388, + "rewards/margins": 0.11011962592601776, + "rewards/rejected": -0.16845703125, + "step": 9740 + }, + { + "epoch": 0.7398694794354227, + "grad_norm": 3.268717899241557, + "learning_rate": 8.101914936669332e-07, + "log_odds_chosen": 1.5139648914337158, + "log_odds_ratio": -0.43901365995407104, + "logits/chosen": -1.012304663658142, + "logits/rejected": -0.916015625, + "logps/chosen": -0.6890624761581421, + "logps/rejected": -1.73828125, + "loss": 0.7166, + "nll_loss": 0.74951171875, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.0689697265625, + "rewards/margins": 0.10474853217601776, + "rewards/rejected": -0.17365722358226776, + "step": 9750 + }, + { + "epoch": 0.7406283199271513, + "grad_norm": 2.639596921121034, + "learning_rate": 8.09776330178916e-07, + "log_odds_chosen": 1.7824218273162842, + "log_odds_ratio": -0.35649412870407104, + "logits/chosen": -1.0369141101837158, + "logits/rejected": -0.915234386920929, + "logps/chosen": -0.634960949420929, + "logps/rejected": -1.90234375, + "loss": 0.7067, + "nll_loss": 0.61767578125, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.06353759765625, + "rewards/margins": 0.12664794921875, + "rewards/rejected": -0.19020995497703552, + "step": 9760 + }, + { + "epoch": 0.74138716041888, + "grad_norm": 2.727969831386695, + "learning_rate": 8.093618042596727e-07, + "log_odds_chosen": 1.7810547351837158, + "log_odds_ratio": -0.3067871034145355, + "logits/chosen": -0.9916015863418579, + "logits/rejected": -0.900390625, + "logps/chosen": -0.61669921875, + "logps/rejected": -1.8624999523162842, + "loss": 0.6931, + "nll_loss": 0.700390636920929, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -0.06169433519244194, + "rewards/margins": 0.12442626804113388, + "rewards/rejected": -0.18623046576976776, + "step": 9770 + }, + { + "epoch": 0.7421460009106086, + "grad_norm": 2.95089832822571, + "learning_rate": 8.089479142790095e-07, + "log_odds_chosen": 1.5273926258087158, + "log_odds_ratio": -0.3857665956020355, + "logits/chosen": -1.022851586341858, + "logits/rejected": -0.910351574420929, + "logps/chosen": -0.615527331829071, + "logps/rejected": -1.6533203125, + "loss": 0.7232, + "nll_loss": 0.693359375, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06160888820886612, + "rewards/margins": 0.10381469875574112, + "rewards/rejected": -0.165283203125, + "step": 9780 + }, + { + "epoch": 0.7429048414023373, + "grad_norm": 3.3794191970100678, + "learning_rate": 8.085346586125621e-07, + "log_odds_chosen": 1.370214819908142, + "log_odds_ratio": -0.3920654356479645, + "logits/chosen": -1.0105469226837158, + "logits/rejected": -0.9052734375, + "logps/chosen": -0.617871105670929, + "logps/rejected": -1.521093726158142, + "loss": 0.6906, + "nll_loss": 0.639453113079071, + "rewards/accuracies": 0.831250011920929, + "rewards/chosen": -0.06179199367761612, + "rewards/margins": 0.09033813327550888, + "rewards/rejected": -0.15212401747703552, + "step": 9790 + }, + { + "epoch": 0.7436636818940658, + "grad_norm": 3.8719705893956666, + "learning_rate": 8.081220356417685e-07, + "log_odds_chosen": 1.421484351158142, + "log_odds_ratio": -0.43310546875, + "logits/chosen": -1.0009765625, + "logits/rejected": -0.937695324420929, + "logps/chosen": -0.619921863079071, + "logps/rejected": -1.566015601158142, + "loss": 0.6978, + "nll_loss": 0.6644531488418579, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06198730319738388, + "rewards/margins": 0.09454345703125, + "rewards/rejected": -0.15642090141773224, + "step": 9800 + }, + { + "epoch": 0.7444225223857945, + "grad_norm": 2.9859368077072466, + "learning_rate": 8.077100437538435e-07, + "log_odds_chosen": 1.490820288658142, + "log_odds_ratio": -0.3731445372104645, + "logits/chosen": -1.071679711341858, + "logits/rejected": -0.959765613079071, + "logps/chosen": -0.6390625238418579, + "logps/rejected": -1.635351538658142, + "loss": 0.7135, + "nll_loss": 0.611523449420929, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.06390380859375, + "rewards/margins": 0.09953613579273224, + "rewards/rejected": -0.16350097954273224, + "step": 9810 + }, + { + "epoch": 0.7451813628775231, + "grad_norm": 2.9458682593614287, + "learning_rate": 8.072986813417512e-07, + "log_odds_chosen": 1.2968261241912842, + "log_odds_ratio": -0.40947264432907104, + "logits/chosen": -1.021484375, + "logits/rejected": -0.9173828363418579, + "logps/chosen": -0.640917956829071, + "logps/rejected": -1.514257788658142, + "loss": 0.7177, + "nll_loss": 0.6927734613418579, + "rewards/accuracies": 0.768750011920929, + "rewards/chosen": -0.06409911811351776, + "rewards/margins": 0.08727417141199112, + "rewards/rejected": -0.15144042670726776, + "step": 9820 + }, + { + "epoch": 0.7459402033692518, + "grad_norm": 2.7256634641769493, + "learning_rate": 8.068879468041791e-07, + "log_odds_chosen": 1.596777319908142, + "log_odds_ratio": -0.37910157442092896, + "logits/chosen": -1.0333983898162842, + "logits/rejected": -0.8929687738418579, + "logps/chosen": -0.6444336175918579, + "logps/rejected": -1.7619140148162842, + "loss": 0.7021, + "nll_loss": 0.6631835699081421, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06450195610523224, + "rewards/margins": 0.11162719875574112, + "rewards/rejected": -0.1761474609375, + "step": 9830 + }, + { + "epoch": 0.7466990438609804, + "grad_norm": 2.461950975465807, + "learning_rate": 8.064778385455118e-07, + "log_odds_chosen": 1.403710961341858, + "log_odds_ratio": -0.42656248807907104, + "logits/chosen": -1.055273413658142, + "logits/rejected": -0.931640625, + "logps/chosen": -0.662109375, + "logps/rejected": -1.67578125, + "loss": 0.6893, + "nll_loss": 0.674023449420929, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06617431342601776, + "rewards/margins": 0.10143432766199112, + "rewards/rejected": -0.16770020127296448, + "step": 9840 + }, + { + "epoch": 0.7474578843527091, + "grad_norm": 2.9381430618600692, + "learning_rate": 8.060683549758054e-07, + "log_odds_chosen": 1.464453101158142, + "log_odds_ratio": -0.39643555879592896, + "logits/chosen": -1.0138671398162842, + "logits/rejected": -0.916015625, + "logps/chosen": -0.6651366949081421, + "logps/rejected": -1.657812476158142, + "loss": 0.7252, + "nll_loss": 0.696582019329071, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06650390475988388, + "rewards/margins": 0.09929809719324112, + "rewards/rejected": -0.16567382216453552, + "step": 9850 + }, + { + "epoch": 0.7482167248444377, + "grad_norm": 3.3001354673190106, + "learning_rate": 8.056594945107608e-07, + "log_odds_chosen": 1.5481445789337158, + "log_odds_ratio": -0.412353515625, + "logits/chosen": -1.0177733898162842, + "logits/rejected": -0.9375, + "logps/chosen": -0.6268554925918579, + "logps/rejected": -1.6853516101837158, + "loss": 0.7111, + "nll_loss": 0.6719726324081421, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06265868991613388, + "rewards/margins": 0.10568847507238388, + "rewards/rejected": -0.16846923530101776, + "step": 9860 + }, + { + "epoch": 0.7489755653361664, + "grad_norm": 3.5461519330612243, + "learning_rate": 8.052512555716987e-07, + "log_odds_chosen": 1.743554711341858, + "log_odds_ratio": -0.3538574278354645, + "logits/chosen": -1.0109374523162842, + "logits/rejected": -0.945507824420929, + "logps/chosen": -0.659960925579071, + "logps/rejected": -1.8671875, + "loss": 0.6991, + "nll_loss": 0.6568359136581421, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06595458835363388, + "rewards/margins": 0.12074585258960724, + "rewards/rejected": -0.18659667670726776, + "step": 9870 + }, + { + "epoch": 0.749734405827895, + "grad_norm": 2.67020379560676, + "learning_rate": 8.048436365855337e-07, + "log_odds_chosen": 1.1943359375, + "log_odds_ratio": -0.4874511659145355, + "logits/chosen": -0.988476574420929, + "logits/rejected": -0.89453125, + "logps/chosen": -0.6356445550918579, + "logps/rejected": -1.4871094226837158, + "loss": 0.6852, + "nll_loss": 0.6796875, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.06356201320886612, + "rewards/margins": 0.08510742336511612, + "rewards/rejected": -0.14865723252296448, + "step": 9880 + }, + { + "epoch": 0.7504932463196237, + "grad_norm": 2.8341309625851516, + "learning_rate": 8.044366359847486e-07, + "log_odds_chosen": 1.6677734851837158, + "log_odds_ratio": -0.38178712129592896, + "logits/chosen": -1.039648413658142, + "logits/rejected": -0.911328136920929, + "logps/chosen": -0.6180664300918579, + "logps/rejected": -1.732812523841858, + "loss": 0.6864, + "nll_loss": 0.610058605670929, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06181640550494194, + "rewards/margins": 0.11149291694164276, + "rewards/rejected": -0.1732177734375, + "step": 9890 + }, + { + "epoch": 0.7512520868113522, + "grad_norm": 2.8976394764539113, + "learning_rate": 8.040302522073696e-07, + "log_odds_chosen": 1.7229492664337158, + "log_odds_ratio": -0.3226074278354645, + "logits/chosen": -0.993945300579071, + "logits/rejected": -0.865039050579071, + "logps/chosen": -0.601367175579071, + "logps/rejected": -1.783203125, + "loss": 0.6997, + "nll_loss": 0.6250976324081421, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.06004638597369194, + "rewards/margins": 0.11831054836511612, + "rewards/rejected": -0.17836913466453552, + "step": 9900 + }, + { + "epoch": 0.752010927303081, + "grad_norm": 3.3050263093017582, + "learning_rate": 8.036244836969407e-07, + "log_odds_chosen": 1.7648437023162842, + "log_odds_ratio": -0.3529296815395355, + "logits/chosen": -1.0378906726837158, + "logits/rejected": -0.9375, + "logps/chosen": -0.5956054925918579, + "logps/rejected": -1.8035156726837158, + "loss": 0.7065, + "nll_loss": 0.6626952886581421, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.05949706956744194, + "rewards/margins": 0.12080077826976776, + "rewards/rejected": -0.18034668266773224, + "step": 9910 + }, + { + "epoch": 0.7527697677948095, + "grad_norm": 2.70363681667673, + "learning_rate": 8.032193289024989e-07, + "log_odds_chosen": 1.8629882335662842, + "log_odds_ratio": -0.3124755918979645, + "logits/chosen": -1.049218773841858, + "logits/rejected": -0.894726574420929, + "logps/chosen": -0.6024414300918579, + "logps/rejected": -1.8781249523162842, + "loss": 0.7, + "nll_loss": 0.6827148199081421, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06024169921875, + "rewards/margins": 0.12754973769187927, + "rewards/rejected": -0.187744140625, + "step": 9920 + }, + { + "epoch": 0.7535286082865381, + "grad_norm": 3.6720386590938063, + "learning_rate": 8.02814786278549e-07, + "log_odds_chosen": 1.5441405773162842, + "log_odds_ratio": -0.41064453125, + "logits/chosen": -1.015039086341858, + "logits/rejected": -0.919921875, + "logps/chosen": -0.623339831829071, + "logps/rejected": -1.6804687976837158, + "loss": 0.7135, + "nll_loss": 0.7216796875, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06236572191119194, + "rewards/margins": 0.105712890625, + "rewards/rejected": -0.16799315810203552, + "step": 9930 + }, + { + "epoch": 0.7542874487782668, + "grad_norm": 3.2424502309904764, + "learning_rate": 8.024108542850394e-07, + "log_odds_chosen": 1.529296875, + "log_odds_ratio": -0.3722167909145355, + "logits/chosen": -1.028710961341858, + "logits/rejected": -0.923828125, + "logps/chosen": -0.6244140863418579, + "logps/rejected": -1.670312523841858, + "loss": 0.699, + "nll_loss": 0.6700195074081421, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06246338039636612, + "rewards/margins": 0.10474853217601776, + "rewards/rejected": -0.16708984971046448, + "step": 9940 + }, + { + "epoch": 0.7550462892699954, + "grad_norm": 2.775687366537809, + "learning_rate": 8.020075313873367e-07, + "log_odds_chosen": 1.4794921875, + "log_odds_ratio": -0.4207519590854645, + "logits/chosen": -1.0089843273162842, + "logits/rejected": -0.8949218988418579, + "logps/chosen": -0.6498047113418579, + "logps/rejected": -1.6681640148162842, + "loss": 0.6929, + "nll_loss": 0.6792968511581421, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": -0.06495361030101776, + "rewards/margins": 0.101806640625, + "rewards/rejected": -0.16684570908546448, + "step": 9950 + }, + { + "epoch": 0.7558051297617241, + "grad_norm": 2.9999965310002845, + "learning_rate": 8.016048160562023e-07, + "log_odds_chosen": 1.5947265625, + "log_odds_ratio": -0.3536621034145355, + "logits/chosen": -0.968945324420929, + "logits/rejected": -0.8851562738418579, + "logps/chosen": -0.5938476324081421, + "logps/rejected": -1.6232421398162842, + "loss": 0.6704, + "nll_loss": 0.606249988079071, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.0594482421875, + "rewards/margins": 0.10288085788488388, + "rewards/rejected": -0.1622314453125, + "step": 9960 + }, + { + "epoch": 0.7565639702534527, + "grad_norm": 3.0533768128561545, + "learning_rate": 8.012027067677667e-07, + "log_odds_chosen": 1.4152343273162842, + "log_odds_ratio": -0.4334960877895355, + "logits/chosen": -0.990039050579071, + "logits/rejected": -0.8798828125, + "logps/chosen": -0.645800769329071, + "logps/rejected": -1.5476562976837158, + "loss": 0.7097, + "nll_loss": 0.670117199420929, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.06455077975988388, + "rewards/margins": 0.09015502780675888, + "rewards/rejected": -0.15476074814796448, + "step": 9970 + }, + { + "epoch": 0.7573228107451814, + "grad_norm": 3.131970361190211, + "learning_rate": 8.008012020035062e-07, + "log_odds_chosen": 1.5291016101837158, + "log_odds_ratio": -0.41162109375, + "logits/chosen": -1.0380859375, + "logits/rejected": -0.9429687261581421, + "logps/chosen": -0.6058593988418579, + "logps/rejected": -1.608789086341858, + "loss": 0.6859, + "nll_loss": 0.6395508050918579, + "rewards/accuracies": 0.8062499761581421, + "rewards/chosen": -0.06052245944738388, + "rewards/margins": 0.10036621242761612, + "rewards/rejected": -0.16091307997703552, + "step": 9980 + }, + { + "epoch": 0.75808165123691, + "grad_norm": 2.890624157805145, + "learning_rate": 8.004003002502188e-07, + "log_odds_chosen": 1.6139647960662842, + "log_odds_ratio": -0.35957032442092896, + "logits/chosen": -1.037695288658142, + "logits/rejected": -0.873828113079071, + "logps/chosen": -0.583789050579071, + "logps/rejected": -1.670507788658142, + "loss": 0.6765, + "nll_loss": 0.673632800579071, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.05837402492761612, + "rewards/margins": 0.10859374701976776, + "rewards/rejected": -0.16691894829273224, + "step": 9990 + }, + { + "epoch": 0.7588404917286387, + "grad_norm": 3.811180333498817, + "learning_rate": 8e-07, + "log_odds_chosen": 1.330896019935608, + "log_odds_ratio": -0.46684569120407104, + "logits/chosen": -1.069726586341858, + "logits/rejected": -0.952929675579071, + "logps/chosen": -0.64404296875, + "logps/rejected": -1.5496094226837158, + "loss": 0.6774, + "nll_loss": 0.5791991949081421, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.06447754055261612, + "rewards/margins": 0.090631864964962, + "rewards/rejected": -0.15488281846046448, + "step": 10000 + }, + { + "epoch": 0.7588404917286387, + "eval_log_odds_chosen": 1.098879098892212, + "eval_log_odds_ratio": -0.499722421169281, + "eval_logits/chosen": -0.8501765131950378, + "eval_logits/rejected": -0.7714591026306152, + "eval_logps/chosen": -0.7597776055335999, + "eval_logps/rejected": -1.5665394067764282, + "eval_loss": 1.2043746709823608, + "eval_nll_loss": 1.1630213260650635, + "eval_rewards/accuracies": 0.7124233245849609, + "eval_rewards/chosen": -0.07597849518060684, + "eval_rewards/margins": 0.0806727185845375, + "eval_rewards/rejected": -0.1566520780324936, + "eval_runtime": 1310.1799, + "eval_samples_per_second": 71.657, + "eval_steps_per_second": 1.12, + "step": 10000 + } + ], + "logging_steps": 10, + "max_steps": 13178, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 5000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}