{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.00195, "eval_steps": 500, "global_step": 195, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7526.0, "completions/max_terminated_length": 7526.0, "completions/mean_length": 6484.28125, "completions/mean_terminated_length": 6484.28125, "completions/min_length": 878.0, "completions/min_terminated_length": 878.0, "entropy": 0.37100820429623127, "epoch": 1e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.697487711906433, "kl": 0.0, "learning_rate": 0.0, "loss": 0.0192, "num_tokens": 233760.0, "reward": -0.4238106608390808, "reward_std": 0.5410988330841064, "rewards/rollout_reward_func/mean": -0.4238106608390808, "rewards/rollout_reward_func/std": 0.5856553912162781, "sampling/importance_sampling_ratio/max": 1.2461258172988892, "sampling/importance_sampling_ratio/mean": 0.9992338418960571, "sampling/importance_sampling_ratio/min": 0.545093834400177, "sampling/sampling_logp_difference/max": 0.6067972183227539, "sampling/sampling_logp_difference/mean": 0.015956316143274307, "step": 1, "step_time": 76.42201462799494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.37100820429623127, "epoch": 2e-05, "grad_norm": 1.695469856262207, "kl": 0.0, "learning_rate": 2.8571428571428573e-06, "loss": 0.0192, "step": 2, "step_time": 29.774197126995205 }, { "clip_ratio/high_max": 0.005729166907258332, "clip_ratio/high_mean": 0.0033453527139499784, "clip_ratio/low_mean": 0.0015625000814907253, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.004907852795440704, "completions/clipped_ratio": 0.0, "completions/max_length": 7539.0, "completions/max_terminated_length": 7539.0, "completions/mean_length": 6288.5, "completions/mean_terminated_length": 6288.5, "completions/min_length": 1189.0, "completions/min_terminated_length": 1189.0, "entropy": 0.3813586551696062, "epoch": 3e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.6009650230407715, "kl": 0.0007843262719688937, "learning_rate": 5.7142857142857145e-06, "loss": 0.0783, "num_tokens": 461542.0, "reward": -0.48042526841163635, "reward_std": 0.36250609159469604, "rewards/rollout_reward_func/mean": -0.48042526841163635, "rewards/rollout_reward_func/std": 0.36595386266708374, "sampling/importance_sampling_ratio/max": 1.4710488319396973, "sampling/importance_sampling_ratio/mean": 1.0026617050170898, "sampling/importance_sampling_ratio/min": 0.565316915512085, "sampling/sampling_logp_difference/max": 0.570368766784668, "sampling/sampling_logp_difference/mean": 0.018272895365953445, "step": 3, "step_time": 74.60770390200014 }, { "clip_ratio/high_max": 0.004166666883975267, "clip_ratio/high_mean": 0.0020833334419876337, "clip_ratio/low_mean": 0.0051243280177004635, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007207661401480436, "entropy": 0.37923179753124714, "epoch": 4e-05, "grad_norm": 1.6179591417312622, "kl": 0.0007783326582284644, "learning_rate": 8.571428571428573e-06, "loss": 0.0772, "step": 4, "step_time": 28.76336038000045 }, { "clip_ratio/high_max": 0.013374302419833839, "clip_ratio/high_mean": 0.007245186949148774, "clip_ratio/low_mean": 0.0031670549942646176, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.0104122418561019, "completions/clipped_ratio": 0.0, "completions/max_length": 7293.0, "completions/max_terminated_length": 7293.0, "completions/mean_length": 6298.90625, "completions/mean_terminated_length": 6298.90625, "completions/min_length": 1698.0, "completions/min_terminated_length": 1698.0, "entropy": 0.33035928197205067, "epoch": 5e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.806251049041748, "kl": 0.0010234421843051678, "learning_rate": 1.1428571428571429e-05, "loss": 0.0617, "num_tokens": 689171.0, "reward": -0.24856753647327423, "reward_std": 0.9191180467605591, "rewards/rollout_reward_func/mean": -0.24856753647327423, "rewards/rollout_reward_func/std": 0.9297739267349243, "sampling/importance_sampling_ratio/max": 1.437739372253418, "sampling/importance_sampling_ratio/mean": 1.0004322528839111, "sampling/importance_sampling_ratio/min": 0.6451173424720764, "sampling/sampling_logp_difference/max": 0.4383230209350586, "sampling/sampling_logp_difference/mean": 0.017739713191986084, "step": 5, "step_time": 75.4136400849984 }, { "clip_ratio/high_max": 0.01141697692219168, "clip_ratio/high_mean": 0.00570848846109584, "clip_ratio/low_mean": 0.007705745549174026, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.013414233893854544, "entropy": 0.32829372212290764, "epoch": 6e-05, "grad_norm": 1.6114195585250854, "kl": 0.0015924154467938934, "learning_rate": 1.4285714285714285e-05, "loss": 0.0576, "step": 6, "step_time": 27.757563825000034 }, { "clip_ratio/high_max": 0.00890926382271573, "clip_ratio/high_mean": 0.004454631911357865, "clip_ratio/low_mean": 0.004478095797821879, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008932727709179744, "completions/clipped_ratio": 0.0, "completions/max_length": 7468.0, "completions/max_terminated_length": 7468.0, "completions/mean_length": 5852.9375, "completions/mean_terminated_length": 5852.9375, "completions/min_length": 650.0, "completions/min_terminated_length": 650.0, "entropy": 0.30918979551643133, "epoch": 7e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.578147530555725, "kl": 0.0027247646903560963, "learning_rate": 1.7142857142857145e-05, "loss": 0.1027, "num_tokens": 903217.0, "reward": -0.45870208740234375, "reward_std": 1.0192453861236572, "rewards/rollout_reward_func/mean": -0.45870208740234375, "rewards/rollout_reward_func/std": 1.0762962102890015, "sampling/importance_sampling_ratio/max": 1.462903380393982, "sampling/importance_sampling_ratio/mean": 1.000950574874878, "sampling/importance_sampling_ratio/min": 0.7107329368591309, "sampling/sampling_logp_difference/max": 0.38042306900024414, "sampling/sampling_logp_difference/mean": 0.016205646097660065, "step": 7, "step_time": 71.04002567400312 }, { "clip_ratio/high_max": 0.015160970797296613, "clip_ratio/high_mean": 0.007580485398648307, "clip_ratio/low_mean": 0.015520830056630075, "clip_ratio/low_min": 0.005208333604969084, "clip_ratio/region_mean": 0.023101315484382212, "entropy": 0.3041369505226612, "epoch": 8e-05, "grad_norm": 1.3657289743423462, "kl": 0.007668246718822047, "learning_rate": 2e-05, "loss": 0.1007, "step": 8, "step_time": 28.340653654999187 }, { "clip_ratio/high_max": 0.011096014932263643, "clip_ratio/high_mean": 0.005548007466131821, "clip_ratio/low_mean": 0.0019631411123555154, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007511148549383506, "completions/clipped_ratio": 0.0, "completions/max_length": 7654.0, "completions/max_terminated_length": 7654.0, "completions/mean_length": 6387.96875, "completions/mean_terminated_length": 6387.96875, "completions/min_length": 1250.0, "completions/min_terminated_length": 1250.0, "entropy": 0.3449282879009843, "epoch": 9e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.3927574157714844, "kl": 0.011275588360149413, "learning_rate": 2.2857142857142858e-05, "loss": 0.1499, "num_tokens": 1134056.0, "reward": -0.539466917514801, "reward_std": 0.5716531276702881, "rewards/rollout_reward_func/mean": -0.539466917514801, "rewards/rollout_reward_func/std": 0.6611050963401794, "sampling/importance_sampling_ratio/max": 1.4283660650253296, "sampling/importance_sampling_ratio/mean": 0.9991940259933472, "sampling/importance_sampling_ratio/min": 0.539411723613739, "sampling/sampling_logp_difference/max": 0.6172761917114258, "sampling/sampling_logp_difference/mean": 0.02549789473414421, "step": 9, "step_time": 73.84611950799626 }, { "clip_ratio/high_max": 0.023596015467774123, "clip_ratio/high_mean": 0.014792799222050235, "clip_ratio/low_mean": 0.02259067157865502, "clip_ratio/low_min": 0.00934139802120626, "clip_ratio/region_mean": 0.03738347071339376, "entropy": 0.33440436236560345, "epoch": 0.0001, "grad_norm": 1.500610113143921, "kl": 0.025877434643916786, "learning_rate": 2.5714285714285714e-05, "loss": 0.1476, "step": 10, "step_time": 29.093251289001273 }, { "clip_ratio/high_max": 0.007545235333964229, "clip_ratio/high_mean": 0.004814284387975931, "clip_ratio/low_mean": 0.0030828596209175885, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007897143950685859, "completions/clipped_ratio": 0.0, "completions/max_length": 7510.0, "completions/max_terminated_length": 7510.0, "completions/mean_length": 6214.28125, "completions/mean_terminated_length": 6214.28125, "completions/min_length": 1132.0, "completions/min_terminated_length": 1132.0, "entropy": 0.3201260110363364, "epoch": 0.00011, "frac_reward_zero_std": 0.0, "grad_norm": 1.2067501544952393, "kl": 0.030233556171879172, "learning_rate": 2.857142857142857e-05, "loss": 0.1797, "num_tokens": 1359267.0, "reward": -0.3541761636734009, "reward_std": 0.895065188407898, "rewards/rollout_reward_func/mean": -0.3541761636734009, "rewards/rollout_reward_func/std": 0.9152629375457764, "sampling/importance_sampling_ratio/max": 1.5590534210205078, "sampling/importance_sampling_ratio/mean": 0.9992601871490479, "sampling/importance_sampling_ratio/min": 0.2902994751930237, "sampling/sampling_logp_difference/max": 1.236842155456543, "sampling/sampling_logp_difference/mean": 0.03060976415872574, "step": 11, "step_time": 87.18317084899718 }, { "clip_ratio/high_max": 0.01579536369536072, "clip_ratio/high_mean": 0.008939348626881838, "clip_ratio/low_mean": 0.0062078598421067, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.015147208468988538, "entropy": 0.3198716137558222, "epoch": 0.00012, "grad_norm": 1.2537555694580078, "kl": 0.048300209222361445, "learning_rate": 3.142857142857143e-05, "loss": 0.1729, "step": 12, "step_time": 29.319526548995782 }, { "clip_ratio/high_max": 0.015590847469866276, "clip_ratio/high_mean": 0.008316257037222385, "clip_ratio/low_mean": 0.0036738234921358526, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.011990080529358238, "completions/clipped_ratio": 0.0, "completions/max_length": 7490.0, "completions/max_terminated_length": 7490.0, "completions/mean_length": 6175.03125, "completions/mean_terminated_length": 6175.03125, "completions/min_length": 1081.0, "completions/min_terminated_length": 1081.0, "entropy": 0.2996310368180275, "epoch": 0.00013, "frac_reward_zero_std": 0.0, "grad_norm": 1.6407757997512817, "kl": 0.04719050519634038, "learning_rate": 3.428571428571429e-05, "loss": 0.1486, "num_tokens": 1583637.0, "reward": -0.4320891499519348, "reward_std": 0.9571229219436646, "rewards/rollout_reward_func/mean": -0.4320891499519348, "rewards/rollout_reward_func/std": 1.0051363706588745, "sampling/importance_sampling_ratio/max": 1.9793360233306885, "sampling/importance_sampling_ratio/mean": 0.9981436729431152, "sampling/importance_sampling_ratio/min": 0.5052387118339539, "sampling/sampling_logp_difference/max": 0.6827614307403564, "sampling/sampling_logp_difference/mean": 0.032780710607767105, "step": 13, "step_time": 81.70137406499816 }, { "clip_ratio/high_max": 0.062379442853853106, "clip_ratio/high_mean": 0.03587722172960639, "clip_ratio/low_mean": 0.02087016322184354, "clip_ratio/low_min": 0.005208333604969084, "clip_ratio/region_mean": 0.056747385300695896, "entropy": 0.30233757197856903, "epoch": 0.00014, "grad_norm": 1.7402547597885132, "kl": 0.040611187811009586, "learning_rate": 3.7142857142857143e-05, "loss": 0.1437, "step": 14, "step_time": 28.446149357998365 }, { "clip_ratio/high_max": 0.008299731416627765, "clip_ratio/high_mean": 0.005017921328544617, "clip_ratio/low_mean": 0.007812500407453626, "clip_ratio/low_min": 0.0020833334419876337, "clip_ratio/region_mean": 0.012830421794205904, "completions/clipped_ratio": 0.0, "completions/max_length": 7745.0, "completions/max_terminated_length": 7745.0, "completions/mean_length": 6624.0, "completions/mean_terminated_length": 6624.0, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.31347857043147087, "epoch": 0.00015, "frac_reward_zero_std": 0.0, "grad_norm": 1.3761166334152222, "kl": 0.03515420196345076, "learning_rate": 4e-05, "loss": 0.1925, "num_tokens": 1822192.0, "reward": -0.2754722833633423, "reward_std": 0.7673642635345459, "rewards/rollout_reward_func/mean": -0.2754722833633423, "rewards/rollout_reward_func/std": 0.7773725390434265, "sampling/importance_sampling_ratio/max": 1.6611887216567993, "sampling/importance_sampling_ratio/mean": 1.001571536064148, "sampling/importance_sampling_ratio/min": 0.4510883688926697, "sampling/sampling_logp_difference/max": 0.7960920333862305, "sampling/sampling_logp_difference/mean": 0.025933649390935898, "step": 15, "step_time": 74.90382605300329 }, { "clip_ratio/high_max": 0.015244176145642996, "clip_ratio/high_mean": 0.01015185023425147, "clip_ratio/low_mean": 0.023821925511583686, "clip_ratio/low_min": 0.015277778205927461, "clip_ratio/region_mean": 0.03397377592045814, "entropy": 0.30452876072376966, "epoch": 0.00016, "grad_norm": 1.1651071310043335, "kl": 0.04755223210668191, "learning_rate": 4.2857142857142856e-05, "loss": 0.1842, "step": 16, "step_time": 29.99366738000208 }, { "clip_ratio/high_max": 0.004687500186264515, "clip_ratio/high_mean": 0.0023437500931322575, "clip_ratio/low_mean": 0.0015625000814907253, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003906250174622983, "completions/clipped_ratio": 0.0, "completions/max_length": 7597.0, "completions/max_terminated_length": 7597.0, "completions/mean_length": 6441.21875, "completions/mean_terminated_length": 6441.21875, "completions/min_length": 1725.0, "completions/min_terminated_length": 1725.0, "entropy": 0.2643199898302555, "epoch": 0.00017, "frac_reward_zero_std": 0.0, "grad_norm": 1.3995674848556519, "kl": 0.16503432305762544, "learning_rate": 4.5714285714285716e-05, "loss": 0.0465, "num_tokens": 2054984.0, "reward": -0.3876546621322632, "reward_std": 0.42256784439086914, "rewards/rollout_reward_func/mean": -0.3876546621322632, "rewards/rollout_reward_func/std": 0.5181643962860107, "sampling/importance_sampling_ratio/max": 1.6701462268829346, "sampling/importance_sampling_ratio/mean": 1.0001591444015503, "sampling/importance_sampling_ratio/min": 0.2944234311580658, "sampling/sampling_logp_difference/max": 1.2227363586425781, "sampling/sampling_logp_difference/mean": 0.02462080493569374, "step": 17, "step_time": 74.01380506300302 }, { "clip_ratio/high_max": 0.018824405735358596, "clip_ratio/high_mean": 0.010156250442378223, "clip_ratio/low_mean": 0.017436452209949493, "clip_ratio/low_min": 0.004036458441987634, "clip_ratio/region_mean": 0.027592702477704734, "entropy": 0.2591747185215354, "epoch": 0.00018, "grad_norm": 1.2044081687927246, "kl": 0.1656075087375939, "learning_rate": 4.8571428571428576e-05, "loss": 0.0403, "step": 18, "step_time": 29.02038000000357 }, { "clip_ratio/high_max": 0.009375000488944352, "clip_ratio/high_mean": 0.004687500244472176, "clip_ratio/low_mean": 0.004687500244472176, "clip_ratio/low_min": 0.0020833334419876337, "clip_ratio/region_mean": 0.009375000488944352, "completions/clipped_ratio": 0.0, "completions/max_length": 7783.0, "completions/max_terminated_length": 7783.0, "completions/mean_length": 6623.5625, "completions/mean_terminated_length": 6623.5625, "completions/min_length": 1029.0, "completions/min_terminated_length": 1029.0, "entropy": 0.2648680079728365, "epoch": 0.00019, "frac_reward_zero_std": 0.0, "grad_norm": 1.3061853647232056, "kl": 0.04802053823368624, "learning_rate": 5.142857142857143e-05, "loss": 0.0627, "num_tokens": 2293530.0, "reward": -0.40361839532852173, "reward_std": 0.4783870577812195, "rewards/rollout_reward_func/mean": -0.40361839532852173, "rewards/rollout_reward_func/std": 0.4853399097919464, "sampling/importance_sampling_ratio/max": 1.685901165008545, "sampling/importance_sampling_ratio/mean": 1.0008888244628906, "sampling/importance_sampling_ratio/min": 0.6258206367492676, "sampling/sampling_logp_difference/max": 0.5223002433776855, "sampling/sampling_logp_difference/mean": 0.02313043177127838, "step": 19, "step_time": 74.80562515100246 }, { "clip_ratio/high_max": 0.031250001047737896, "clip_ratio/high_mean": 0.016145833826158196, "clip_ratio/low_mean": 0.0237903383676894, "clip_ratio/low_min": 0.008593750419095159, "clip_ratio/region_mean": 0.03993617236847058, "entropy": 0.27113567758351564, "epoch": 0.0002, "grad_norm": 1.37777841091156, "kl": 0.06536271190270782, "learning_rate": 5.428571428571428e-05, "loss": 0.0604, "step": 20, "step_time": 29.113133827997444 }, { "clip_ratio/high_max": 0.00937500037252903, "clip_ratio/high_mean": 0.004687500186264515, "clip_ratio/low_mean": 0.004427083535119891, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009114583779592067, "completions/clipped_ratio": 0.0, "completions/max_length": 7661.0, "completions/max_terminated_length": 7661.0, "completions/mean_length": 5893.59375, "completions/mean_terminated_length": 5893.59375, "completions/min_length": 623.0, "completions/min_terminated_length": 623.0, "entropy": 0.31584511045366526, "epoch": 0.00021, "frac_reward_zero_std": 0.0, "grad_norm": 1.6288220882415771, "kl": 0.07922123826574534, "learning_rate": 5.714285714285714e-05, "loss": -0.001, "num_tokens": 2508670.0, "reward": -0.4046142101287842, "reward_std": 0.5082894563674927, "rewards/rollout_reward_func/mean": -0.4046142101287842, "rewards/rollout_reward_func/std": 0.5562217235565186, "sampling/importance_sampling_ratio/max": 2.4925785064697266, "sampling/importance_sampling_ratio/mean": 1.000154972076416, "sampling/importance_sampling_ratio/min": 5.001536337867485e-11, "sampling/sampling_logp_difference/max": 23.718690872192383, "sampling/sampling_logp_difference/mean": 0.04763348400592804, "step": 21, "step_time": 78.20269093000024 }, { "clip_ratio/high_max": 0.06250000186264515, "clip_ratio/high_mean": 0.03281250083819032, "clip_ratio/low_mean": 0.03454078524373472, "clip_ratio/low_min": 0.007986111391801387, "clip_ratio/region_mean": 0.06735328631475568, "entropy": 0.3279923405498266, "epoch": 0.00022, "grad_norm": 2.679633140563965, "kl": 0.07559517433401197, "learning_rate": 6e-05, "loss": -0.0015, "step": 22, "step_time": 29.090849199999866 }, { "clip_ratio/high_max": 0.009523810003884137, "clip_ratio/high_mean": 0.004761905001942068, "clip_ratio/low_mean": 0.004989035369362682, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.009750940429512411, "completions/clipped_ratio": 0.0, "completions/max_length": 7596.0, "completions/max_terminated_length": 7596.0, "completions/mean_length": 6510.53125, "completions/mean_terminated_length": 6510.53125, "completions/min_length": 1463.0, "completions/min_terminated_length": 1463.0, "entropy": 0.32954963482916355, "epoch": 0.00023, "frac_reward_zero_std": 0.0, "grad_norm": 1.3850574493408203, "kl": 0.08716068905778229, "learning_rate": 6.285714285714286e-05, "loss": 0.0286, "num_tokens": 2743357.0, "reward": -0.3586551547050476, "reward_std": 0.6654874086380005, "rewards/rollout_reward_func/mean": -0.3586551547050476, "rewards/rollout_reward_func/std": 0.7318810224533081, "sampling/importance_sampling_ratio/max": 1.5715701580047607, "sampling/importance_sampling_ratio/mean": 0.9974108934402466, "sampling/importance_sampling_ratio/min": 0.46494734287261963, "sampling/sampling_logp_difference/max": 0.7658311128616333, "sampling/sampling_logp_difference/mean": 0.02449648827314377, "step": 23, "step_time": 74.14140901399696 }, { "clip_ratio/high_max": 0.03601190622430295, "clip_ratio/high_mean": 0.02373511967016384, "clip_ratio/low_mean": 0.0511295928445179, "clip_ratio/low_min": 0.030681819072924554, "clip_ratio/region_mean": 0.07486471280572005, "entropy": 0.32481694035232067, "epoch": 0.00024, "grad_norm": 2.7298827171325684, "kl": 0.12141749600414187, "learning_rate": 6.571428571428571e-05, "loss": 0.0376, "step": 24, "step_time": 30.56049549600175 }, { "clip_ratio/high_max": 0.007093254243955016, "clip_ratio/high_mean": 0.0040506593068130314, "clip_ratio/low_mean": 0.0031081988709047437, "clip_ratio/low_min": 0.0010080644860863686, "clip_ratio/region_mean": 0.007158858177717775, "completions/clipped_ratio": 0.0, "completions/max_length": 7515.0, "completions/max_terminated_length": 7515.0, "completions/mean_length": 5735.25, "completions/mean_terminated_length": 5735.25, "completions/min_length": 629.0, "completions/min_terminated_length": 629.0, "entropy": 0.33798862248659134, "epoch": 0.00025, "frac_reward_zero_std": 0.0, "grad_norm": 1.2798893451690674, "kl": 0.08759039035066962, "learning_rate": 6.857142857142858e-05, "loss": 0.0511, "num_tokens": 2952764.0, "reward": -0.1837199330329895, "reward_std": 0.7476290464401245, "rewards/rollout_reward_func/mean": -0.1837199330329895, "rewards/rollout_reward_func/std": 0.742720901966095, "sampling/importance_sampling_ratio/max": 1.7167617082595825, "sampling/importance_sampling_ratio/mean": 1.0019567012786865, "sampling/importance_sampling_ratio/min": 0.3606078624725342, "sampling/sampling_logp_difference/max": 1.0199642181396484, "sampling/sampling_logp_difference/mean": 0.026409240439534187, "step": 25, "step_time": 71.47596238900223 }, { "clip_ratio/high_max": 0.03890659159515053, "clip_ratio/high_mean": 0.022751906712073833, "clip_ratio/low_mean": 0.07101231021806598, "clip_ratio/low_min": 0.03450100845657289, "clip_ratio/region_mean": 0.09376421687193215, "entropy": 0.34598700143396854, "epoch": 0.00026, "grad_norm": 1.7697508335113525, "kl": 0.08915858040563762, "learning_rate": 7.142857142857143e-05, "loss": 0.0498, "step": 26, "step_time": 28.04239848200632 }, { "clip_ratio/high_max": 0.005208333604969084, "clip_ratio/high_mean": 0.002604166802484542, "clip_ratio/low_mean": 0.00410473023657687, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.006708897039061412, "completions/clipped_ratio": 0.0, "completions/max_length": 7490.0, "completions/max_terminated_length": 7490.0, "completions/mean_length": 6756.65625, "completions/mean_terminated_length": 6756.65625, "completions/min_length": 1559.0, "completions/min_terminated_length": 1559.0, "entropy": 0.3758588843047619, "epoch": 0.00027, "frac_reward_zero_std": 0.0, "grad_norm": 1.3073501586914062, "kl": 0.08391126664355397, "learning_rate": 7.428571428571429e-05, "loss": -0.0213, "num_tokens": 3194912.0, "reward": -0.19097471237182617, "reward_std": 0.5197073817253113, "rewards/rollout_reward_func/mean": -0.19097471237182617, "rewards/rollout_reward_func/std": 0.5816971063613892, "sampling/importance_sampling_ratio/max": 1.400797724723816, "sampling/importance_sampling_ratio/mean": 0.999113917350769, "sampling/importance_sampling_ratio/min": 0.4852893054485321, "sampling/sampling_logp_difference/max": 0.7230100631713867, "sampling/sampling_logp_difference/mean": 0.023783450946211815, "step": 27, "step_time": 77.1986499249997 }, { "clip_ratio/high_max": 0.06250000186264515, "clip_ratio/high_mean": 0.041235633194446564, "clip_ratio/low_mean": 0.07196544599719346, "clip_ratio/low_min": 0.03801243333145976, "clip_ratio/region_mean": 0.11320107895880938, "entropy": 0.36850391887128353, "epoch": 0.00028, "grad_norm": 2.796506404876709, "kl": 0.11568774632178247, "learning_rate": 7.714285714285715e-05, "loss": -0.0196, "step": 28, "step_time": 28.698328271997525 }, { "clip_ratio/high_max": 0.0068772402592003345, "clip_ratio/high_mean": 0.0034386201296001673, "clip_ratio/low_mean": 0.008837365778163075, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012275985849555582, "completions/clipped_ratio": 0.03125, "completions/max_length": 7663.0, "completions/max_terminated_length": 7663.0, "completions/mean_length": 6515.75, "completions/mean_terminated_length": 6653.48388671875, "completions/min_length": 999.0, "completions/min_terminated_length": 999.0, "entropy": 0.36264942586421967, "epoch": 0.00029, "frac_reward_zero_std": 0.0, "grad_norm": 1.186471700668335, "kl": 0.10322241578251123, "learning_rate": 8e-05, "loss": 0.1284, "num_tokens": 3429334.0, "reward": -0.3291829526424408, "reward_std": 0.9100008606910706, "rewards/rollout_reward_func/mean": -0.3291829526424408, "rewards/rollout_reward_func/std": 0.9173133969306946, "sampling/importance_sampling_ratio/max": 1.8069854974746704, "sampling/importance_sampling_ratio/mean": 1.0005632638931274, "sampling/importance_sampling_ratio/min": 0.6562378406524658, "sampling/sampling_logp_difference/max": 0.5916600227355957, "sampling/sampling_logp_difference/mean": 0.023244231939315796, "step": 29, "step_time": 75.12748930800444 }, { "clip_ratio/high_max": 0.05988007667474449, "clip_ratio/high_mean": 0.03098170505836606, "clip_ratio/low_mean": 0.0681427443632856, "clip_ratio/low_min": 0.028056694194674492, "clip_ratio/region_mean": 0.0991244496544823, "entropy": 0.34883980453014374, "epoch": 0.0003, "grad_norm": 1.7960748672485352, "kl": 0.1314481229055673, "learning_rate": 8.285714285714287e-05, "loss": 0.1269, "step": 30, "step_time": 29.35103237200019 }, { "clip_ratio/high_max": 0.008299731533043087, "clip_ratio/high_mean": 0.005017921386752278, "clip_ratio/low_mean": 0.005158980435226113, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.01017690182197839, "completions/clipped_ratio": 0.0, "completions/max_length": 7588.0, "completions/max_terminated_length": 7588.0, "completions/mean_length": 6533.40625, "completions/mean_terminated_length": 6533.40625, "completions/min_length": 1190.0, "completions/min_terminated_length": 1190.0, "entropy": 0.370091924443841, "epoch": 0.00031, "frac_reward_zero_std": 0.0, "grad_norm": 1.2005739212036133, "kl": 0.0893048639409244, "learning_rate": 8.571428571428571e-05, "loss": 0.0715, "num_tokens": 3664488.0, "reward": -0.2769165635108948, "reward_std": 0.6585308313369751, "rewards/rollout_reward_func/mean": -0.2769165635108948, "rewards/rollout_reward_func/std": 0.6615685820579529, "sampling/importance_sampling_ratio/max": 1.624229907989502, "sampling/importance_sampling_ratio/mean": 0.9981648325920105, "sampling/importance_sampling_ratio/min": 0.6380103230476379, "sampling/sampling_logp_difference/max": 0.4850337505340576, "sampling/sampling_logp_difference/mean": 0.02116064913570881, "step": 31, "step_time": 74.05255860899888 }, { "clip_ratio/high_max": 0.09285618434660137, "clip_ratio/high_mean": 0.05181003676261753, "clip_ratio/low_mean": 0.06460288155358285, "clip_ratio/low_min": 0.021354167256504297, "clip_ratio/region_mean": 0.11641291866544634, "entropy": 0.35460062325000763, "epoch": 0.00032, "grad_norm": 2.2096006870269775, "kl": 0.12510896613821387, "learning_rate": 8.857142857142857e-05, "loss": 0.0803, "step": 32, "step_time": 29.878413379994527 }, { "clip_ratio/high_max": 0.007559092831797898, "clip_ratio/high_mean": 0.004821213253308088, "clip_ratio/low_mean": 0.004480286879697815, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009301500133005902, "completions/clipped_ratio": 0.0, "completions/max_length": 7428.0, "completions/max_terminated_length": 7428.0, "completions/mean_length": 6528.5, "completions/mean_terminated_length": 6528.5, "completions/min_length": 2087.0, "completions/min_terminated_length": 2087.0, "entropy": 0.3161447402089834, "epoch": 0.00033, "frac_reward_zero_std": 0.0, "grad_norm": 1.0615328550338745, "kl": 0.1485683936625719, "learning_rate": 9.142857142857143e-05, "loss": 0.0921, "num_tokens": 3899142.0, "reward": -0.14407192170619965, "reward_std": 0.9980674386024475, "rewards/rollout_reward_func/mean": -0.14407192170619965, "rewards/rollout_reward_func/std": 1.0452619791030884, "sampling/importance_sampling_ratio/max": 1.4222201108932495, "sampling/importance_sampling_ratio/mean": 1.0011844635009766, "sampling/importance_sampling_ratio/min": 0.7389679551124573, "sampling/sampling_logp_difference/max": 0.3522191047668457, "sampling/sampling_logp_difference/mean": 0.017726991325616837, "step": 33, "step_time": 73.88488937800503 }, { "clip_ratio/high_max": 0.06421604077331722, "clip_ratio/high_mean": 0.03419135382864624, "clip_ratio/low_mean": 0.03114119404926896, "clip_ratio/low_min": 0.010416666977107525, "clip_ratio/region_mean": 0.06533254752866924, "entropy": 0.3158456925302744, "epoch": 0.00034, "grad_norm": 1.1703853607177734, "kl": 0.14151222538203, "learning_rate": 9.428571428571429e-05, "loss": 0.086, "step": 34, "step_time": 28.342066529001386 }, { "clip_ratio/high_max": 0.004166666883975267, "clip_ratio/high_mean": 0.0020833334419876337, "clip_ratio/low_mean": 0.005729166907258332, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.007812500407453626, "completions/clipped_ratio": 0.0, "completions/max_length": 7681.0, "completions/max_terminated_length": 7681.0, "completions/mean_length": 6327.84375, "completions/mean_terminated_length": 6327.84375, "completions/min_length": 1077.0, "completions/min_terminated_length": 1077.0, "entropy": 0.3124512918293476, "epoch": 0.00035, "frac_reward_zero_std": 0.0, "grad_norm": 1.0792138576507568, "kl": 0.14913526410236955, "learning_rate": 9.714285714285715e-05, "loss": 0.0041, "num_tokens": 4127763.0, "reward": -0.05273808166384697, "reward_std": 0.7115249633789062, "rewards/rollout_reward_func/mean": -0.05273808166384697, "rewards/rollout_reward_func/std": 0.7688351273536682, "sampling/importance_sampling_ratio/max": 1.573601245880127, "sampling/importance_sampling_ratio/mean": 1.000023365020752, "sampling/importance_sampling_ratio/min": 0.5948914885520935, "sampling/sampling_logp_difference/max": 0.519376277923584, "sampling/sampling_logp_difference/mean": 0.018448850139975548, "step": 35, "step_time": 74.24198849099594 }, { "clip_ratio/high_max": 0.05934027908369899, "clip_ratio/high_mean": 0.03571633482351899, "clip_ratio/low_mean": 0.06956845405511558, "clip_ratio/low_min": 0.04062500107102096, "clip_ratio/region_mean": 0.10528478911146522, "entropy": 0.28535627014935017, "epoch": 0.00036, "grad_norm": 1.6060221195220947, "kl": 0.2546167550608516, "learning_rate": 0.0001, "loss": 0.002, "step": 36, "step_time": 29.2567209630015 }, { "clip_ratio/high_max": 0.007007576059550047, "clip_ratio/high_mean": 0.005066288111265749, "clip_ratio/low_mean": 0.0030913979280740023, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008157686039339751, "completions/clipped_ratio": 0.0, "completions/max_length": 7603.0, "completions/max_terminated_length": 7603.0, "completions/mean_length": 6742.375, "completions/mean_terminated_length": 6742.375, "completions/min_length": 1976.0, "completions/min_terminated_length": 1976.0, "entropy": 0.2829171046614647, "epoch": 0.00037, "frac_reward_zero_std": 0.0, "grad_norm": 1.0890644788742065, "kl": 0.19634687714278698, "learning_rate": 9.999736485702831e-05, "loss": -0.0638, "num_tokens": 4369544.0, "reward": -0.12373267114162445, "reward_std": 0.41749435663223267, "rewards/rollout_reward_func/mean": -0.12373267114162445, "rewards/rollout_reward_func/std": 0.47090527415275574, "sampling/importance_sampling_ratio/max": 1.512851357460022, "sampling/importance_sampling_ratio/mean": 0.9993549585342407, "sampling/importance_sampling_ratio/min": 0.6950358152389526, "sampling/sampling_logp_difference/max": 0.41399621963500977, "sampling/sampling_logp_difference/mean": 0.015387965366244316, "step": 37, "step_time": 73.93194824100101 }, { "clip_ratio/high_max": 0.07025463134050369, "clip_ratio/high_mean": 0.04312921012751758, "clip_ratio/low_mean": 0.08639893110375851, "clip_ratio/low_min": 0.05557795753702521, "clip_ratio/region_mean": 0.12952814064919949, "entropy": 0.28258804604411125, "epoch": 0.00038, "grad_norm": 1.9825063943862915, "kl": 0.2708545783534646, "learning_rate": 9.998945979845876e-05, "loss": -0.0435, "step": 38, "step_time": 30.59339555200313 }, { "clip_ratio/high_max": 0.00716575124533847, "clip_ratio/high_mean": 0.003582875622669235, "clip_ratio/low_mean": 0.004315476398915052, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.007898352021584287, "completions/clipped_ratio": 0.0, "completions/max_length": 7509.0, "completions/max_terminated_length": 7509.0, "completions/mean_length": 5088.96875, "completions/mean_terminated_length": 5088.96875, "completions/min_length": 951.0, "completions/min_terminated_length": 951.0, "entropy": 0.283947229385376, "epoch": 0.00039, "frac_reward_zero_std": 0.0, "grad_norm": 0.9808809161186218, "kl": 0.4519640696235001, "learning_rate": 9.997628593527586e-05, "loss": 0.0853, "num_tokens": 4558708.0, "reward": 0.10180412977933884, "reward_std": 0.8016654253005981, "rewards/rollout_reward_func/mean": 0.10180412977933884, "rewards/rollout_reward_func/std": 0.8614758849143982, "sampling/importance_sampling_ratio/max": 1.4223048686981201, "sampling/importance_sampling_ratio/mean": 0.9974642395973206, "sampling/importance_sampling_ratio/min": 0.5820598602294922, "sampling/sampling_logp_difference/max": 0.5411820411682129, "sampling/sampling_logp_difference/mean": 0.01900225132703781, "step": 39, "step_time": 71.3349767670079 }, { "clip_ratio/high_max": 0.030006105778738856, "clip_ratio/high_mean": 0.017387921339832246, "clip_ratio/low_mean": 0.03726992168230936, "clip_ratio/low_min": 0.020089286495931447, "clip_ratio/region_mean": 0.05465784267289564, "entropy": 0.2824964262545109, "epoch": 0.0004, "grad_norm": 1.0136867761611938, "kl": 0.40690419310703874, "learning_rate": 9.995784511894694e-05, "loss": 0.076, "step": 40, "step_time": 28.261510162006743 }, { "clip_ratio/high_max": 0.0096726194024086, "clip_ratio/high_mean": 0.0058128722012043, "clip_ratio/low_mean": 0.0020665322663262486, "clip_ratio/low_min": 0.0010080644860863686, "clip_ratio/region_mean": 0.007879404467530549, "completions/clipped_ratio": 0.0, "completions/max_length": 7599.0, "completions/max_terminated_length": 7599.0, "completions/mean_length": 4873.46875, "completions/mean_terminated_length": 4873.46875, "completions/min_length": 694.0, "completions/min_terminated_length": 694.0, "entropy": 0.2860498381778598, "epoch": 0.00041, "frac_reward_zero_std": 0.0, "grad_norm": 0.894948422908783, "kl": 0.24301961436867714, "learning_rate": 9.993413994116206e-05, "loss": -0.0672, "num_tokens": 4740691.0, "reward": 0.39664557576179504, "reward_std": 0.7970774173736572, "rewards/rollout_reward_func/mean": 0.39664557576179504, "rewards/rollout_reward_func/std": 0.8819013237953186, "sampling/importance_sampling_ratio/max": 1.5840785503387451, "sampling/importance_sampling_ratio/mean": 1.0020239353179932, "sampling/importance_sampling_ratio/min": 0.5295067429542542, "sampling/sampling_logp_difference/max": 0.6358094215393066, "sampling/sampling_logp_difference/mean": 0.014837441965937614, "step": 41, "step_time": 65.9710494339979 }, { "clip_ratio/high_max": 0.0284070186316967, "clip_ratio/high_mean": 0.017205960233695805, "clip_ratio/low_mean": 0.031194624374620616, "clip_ratio/low_min": 0.011708158301189542, "clip_ratio/region_mean": 0.04840058443369344, "entropy": 0.3145472202450037, "epoch": 0.00042, "grad_norm": 0.7113205194473267, "kl": 0.24477723240852356, "learning_rate": 9.990517373346957e-05, "loss": -0.0744, "step": 42, "step_time": 27.98081090999949 }, { "clip_ratio/high_max": 0.004838809370994568, "clip_ratio/high_mean": 0.002419404685497284, "clip_ratio/low_mean": 0.0020926339784637094, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004512038663960993, "completions/clipped_ratio": 0.0, "completions/max_length": 7728.0, "completions/max_terminated_length": 7728.0, "completions/mean_length": 5987.8125, "completions/mean_terminated_length": 5987.8125, "completions/min_length": 494.0, "completions/min_terminated_length": 494.0, "entropy": 0.3688400685787201, "epoch": 0.00043, "frac_reward_zero_std": 0.0, "grad_norm": 0.881762683391571, "kl": 0.3142514768987894, "learning_rate": 9.98709505668081e-05, "loss": -0.0912, "num_tokens": 4958363.0, "reward": 0.09718979895114899, "reward_std": 0.7198001742362976, "rewards/rollout_reward_func/mean": 0.09718979895114899, "rewards/rollout_reward_func/std": 0.6971056461334229, "sampling/importance_sampling_ratio/max": 1.4162718057632446, "sampling/importance_sampling_ratio/mean": 0.9995008707046509, "sampling/importance_sampling_ratio/min": 0.643088161945343, "sampling/sampling_logp_difference/max": 0.44147348403930664, "sampling/sampling_logp_difference/mean": 0.017590995877981186, "step": 43, "step_time": 73.6047150669965 }, { "clip_ratio/high_max": 0.030674015870317817, "clip_ratio/high_mean": 0.0172546214889735, "clip_ratio/low_mean": 0.0454135142499581, "clip_ratio/low_min": 0.016698232968337834, "clip_ratio/region_mean": 0.06266813568072394, "entropy": 0.36243374459445477, "epoch": 0.00044, "grad_norm": 0.8560315370559692, "kl": 0.3110799826681614, "learning_rate": 9.983147525093428e-05, "loss": -0.1008, "step": 44, "step_time": 28.81937372000357 }, { "clip_ratio/high_max": 0.011496570543386042, "clip_ratio/high_mean": 0.005748285271693021, "clip_ratio/low_mean": 0.002752976317424327, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008501261589117348, "completions/clipped_ratio": 0.0, "completions/max_length": 7621.0, "completions/max_terminated_length": 7621.0, "completions/mean_length": 5695.53125, "completions/mean_terminated_length": 5695.53125, "completions/min_length": 774.0, "completions/min_terminated_length": 774.0, "entropy": 0.33075752947479486, "epoch": 0.00045, "frac_reward_zero_std": 0.0, "grad_norm": 1.3113490343093872, "kl": 0.224155243486166, "learning_rate": 9.978675333374685e-05, "loss": 0.0467, "num_tokens": 5166839.0, "reward": 0.1164797991514206, "reward_std": 0.7211669683456421, "rewards/rollout_reward_func/mean": 0.1164797991514206, "rewards/rollout_reward_func/std": 0.7984298467636108, "sampling/importance_sampling_ratio/max": 1.377953052520752, "sampling/importance_sampling_ratio/mean": 0.9993555545806885, "sampling/importance_sampling_ratio/min": 0.6047067046165466, "sampling/sampling_logp_difference/max": 0.5030117034912109, "sampling/sampling_logp_difference/mean": 0.019132370129227638, "step": 45, "step_time": 70.18080313399696 }, { "clip_ratio/high_max": 0.04167593677993864, "clip_ratio/high_mean": 0.02203989162808284, "clip_ratio/low_mean": 0.037681632733438164, "clip_ratio/low_min": 0.022104313364252448, "clip_ratio/region_mean": 0.059721524245105684, "entropy": 0.31185934972018003, "epoch": 0.00046, "grad_norm": 0.975906491279602, "kl": 0.24094799999147654, "learning_rate": 9.973679110050689e-05, "loss": 0.0389, "step": 46, "step_time": 29.416862356994898 }, { "clip_ratio/high_max": 0.004427083535119891, "clip_ratio/high_mean": 0.0027175740688107908, "clip_ratio/low_mean": 0.004687500186264515, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007405074313282967, "completions/clipped_ratio": 0.0, "completions/max_length": 7655.0, "completions/max_terminated_length": 7655.0, "completions/mean_length": 5746.0, "completions/mean_terminated_length": 5746.0, "completions/min_length": 1173.0, "completions/min_terminated_length": 1173.0, "entropy": 0.3087401306256652, "epoch": 0.00047, "frac_reward_zero_std": 0.0, "grad_norm": 1.1199007034301758, "kl": 0.25847794907167554, "learning_rate": 9.968159557295458e-05, "loss": 0.0078, "num_tokens": 5377018.0, "reward": 0.14880667626857758, "reward_std": 0.627174973487854, "rewards/rollout_reward_func/mean": 0.14880667626857758, "rewards/rollout_reward_func/std": 0.6551938056945801, "sampling/importance_sampling_ratio/max": 1.3322476148605347, "sampling/importance_sampling_ratio/mean": 0.9995932579040527, "sampling/importance_sampling_ratio/min": 0.6932395100593567, "sampling/sampling_logp_difference/max": 0.3663797378540039, "sampling/sampling_logp_difference/mean": 0.01716955006122589, "step": 47, "step_time": 71.10858440999982 }, { "clip_ratio/high_max": 0.07631659507751465, "clip_ratio/high_mean": 0.04591079382225871, "clip_ratio/low_mean": 0.07935288141015917, "clip_ratio/low_min": 0.053645835258066654, "clip_ratio/region_mean": 0.12526367511600256, "entropy": 0.28964369650930166, "epoch": 0.00048, "grad_norm": 1.2288544178009033, "kl": 0.3888731300830841, "learning_rate": 9.962117450832225e-05, "loss": 0.0171, "step": 48, "step_time": 29.153815419005696 }, { "clip_ratio/high_max": 0.01289592799730599, "clip_ratio/high_mean": 0.006447963998652995, "clip_ratio/low_mean": 0.0031081989873200655, "clip_ratio/low_min": 0.0020497312070801854, "clip_ratio/region_mean": 0.009556163044180721, "completions/clipped_ratio": 0.0, "completions/max_length": 7596.0, "completions/max_terminated_length": 7596.0, "completions/mean_length": 6635.625, "completions/mean_terminated_length": 6635.625, "completions/min_length": 1043.0, "completions/min_terminated_length": 1043.0, "entropy": 0.29299468267709017, "epoch": 0.00049, "frac_reward_zero_std": 0.0, "grad_norm": 0.7834474444389343, "kl": 0.2757651209831238, "learning_rate": 9.955553639824423e-05, "loss": 0.2173, "num_tokens": 5615364.0, "reward": 0.32925018668174744, "reward_std": 0.8307831883430481, "rewards/rollout_reward_func/mean": 0.32925018668174744, "rewards/rollout_reward_func/std": 0.8916731476783752, "sampling/importance_sampling_ratio/max": 1.3534932136535645, "sampling/importance_sampling_ratio/mean": 0.9990413188934326, "sampling/importance_sampling_ratio/min": 0.49456140398979187, "sampling/sampling_logp_difference/max": 0.7040839195251465, "sampling/sampling_logp_difference/mean": 0.01665666699409485, "step": 49, "step_time": 74.14242603500315 }, { "clip_ratio/high_max": 0.028901536716148257, "clip_ratio/high_mean": 0.01505172997713089, "clip_ratio/low_mean": 0.026322524121496826, "clip_ratio/low_min": 0.010375310201197863, "clip_ratio/region_mean": 0.04137425415683538, "entropy": 0.3015470812097192, "epoch": 0.0005, "grad_norm": 0.5381351709365845, "kl": 0.2881705882027745, "learning_rate": 9.948469046756344e-05, "loss": 0.207, "step": 50, "step_time": 29.138568417003626 }, { "clip_ratio/high_max": 0.00781395920785144, "clip_ratio/high_mean": 0.00390697960392572, "clip_ratio/low_mean": 0.0020833334419876337, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005990313162328675, "completions/clipped_ratio": 0.0, "completions/max_length": 7635.0, "completions/max_terminated_length": 7635.0, "completions/mean_length": 6060.40625, "completions/mean_terminated_length": 6060.40625, "completions/min_length": 460.0, "completions/min_terminated_length": 460.0, "entropy": 0.3280220106244087, "epoch": 0.00051, "frac_reward_zero_std": 0.0, "grad_norm": 1.1158900260925293, "kl": 0.36202954687178135, "learning_rate": 9.940864667303489e-05, "loss": 0.0681, "num_tokens": 5835156.0, "reward": 0.13787294924259186, "reward_std": 0.5426970720291138, "rewards/rollout_reward_func/mean": 0.13787294924259186, "rewards/rollout_reward_func/std": 0.5641806125640869, "sampling/importance_sampling_ratio/max": 1.8794081211090088, "sampling/importance_sampling_ratio/mean": 0.9995182752609253, "sampling/importance_sampling_ratio/min": 0.25508245825767517, "sampling/sampling_logp_difference/max": 1.36616849899292, "sampling/sampling_logp_difference/mean": 0.018341850489377975, "step": 51, "step_time": 75.39097913900332 }, { "clip_ratio/high_max": 0.0497949382988736, "clip_ratio/high_mean": 0.028492899087723345, "clip_ratio/low_mean": 0.031691792828496546, "clip_ratio/low_min": 0.01328125037252903, "clip_ratio/region_mean": 0.06018469203263521, "entropy": 0.3533763512969017, "epoch": 0.00052, "grad_norm": 0.6478288769721985, "kl": 0.28462592978030443, "learning_rate": 9.932741570192633e-05, "loss": 0.0576, "step": 52, "step_time": 28.780319199000587 }, { "clip_ratio/high_max": 0.002016128972172737, "clip_ratio/high_mean": 0.0010080644860863686, "clip_ratio/low_mean": 0.006937850819667801, "clip_ratio/low_min": 0.0019965278333984315, "clip_ratio/region_mean": 0.00794591530575417, "completions/clipped_ratio": 0.0, "completions/max_length": 7602.0, "completions/max_terminated_length": 7602.0, "completions/mean_length": 6027.96875, "completions/mean_terminated_length": 6027.96875, "completions/min_length": 1007.0, "completions/min_terminated_length": 1007.0, "entropy": 0.3581601083278656, "epoch": 0.00053, "frac_reward_zero_std": 0.0, "grad_norm": 0.7359389066696167, "kl": 0.2905560820363462, "learning_rate": 9.924100897051629e-05, "loss": 0.2862, "num_tokens": 6054150.0, "reward": 0.21641331911087036, "reward_std": 0.7501168251037598, "rewards/rollout_reward_func/mean": 0.21641331911087036, "rewards/rollout_reward_func/std": 0.7459017634391785, "sampling/importance_sampling_ratio/max": 1.6359493732452393, "sampling/importance_sampling_ratio/mean": 1.001375675201416, "sampling/importance_sampling_ratio/min": 0.7042098045349121, "sampling/sampling_logp_difference/max": 0.49222326278686523, "sampling/sampling_logp_difference/mean": 0.018105559051036835, "step": 53, "step_time": 73.25262797300275 }, { "clip_ratio/high_max": 0.031247199745848775, "clip_ratio/high_mean": 0.018971814191900194, "clip_ratio/low_mean": 0.025980107544455677, "clip_ratio/low_min": 0.010829763719812036, "clip_ratio/region_mean": 0.04495192161994055, "entropy": 0.40328157134354115, "epoch": 0.00054, "grad_norm": 0.5571223497390747, "kl": 0.24240488233044744, "learning_rate": 9.914943862248966e-05, "loss": 0.2818, "step": 54, "step_time": 29.45206928499465 }, { "clip_ratio/high_max": 0.003504224237985909, "clip_ratio/high_mean": 0.0017521121189929545, "clip_ratio/low_mean": 0.007221993524581194, "clip_ratio/low_min": 0.0010080644860863686, "clip_ratio/region_mean": 0.008974105643574148, "completions/clipped_ratio": 0.0, "completions/max_length": 7642.0, "completions/max_terminated_length": 7642.0, "completions/mean_length": 6042.875, "completions/mean_terminated_length": 6042.875, "completions/min_length": 2016.0, "completions/min_terminated_length": 2016.0, "entropy": 0.4168450031429529, "epoch": 0.00055, "frac_reward_zero_std": 0.0, "grad_norm": 0.8299343585968018, "kl": 0.23489390965551138, "learning_rate": 9.905271752723088e-05, "loss": 0.1819, "num_tokens": 6273022.0, "reward": 0.23816508054733276, "reward_std": 0.8371602296829224, "rewards/rollout_reward_func/mean": 0.23816508054733276, "rewards/rollout_reward_func/std": 0.8158068656921387, "sampling/importance_sampling_ratio/max": 1.4655125141143799, "sampling/importance_sampling_ratio/mean": 1.0008599758148193, "sampling/importance_sampling_ratio/min": 0.05531880632042885, "sampling/sampling_logp_difference/max": 2.8946423530578613, "sampling/sampling_logp_difference/mean": 0.019126659259200096, "step": 55, "step_time": 71.40723781300039 }, { "clip_ratio/high_max": 0.046803416684269905, "clip_ratio/high_mean": 0.02541783731430769, "clip_ratio/low_mean": 0.08520147681701928, "clip_ratio/low_min": 0.049988799495622516, "clip_ratio/region_mean": 0.11061931506264955, "entropy": 0.42906123399734497, "epoch": 0.00056, "grad_norm": 0.9861308932304382, "kl": 0.27001852355897427, "learning_rate": 9.895085927801542e-05, "loss": 0.1762, "step": 56, "step_time": 28.45427264400132 }, { "clip_ratio/high_max": 0.0037634409382008016, "clip_ratio/high_mean": 0.0024025538295973092, "clip_ratio/low_mean": 0.0043235087068751454, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006726062536472455, "completions/clipped_ratio": 0.0, "completions/max_length": 7818.0, "completions/max_terminated_length": 7818.0, "completions/mean_length": 5434.5, "completions/mean_terminated_length": 5434.5, "completions/min_length": 678.0, "completions/min_terminated_length": 678.0, "entropy": 0.4097133297473192, "epoch": 0.00057, "frac_reward_zero_std": 0.0, "grad_norm": 0.7375171184539795, "kl": 0.27723561972379684, "learning_rate": 9.884387819009922e-05, "loss": -0.1695, "num_tokens": 6472524.0, "reward": 0.19157832860946655, "reward_std": 0.573657751083374, "rewards/rollout_reward_func/mean": 0.19157832860946655, "rewards/rollout_reward_func/std": 0.6112639904022217, "sampling/importance_sampling_ratio/max": 1.5466991662979126, "sampling/importance_sampling_ratio/mean": 1.0001226663589478, "sampling/importance_sampling_ratio/min": 0.2603280246257782, "sampling/sampling_logp_difference/max": 1.3458127975463867, "sampling/sampling_logp_difference/mean": 0.017698664218187332, "step": 57, "step_time": 70.43534664199979 }, { "clip_ratio/high_max": 0.03705788997467607, "clip_ratio/high_mean": 0.026643864752259105, "clip_ratio/low_mean": 0.04439880105201155, "clip_ratio/low_min": 0.020043682772666216, "clip_ratio/region_mean": 0.071042665746063, "entropy": 0.3895236197859049, "epoch": 0.00058, "grad_norm": 0.8878812193870544, "kl": 0.2884919047355652, "learning_rate": 9.873178929870695e-05, "loss": -0.1734, "step": 58, "step_time": 28.610490492996178 }, { "clip_ratio/high_max": 0.007988505763933063, "clip_ratio/high_mean": 0.003994252881966531, "clip_ratio/low_mean": 0.0030996607383713126, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007093913620337844, "completions/clipped_ratio": 0.0, "completions/max_length": 7822.0, "completions/max_terminated_length": 7822.0, "completions/mean_length": 5622.5, "completions/mean_terminated_length": 5622.5, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.4184044487774372, "epoch": 0.00059, "frac_reward_zero_std": 0.0, "grad_norm": 0.9892687201499939, "kl": 0.3114945339038968, "learning_rate": 9.86146083569188e-05, "loss": -0.0006, "num_tokens": 6678073.0, "reward": 0.46102336049079895, "reward_std": 0.8252051472663879, "rewards/rollout_reward_func/mean": 0.46102336049079895, "rewards/rollout_reward_func/std": 0.8315688967704773, "sampling/importance_sampling_ratio/max": 2.0225541591644287, "sampling/importance_sampling_ratio/mean": 1.0026776790618896, "sampling/importance_sampling_ratio/min": 0.5649540424346924, "sampling/sampling_logp_difference/max": 0.7043611407279968, "sampling/sampling_logp_difference/mean": 0.017727889120578766, "step": 59, "step_time": 71.46863787700022 }, { "clip_ratio/high_max": 0.06055360846221447, "clip_ratio/high_mean": 0.03340180404484272, "clip_ratio/low_mean": 0.08376732922624797, "clip_ratio/low_min": 0.03504428896121681, "clip_ratio/region_mean": 0.11716913338750601, "entropy": 0.41077898256480694, "epoch": 0.0006, "grad_norm": 1.108790397644043, "kl": 0.29847336653620005, "learning_rate": 9.84923518334567e-05, "loss": -0.002, "step": 60, "step_time": 28.821211089998542 }, { "clip_ratio/high_max": 0.004427083418704569, "clip_ratio/high_mean": 0.002734375069849193, "clip_ratio/low_mean": 0.0020259797456674278, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004760354815516621, "completions/clipped_ratio": 0.0, "completions/max_length": 7716.0, "completions/max_terminated_length": 7716.0, "completions/mean_length": 6686.28125, "completions/mean_terminated_length": 6686.28125, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.3922265861183405, "epoch": 0.00061, "frac_reward_zero_std": 0.0, "grad_norm": 0.8862587213516235, "kl": 0.2304363353177905, "learning_rate": 9.83650369103696e-05, "loss": 0.0665, "num_tokens": 6917840.0, "reward": 0.26562872529029846, "reward_std": 0.5621640682220459, "rewards/rollout_reward_func/mean": 0.26562872529029846, "rewards/rollout_reward_func/std": 0.6765706539154053, "sampling/importance_sampling_ratio/max": 1.5996381044387817, "sampling/importance_sampling_ratio/mean": 1.0006206035614014, "sampling/importance_sampling_ratio/min": 0.34086817502975464, "sampling/sampling_logp_difference/max": 1.0762594938278198, "sampling/sampling_logp_difference/mean": 0.017746558412909508, "step": 61, "step_time": 73.57793928200226 }, { "clip_ratio/high_max": 0.07282545696943998, "clip_ratio/high_mean": 0.04729982512071729, "clip_ratio/low_mean": 0.059949032263830304, "clip_ratio/low_min": 0.031854838598519564, "clip_ratio/region_mean": 0.10724885761737823, "entropy": 0.3318217499181628, "epoch": 0.00062, "grad_norm": 1.2174389362335205, "kl": 0.2543012909591198, "learning_rate": 9.823268148061883e-05, "loss": 0.0686, "step": 62, "step_time": 30.085744329006047 }, { "clip_ratio/high_max": 0.002003205183427781, "clip_ratio/high_mean": 0.0010016025917138904, "clip_ratio/low_mean": 0.0038343425694620237, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.004835945161175914, "completions/clipped_ratio": 0.0, "completions/max_length": 7844.0, "completions/max_terminated_length": 7844.0, "completions/mean_length": 6572.125, "completions/mean_terminated_length": 6572.125, "completions/min_length": 842.0, "completions/min_terminated_length": 842.0, "entropy": 0.34065084904432297, "epoch": 0.00063, "frac_reward_zero_std": 0.0, "grad_norm": 0.9939379692077637, "kl": 0.28652930725365877, "learning_rate": 9.809530414556335e-05, "loss": 0.182, "num_tokens": 7153874.0, "reward": 0.2723328173160553, "reward_std": 0.6224352121353149, "rewards/rollout_reward_func/mean": 0.2723328173160553, "rewards/rollout_reward_func/std": 0.6490558385848999, "sampling/importance_sampling_ratio/max": 1.268694519996643, "sampling/importance_sampling_ratio/mean": 0.9983758926391602, "sampling/importance_sampling_ratio/min": 0.10849106311798096, "sampling/sampling_logp_difference/max": 2.2210874557495117, "sampling/sampling_logp_difference/mean": 0.01489231176674366, "step": 63, "step_time": 72.65448498899605 }, { "clip_ratio/high_max": 0.041519766324199736, "clip_ratio/high_mean": 0.021280716580804437, "clip_ratio/low_mean": 0.04069274431094527, "clip_ratio/low_min": 0.01329435856314376, "clip_ratio/region_mean": 0.06197346071712673, "entropy": 0.36371681839227676, "epoch": 0.00064, "grad_norm": 0.7126179337501526, "kl": 0.3891989327967167, "learning_rate": 9.79529242123455e-05, "loss": 0.1739, "step": 64, "step_time": 29.6159638930003 }, { "clip_ratio/high_max": 0.0038133251946419477, "clip_ratio/high_mean": 0.002531662641558796, "clip_ratio/low_mean": 0.0052659373031929135, "clip_ratio/low_min": 0.0020497312070801854, "clip_ratio/region_mean": 0.00779759994475171, "completions/clipped_ratio": 0.0, "completions/max_length": 7539.0, "completions/max_terminated_length": 7539.0, "completions/mean_length": 6040.46875, "completions/mean_terminated_length": 6040.46875, "completions/min_length": 1941.0, "completions/min_terminated_length": 1941.0, "entropy": 0.38560492917895317, "epoch": 0.00065, "frac_reward_zero_std": 0.0, "grad_norm": 0.7672582864761353, "kl": 0.32619187142699957, "learning_rate": 9.780556169117757e-05, "loss": 0.2255, "num_tokens": 7372685.0, "reward": 0.3717280626296997, "reward_std": 0.9961870312690735, "rewards/rollout_reward_func/mean": 0.3717280626296997, "rewards/rollout_reward_func/std": 0.9805747270584106, "sampling/importance_sampling_ratio/max": 1.4619451761245728, "sampling/importance_sampling_ratio/mean": 0.9994047284126282, "sampling/importance_sampling_ratio/min": 0.6509100794792175, "sampling/sampling_logp_difference/max": 0.4293837547302246, "sampling/sampling_logp_difference/mean": 0.01682940497994423, "step": 65, "step_time": 73.46941840999898 }, { "clip_ratio/high_max": 0.03314234549179673, "clip_ratio/high_mean": 0.01741576730273664, "clip_ratio/low_mean": 0.025530758663080633, "clip_ratio/low_min": 0.015594998490996659, "clip_ratio/region_mean": 0.04294652584940195, "entropy": 0.3837909437716007, "epoch": 0.00066, "grad_norm": 0.5528827905654907, "kl": 0.3576068542897701, "learning_rate": 9.765323729252955e-05, "loss": 0.2161, "step": 66, "step_time": 28.062951383997643 }, { "clip_ratio/high_max": 0.003645833465270698, "clip_ratio/high_mean": 0.001822916732635349, "clip_ratio/low_mean": 0.004098911420442164, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005921828153077513, "completions/clipped_ratio": 0.0, "completions/max_length": 7583.0, "completions/max_terminated_length": 7583.0, "completions/mean_length": 6035.90625, "completions/mean_terminated_length": 6035.90625, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.37318017426878214, "epoch": 0.00067, "frac_reward_zero_std": 0.0, "grad_norm": 0.7696178555488586, "kl": 0.346427702344954, "learning_rate": 9.749597242421838e-05, "loss": 0.1039, "num_tokens": 7591526.0, "reward": 0.19144931435585022, "reward_std": 0.7946555614471436, "rewards/rollout_reward_func/mean": 0.19144931435585022, "rewards/rollout_reward_func/std": 0.8030545115470886, "sampling/importance_sampling_ratio/max": 1.486856460571289, "sampling/importance_sampling_ratio/mean": 1.0006353855133057, "sampling/importance_sampling_ratio/min": 0.5241217017173767, "sampling/sampling_logp_difference/max": 0.646031379699707, "sampling/sampling_logp_difference/mean": 0.015466831624507904, "step": 67, "step_time": 73.64406822900128 }, { "clip_ratio/high_max": 0.02897409035358578, "clip_ratio/high_mean": 0.01448704517679289, "clip_ratio/low_mean": 0.03051818982930854, "clip_ratio/low_min": 0.009240591549314559, "clip_ratio/region_mean": 0.04500523506430909, "entropy": 0.36798784602433443, "epoch": 0.00068, "grad_norm": 0.5015326142311096, "kl": 0.37261209450662136, "learning_rate": 9.733378918839942e-05, "loss": 0.0944, "step": 68, "step_time": 28.811763866997353 }, { "clip_ratio/high_max": 0.014585631084628403, "clip_ratio/high_mean": 0.007871519133914262, "clip_ratio/low_mean": 0.004036458383779973, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.011907977517694235, "completions/clipped_ratio": 0.0, "completions/max_length": 7741.0, "completions/max_terminated_length": 7741.0, "completions/mean_length": 6178.28125, "completions/mean_terminated_length": 6178.28125, "completions/min_length": 1022.0, "completions/min_terminated_length": 1022.0, "entropy": 0.37665506824851036, "epoch": 0.00069, "frac_reward_zero_std": 0.0, "grad_norm": 0.7359852194786072, "kl": 0.33274756371974945, "learning_rate": 9.716671037846007e-05, "loss": -0.0416, "num_tokens": 7815041.0, "reward": 0.23862102627754211, "reward_std": 0.7055730819702148, "rewards/rollout_reward_func/mean": 0.23862102627754211, "rewards/rollout_reward_func/std": 0.841029703617096, "sampling/importance_sampling_ratio/max": 1.44895339012146, "sampling/importance_sampling_ratio/mean": 0.9988398551940918, "sampling/importance_sampling_ratio/min": 0.5431250929832458, "sampling/sampling_logp_difference/max": 0.6104156970977783, "sampling/sampling_logp_difference/mean": 0.015206092037260532, "step": 69, "step_time": 69.3552664519957 }, { "clip_ratio/high_max": 0.03422048385255039, "clip_ratio/high_mean": 0.01811830629594624, "clip_ratio/low_mean": 0.029536015121266246, "clip_ratio/low_min": 0.006182795739732683, "clip_ratio/region_mean": 0.047654321417212486, "entropy": 0.378166276961565, "epoch": 0.0007, "grad_norm": 0.5554056763648987, "kl": 0.3368592942133546, "learning_rate": 9.699475947581644e-05, "loss": -0.0543, "step": 70, "step_time": 29.88547446099983 }, { "clip_ratio/high_max": 0.002119252923876047, "clip_ratio/high_mean": 0.0010596264619380236, "clip_ratio/low_mean": 0.0010416667209938169, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021012931829318404, "completions/clipped_ratio": 0.0, "completions/max_length": 7532.0, "completions/max_terminated_length": 7532.0, "completions/mean_length": 5635.59375, "completions/mean_terminated_length": 5635.59375, "completions/min_length": 1460.0, "completions/min_terminated_length": 1460.0, "entropy": 0.37612876668572426, "epoch": 0.00071, "frac_reward_zero_std": 0.0, "grad_norm": 0.765031099319458, "kl": 0.32279551960527897, "learning_rate": 9.681796064661319e-05, "loss": 0.1589, "num_tokens": 8020959.0, "reward": 0.6217184066772461, "reward_std": 0.9535914063453674, "rewards/rollout_reward_func/mean": 0.6217184066772461, "rewards/rollout_reward_func/std": 0.948441743850708, "sampling/importance_sampling_ratio/max": 1.2822309732437134, "sampling/importance_sampling_ratio/mean": 0.9984654784202576, "sampling/importance_sampling_ratio/min": 0.6975119113922119, "sampling/sampling_logp_difference/max": 0.36023569107055664, "sampling/sampling_logp_difference/mean": 0.014132177457213402, "step": 71, "step_time": 70.46250126599807 }, { "clip_ratio/high_max": 0.03790286253206432, "clip_ratio/high_mean": 0.021369586000218987, "clip_ratio/low_mean": 0.0306891449727118, "clip_ratio/low_min": 0.012837638845667243, "clip_ratio/region_mean": 0.05205873120576143, "entropy": 0.3844588492065668, "epoch": 0.00072, "grad_norm": 0.6370178461074829, "kl": 0.36172858253121376, "learning_rate": 9.663633873832725e-05, "loss": 0.1532, "step": 72, "step_time": 27.620555648005393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0037812883383594453, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037812883383594453, "completions/clipped_ratio": 0.0, "completions/max_length": 7633.0, "completions/max_terminated_length": 7633.0, "completions/mean_length": 5546.9375, "completions/mean_terminated_length": 5546.9375, "completions/min_length": 1014.0, "completions/min_terminated_length": 1014.0, "entropy": 0.40227570198476315, "epoch": 0.00073, "frac_reward_zero_std": 0.0, "grad_norm": 2.0858843326568604, "kl": 3.081875999458134, "learning_rate": 9.644991927627566e-05, "loss": 0.2332, "num_tokens": 8223990.0, "reward": 0.3163941502571106, "reward_std": 0.7809635400772095, "rewards/rollout_reward_func/mean": 0.3163941502571106, "rewards/rollout_reward_func/std": 0.7874804735183716, "sampling/importance_sampling_ratio/max": 1.3201565742492676, "sampling/importance_sampling_ratio/mean": 0.9998225569725037, "sampling/importance_sampling_ratio/min": 0.6809157133102417, "sampling/sampling_logp_difference/max": 0.3843168020248413, "sampling/sampling_logp_difference/mean": 0.01574765145778656, "step": 73, "step_time": 70.71805355600009 }, { "clip_ratio/high_max": 0.042718877317383885, "clip_ratio/high_mean": 0.023442772100679576, "clip_ratio/low_mean": 0.010820692812558264, "clip_ratio/low_min": 0.003057795693166554, "clip_ratio/region_mean": 0.0342634649714455, "entropy": 0.43687280640006065, "epoch": 0.00074, "grad_norm": 2.8712551593780518, "kl": 0.4205539054237306, "learning_rate": 9.625872846002834e-05, "loss": 0.2452, "step": 74, "step_time": 28.811917900999106 }, { "clip_ratio/high_max": 0.0030908469343557954, "clip_ratio/high_mean": 0.0015454234671778977, "clip_ratio/low_mean": 0.002221201022621244, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003766624489799142, "completions/clipped_ratio": 0.0, "completions/max_length": 7343.0, "completions/max_terminated_length": 7343.0, "completions/mean_length": 5579.875, "completions/mean_terminated_length": 5579.875, "completions/min_length": 1716.0, "completions/min_terminated_length": 1716.0, "entropy": 0.4409548845142126, "epoch": 0.00075, "frac_reward_zero_std": 0.0, "grad_norm": 0.7172544002532959, "kl": 0.42821107245981693, "learning_rate": 9.606279315972582e-05, "loss": -0.0623, "num_tokens": 8428116.0, "reward": 0.2148970365524292, "reward_std": 0.6180732250213623, "rewards/rollout_reward_func/mean": 0.2148970365524292, "rewards/rollout_reward_func/std": 0.705778181552887, "sampling/importance_sampling_ratio/max": 1.2181819677352905, "sampling/importance_sampling_ratio/mean": 1.0005476474761963, "sampling/importance_sampling_ratio/min": 0.6817307472229004, "sampling/sampling_logp_difference/max": 0.3831205368041992, "sampling/sampling_logp_difference/mean": 0.015602965839207172, "step": 75, "step_time": 70.24528893599927 }, { "clip_ratio/high_max": 0.024824588908813894, "clip_ratio/high_mean": 0.016536269744392484, "clip_ratio/low_mean": 0.028347637795377523, "clip_ratio/low_min": 0.011356975883245468, "clip_ratio/region_mean": 0.04488390753977001, "entropy": 0.42724805884063244, "epoch": 0.00076, "grad_norm": 0.5589272379875183, "kl": 0.4489877112209797, "learning_rate": 9.586214091230304e-05, "loss": -0.0707, "step": 76, "step_time": 28.178340849997767 }, { "clip_ratio/high_max": 0.004374903510324657, "clip_ratio/high_mean": 0.0021874517551623285, "clip_ratio/low_mean": 0.004158128576818854, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.006345580331981182, "completions/clipped_ratio": 0.0, "completions/max_length": 7421.0, "completions/max_terminated_length": 7421.0, "completions/mean_length": 6100.25, "completions/mean_terminated_length": 6100.25, "completions/min_length": 1495.0, "completions/min_terminated_length": 1495.0, "entropy": 0.40994538739323616, "epoch": 0.00077, "frac_reward_zero_std": 0.0, "grad_norm": 0.784587562084198, "kl": 0.4144989335909486, "learning_rate": 9.565679991761914e-05, "loss": 0.1421, "num_tokens": 8649014.0, "reward": 0.5734952688217163, "reward_std": 0.6661400198936462, "rewards/rollout_reward_func/mean": 0.5734952688217163, "rewards/rollout_reward_func/std": 0.7511870861053467, "sampling/importance_sampling_ratio/max": 1.2402855157852173, "sampling/importance_sampling_ratio/mean": 0.9996535778045654, "sampling/importance_sampling_ratio/min": 0.7516164183616638, "sampling/sampling_logp_difference/max": 0.28552913665771484, "sampling/sampling_logp_difference/mean": 0.014470338821411133, "step": 77, "step_time": 69.64059652399737 }, { "clip_ratio/high_max": 0.033002695185132325, "clip_ratio/high_mean": 0.020024632045533508, "clip_ratio/low_mean": 0.05265966639854014, "clip_ratio/low_min": 0.016666667303070426, "clip_ratio/region_mean": 0.07268429850228131, "entropy": 0.39154940471053123, "epoch": 0.00078, "grad_norm": 0.7918930053710938, "kl": 0.41949744895100594, "learning_rate": 9.544679903449437e-05, "loss": 0.1331, "step": 78, "step_time": 29.688970042998335 }, { "clip_ratio/high_max": 0.0035214433446526527, "clip_ratio/high_mean": 0.0017607216723263264, "clip_ratio/low_mean": 0.003776041849050671, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005536763521376997, "completions/clipped_ratio": 0.0, "completions/max_length": 7664.0, "completions/max_terminated_length": 7664.0, "completions/mean_length": 6393.0, "completions/mean_terminated_length": 6393.0, "completions/min_length": 976.0, "completions/min_terminated_length": 976.0, "entropy": 0.40566546842455864, "epoch": 0.00079, "frac_reward_zero_std": 0.0, "grad_norm": 0.674810528755188, "kl": 0.4412749791517854, "learning_rate": 9.523216777665409e-05, "loss": -0.0678, "num_tokens": 8879033.0, "reward": 0.4701535105705261, "reward_std": 0.682727575302124, "rewards/rollout_reward_func/mean": 0.4701535105705261, "rewards/rollout_reward_func/std": 0.6995026469230652, "sampling/importance_sampling_ratio/max": 1.2723726034164429, "sampling/importance_sampling_ratio/mean": 0.9994304180145264, "sampling/importance_sampling_ratio/min": 0.7725619673728943, "sampling/sampling_logp_difference/max": 0.2580430507659912, "sampling/sampling_logp_difference/mean": 0.013721656054258347, "step": 79, "step_time": 72.87571495900374 }, { "clip_ratio/high_max": 0.019419116200879216, "clip_ratio/high_mean": 0.01033455808646977, "clip_ratio/low_mean": 0.02202079276321456, "clip_ratio/low_min": 0.008234293316490948, "clip_ratio/region_mean": 0.032355350733269006, "entropy": 0.4276595711708069, "epoch": 0.0008, "grad_norm": 0.5430514812469482, "kl": 0.4195341719314456, "learning_rate": 9.501293630858103e-05, "loss": -0.0765, "step": 80, "step_time": 29.131975957001487 }, { "clip_ratio/high_max": 0.003564814804121852, "clip_ratio/high_mean": 0.001782407402060926, "clip_ratio/low_mean": 0.009357923467177898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011140330869238824, "completions/clipped_ratio": 0.0, "completions/max_length": 7626.0, "completions/max_terminated_length": 7626.0, "completions/mean_length": 5398.875, "completions/mean_terminated_length": 5398.875, "completions/min_length": 175.0, "completions/min_terminated_length": 175.0, "entropy": 0.4494451526552439, "epoch": 0.00081, "frac_reward_zero_std": 0.0, "grad_norm": 0.6621186137199402, "kl": 0.6062396988272667, "learning_rate": 9.478913544127583e-05, "loss": 0.1128, "num_tokens": 9077491.0, "reward": 0.46084797382354736, "reward_std": 0.8951297998428345, "rewards/rollout_reward_func/mean": 0.46084797382354736, "rewards/rollout_reward_func/std": 0.8674929141998291, "sampling/importance_sampling_ratio/max": 1.2711063623428345, "sampling/importance_sampling_ratio/mean": 0.9985975027084351, "sampling/importance_sampling_ratio/min": 0.687078595161438, "sampling/sampling_logp_difference/max": 0.3753066062927246, "sampling/sampling_logp_difference/mean": 0.01631605625152588, "step": 81, "step_time": 69.41076812199753 }, { "clip_ratio/high_max": 0.03398265643045306, "clip_ratio/high_mean": 0.018866328289732337, "clip_ratio/low_mean": 0.017563854285981506, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.036430182750336826, "entropy": 0.46204396337270737, "epoch": 0.00082, "grad_norm": 0.49996891617774963, "kl": 0.5966967437416315, "learning_rate": 9.45607966279269e-05, "loss": 0.1057, "step": 82, "step_time": 28.260208684001555 }, { "clip_ratio/high_max": 0.0021873097866773605, "clip_ratio/high_mean": 0.0010936548933386803, "clip_ratio/low_mean": 0.0012001811992377043, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022938361507840455, "completions/clipped_ratio": 0.0, "completions/max_length": 7655.0, "completions/max_terminated_length": 7655.0, "completions/mean_length": 6103.28125, "completions/mean_terminated_length": 6103.28125, "completions/min_length": 1520.0, "completions/min_terminated_length": 1520.0, "entropy": 0.5095194447785616, "epoch": 0.00083, "frac_reward_zero_std": 0.0, "grad_norm": 0.7111266851425171, "kl": 0.47536701802164316, "learning_rate": 9.432795195948994e-05, "loss": 0.1306, "num_tokens": 9298521.0, "reward": 0.47608229517936707, "reward_std": 0.654423177242279, "rewards/rollout_reward_func/mean": 0.47608229517936707, "rewards/rollout_reward_func/std": 0.7807072401046753, "sampling/importance_sampling_ratio/max": 1.4298118352890015, "sampling/importance_sampling_ratio/mean": 1.0000498294830322, "sampling/importance_sampling_ratio/min": 0.6157028079032898, "sampling/sampling_logp_difference/max": 0.4849909543991089, "sampling/sampling_logp_difference/mean": 0.01652347669005394, "step": 83, "step_time": 72.13695794999876 }, { "clip_ratio/high_max": 0.030542047577910125, "clip_ratio/high_mean": 0.016450269089546055, "clip_ratio/low_mean": 0.03119568293914199, "clip_ratio/low_min": 0.014532104483805597, "clip_ratio/region_mean": 0.04764595243614167, "entropy": 0.5092578958719969, "epoch": 0.00084, "grad_norm": 0.5727465152740479, "kl": 0.48810105863958597, "learning_rate": 9.409063416017778e-05, "loss": 0.1195, "step": 84, "step_time": 29.438322223000796 }, { "clip_ratio/high_max": 0.0018382353009656072, "clip_ratio/high_mean": 0.0009191176504828036, "clip_ratio/low_mean": 0.003596204798668623, "clip_ratio/low_min": 0.0010245901066809893, "clip_ratio/region_mean": 0.0045153224491514266, "completions/clipped_ratio": 0.0, "completions/max_length": 7581.0, "completions/max_terminated_length": 7581.0, "completions/mean_length": 5999.6875, "completions/mean_terminated_length": 5999.6875, "completions/min_length": 1092.0, "completions/min_terminated_length": 1092.0, "entropy": 0.47864437103271484, "epoch": 0.00085, "frac_reward_zero_std": 0.0, "grad_norm": 0.7291382551193237, "kl": 0.5192659590393305, "learning_rate": 9.384887658286146e-05, "loss": 0.0204, "num_tokens": 9515912.0, "reward": 0.36224642395973206, "reward_std": 0.6394367814064026, "rewards/rollout_reward_func/mean": 0.36224642395973206, "rewards/rollout_reward_func/std": 0.7219645977020264, "sampling/importance_sampling_ratio/max": 1.382703185081482, "sampling/importance_sampling_ratio/mean": 1.0010368824005127, "sampling/importance_sampling_ratio/min": 0.7111052870750427, "sampling/sampling_logp_difference/max": 0.34093475341796875, "sampling/sampling_logp_difference/mean": 0.014877153560519218, "step": 85, "step_time": 71.58443966300183 }, { "clip_ratio/high_max": 0.03432386869098991, "clip_ratio/high_mean": 0.017682767647784203, "clip_ratio/low_mean": 0.03623598051490262, "clip_ratio/low_min": 0.01192793739028275, "clip_ratio/region_mean": 0.05391874792985618, "entropy": 0.49723889492452145, "epoch": 0.00086, "grad_norm": 0.5472823977470398, "kl": 0.581278195604682, "learning_rate": 9.360271320438257e-05, "loss": 0.0109, "step": 86, "step_time": 29.024181225006032 }, { "clip_ratio/high_max": 0.0056201552506536245, "clip_ratio/high_mean": 0.0028100776253268123, "clip_ratio/low_mean": 0.0015625000814907253, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0043725777068175375, "completions/clipped_ratio": 0.0, "completions/max_length": 7501.0, "completions/max_terminated_length": 7501.0, "completions/mean_length": 5934.15625, "completions/mean_terminated_length": 5934.15625, "completions/min_length": 1989.0, "completions/min_terminated_length": 1989.0, "entropy": 0.49490711465477943, "epoch": 0.00087, "frac_reward_zero_std": 0.0, "grad_norm": 0.7214106917381287, "kl": 0.5088155306875706, "learning_rate": 9.33521786207783e-05, "loss": 0.0658, "num_tokens": 9731130.0, "reward": 0.3657677173614502, "reward_std": 0.6968203186988831, "rewards/rollout_reward_func/mean": 0.3657677173614502, "rewards/rollout_reward_func/std": 0.7031262516975403, "sampling/importance_sampling_ratio/max": 1.242927074432373, "sampling/importance_sampling_ratio/mean": 1.000451683998108, "sampling/importance_sampling_ratio/min": 0.6069751977920532, "sampling/sampling_logp_difference/max": 0.4992673397064209, "sampling/sampling_logp_difference/mean": 0.014659843407571316, "step": 87, "step_time": 69.90900443599821 }, { "clip_ratio/high_max": 0.034180735470727086, "clip_ratio/high_mean": 0.02109334385022521, "clip_ratio/low_mean": 0.020963759627193213, "clip_ratio/low_min": 0.009244791930541396, "clip_ratio/region_mean": 0.04205710318638012, "entropy": 0.5163823496550322, "epoch": 0.00088, "grad_norm": 0.5823494791984558, "kl": 0.5176579030230641, "learning_rate": 9.309730804241916e-05, "loss": 0.0555, "step": 88, "step_time": 28.185624415998973 }, { "clip_ratio/high_max": 0.0040902887703850865, "clip_ratio/high_mean": 0.0020451443851925433, "clip_ratio/low_mean": 0.0031992303556762636, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005244374740868807, "completions/clipped_ratio": 0.0, "completions/max_length": 7441.0, "completions/max_terminated_length": 7441.0, "completions/mean_length": 5460.90625, "completions/mean_terminated_length": 5460.90625, "completions/min_length": 1403.0, "completions/min_terminated_length": 1403.0, "entropy": 0.5702018495649099, "epoch": 0.00089, "frac_reward_zero_std": 0.0, "grad_norm": 0.8969010710716248, "kl": 0.608772374689579, "learning_rate": 9.283813728906054e-05, "loss": 0.0756, "num_tokens": 9931443.0, "reward": 0.3105483651161194, "reward_std": 0.8500173091888428, "rewards/rollout_reward_func/mean": 0.3105483651161194, "rewards/rollout_reward_func/std": 0.8466864824295044, "sampling/importance_sampling_ratio/max": 1.3580117225646973, "sampling/importance_sampling_ratio/mean": 0.9979798197746277, "sampling/importance_sampling_ratio/min": 0.7695879936218262, "sampling/sampling_logp_difference/max": 0.30602169036865234, "sampling/sampling_logp_difference/mean": 0.016787465661764145, "step": 89, "step_time": 71.05121652899652 }, { "clip_ratio/high_max": 0.06229703710414469, "clip_ratio/high_mean": 0.031148518552072346, "clip_ratio/low_mean": 0.04073557164520025, "clip_ratio/low_min": 0.011149403639137745, "clip_ratio/region_mean": 0.07188408949878067, "entropy": 0.5980537086725235, "epoch": 0.0009, "grad_norm": 0.5781596302986145, "kl": 0.549474336206913, "learning_rate": 9.257470278480848e-05, "loss": 0.0651, "step": 90, "step_time": 27.60651146100281 }, { "clip_ratio/high_max": 0.004329249961301684, "clip_ratio/high_mean": 0.0028913691639900208, "clip_ratio/low_mean": 0.0020665323245339096, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00495790148852393, "completions/clipped_ratio": 0.0, "completions/max_length": 7488.0, "completions/max_terminated_length": 7488.0, "completions/mean_length": 5900.15625, "completions/mean_terminated_length": 5900.15625, "completions/min_length": 658.0, "completions/min_terminated_length": 658.0, "entropy": 0.5577217228710651, "epoch": 0.00091, "frac_reward_zero_std": 0.0, "grad_norm": 0.8316934704780579, "kl": 0.5660237148404121, "learning_rate": 9.230704155300075e-05, "loss": 0.0711, "num_tokens": 10145685.0, "reward": 0.4379217028617859, "reward_std": 0.8983248472213745, "rewards/rollout_reward_func/mean": 0.4379217028617859, "rewards/rollout_reward_func/std": 0.9448426961898804, "sampling/importance_sampling_ratio/max": 1.2334064245224, "sampling/importance_sampling_ratio/mean": 1.001152515411377, "sampling/importance_sampling_ratio/min": 0.7529183626174927, "sampling/sampling_logp_difference/max": 0.2837984561920166, "sampling/sampling_logp_difference/mean": 0.016148308292031288, "step": 91, "step_time": 70.24466091300201 }, { "clip_ratio/high_max": 0.026693037478253245, "clip_ratio/high_mean": 0.016509755398146808, "clip_ratio/low_mean": 0.0332915322505869, "clip_ratio/low_min": 0.012363388203084469, "clip_ratio/region_mean": 0.049801287648733705, "entropy": 0.5256938096135855, "epoch": 0.00092, "grad_norm": 0.6605801582336426, "kl": 0.592466913163662, "learning_rate": 9.20351912110034e-05, "loss": 0.0601, "step": 92, "step_time": 29.181030667001323 }, { "clip_ratio/high_max": 0.005846500280313194, "clip_ratio/high_mean": 0.002923250140156597, "clip_ratio/low_mean": 0.004082385799847543, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007005635998211801, "completions/clipped_ratio": 0.0, "completions/max_length": 7530.0, "completions/max_terminated_length": 7530.0, "completions/mean_length": 6185.875, "completions/mean_terminated_length": 6185.875, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.5268344059586525, "epoch": 0.00093, "frac_reward_zero_std": 0.0, "grad_norm": 0.8448914885520935, "kl": 0.4634496849030256, "learning_rate": 9.175918996492408e-05, "loss": 0.0301, "num_tokens": 10369238.0, "reward": 0.6354137659072876, "reward_std": 0.9133501052856445, "rewards/rollout_reward_func/mean": 0.6354137659072876, "rewards/rollout_reward_func/std": 0.872856080532074, "sampling/importance_sampling_ratio/max": 1.5508137941360474, "sampling/importance_sampling_ratio/mean": 1.0002168416976929, "sampling/importance_sampling_ratio/min": 0.6144588589668274, "sampling/sampling_logp_difference/max": 0.4870133399963379, "sampling/sampling_logp_difference/mean": 0.01783657632768154, "step": 93, "step_time": 73.25490733300103 }, { "clip_ratio/high_max": 0.037143203779123724, "clip_ratio/high_mean": 0.020906860067043453, "clip_ratio/low_mean": 0.032359570293920115, "clip_ratio/low_min": 0.010215053800493479, "clip_ratio/region_mean": 0.05326643056469038, "entropy": 0.5237733330577612, "epoch": 0.00094, "grad_norm": 0.6308583617210388, "kl": 0.4853533152490854, "learning_rate": 9.147907660424242e-05, "loss": 0.0167, "step": 94, "step_time": 28.745761320000383 }, { "clip_ratio/high_max": 0.0045244983630254865, "clip_ratio/high_mean": 0.0022622491815127432, "clip_ratio/low_mean": 0.0020665323245339096, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004328781506046653, "completions/clipped_ratio": 0.0, "completions/max_length": 7449.0, "completions/max_terminated_length": 7449.0, "completions/mean_length": 5757.25, "completions/mean_terminated_length": 5757.25, "completions/min_length": 760.0, "completions/min_terminated_length": 760.0, "entropy": 0.508440975099802, "epoch": 0.00095, "frac_reward_zero_std": 0.0, "grad_norm": 0.8011288046836853, "kl": 0.5558518571779132, "learning_rate": 9.119489049635865e-05, "loss": 0.1729, "num_tokens": 10578915.0, "reward": 0.4583550691604614, "reward_std": 0.813267707824707, "rewards/rollout_reward_func/mean": 0.4583550691604614, "rewards/rollout_reward_func/std": 0.8316054940223694, "sampling/importance_sampling_ratio/max": 1.309999704360962, "sampling/importance_sampling_ratio/mean": 0.9987903833389282, "sampling/importance_sampling_ratio/min": 0.6942355036735535, "sampling/sampling_logp_difference/max": 0.3649439811706543, "sampling/sampling_logp_difference/mean": 0.016344327479600906, "step": 95, "step_time": 72.8988592930018 }, { "clip_ratio/high_max": 0.013936874573118985, "clip_ratio/high_mean": 0.006968437286559492, "clip_ratio/low_mean": 0.034929006826132536, "clip_ratio/low_min": 0.017001205822452903, "clip_ratio/region_mean": 0.04189744417089969, "entropy": 0.4919160632416606, "epoch": 0.00096, "grad_norm": 0.72597736120224, "kl": 0.5817184709012508, "learning_rate": 9.090667158106077e-05, "loss": 0.1629, "step": 96, "step_time": 27.976823670996964 }, { "clip_ratio/high_max": 0.002032654592767358, "clip_ratio/high_mean": 0.001016327296383679, "clip_ratio/low_mean": 0.0032271373784169555, "clip_ratio/low_min": 0.0011363636003807187, "clip_ratio/region_mean": 0.004243464674800634, "completions/clipped_ratio": 0.0, "completions/max_length": 7601.0, "completions/max_terminated_length": 7601.0, "completions/mean_length": 6449.625, "completions/mean_terminated_length": 6449.625, "completions/min_length": 3588.0, "completions/min_terminated_length": 3588.0, "entropy": 0.5690648220479488, "epoch": 0.00097, "frac_reward_zero_std": 0.0, "grad_norm": 0.9542885422706604, "kl": 0.5515561867505312, "learning_rate": 9.061446036491135e-05, "loss": 0.0843, "num_tokens": 10810600.0, "reward": 0.49962183833122253, "reward_std": 0.7437477111816406, "rewards/rollout_reward_func/mean": 0.49962183833122253, "rewards/rollout_reward_func/std": 0.7686313986778259, "sampling/importance_sampling_ratio/max": 1.2191413640975952, "sampling/importance_sampling_ratio/mean": 0.9996674060821533, "sampling/importance_sampling_ratio/min": 0.6939353346824646, "sampling/sampling_logp_difference/max": 0.36537647247314453, "sampling/sampling_logp_difference/mean": 0.015800289809703827, "step": 97, "step_time": 72.87922959299613 }, { "clip_ratio/high_max": 0.03395288507454097, "clip_ratio/high_mean": 0.01991559809539467, "clip_ratio/low_mean": 0.061350286239758134, "clip_ratio/low_min": 0.02331683784723282, "clip_ratio/region_mean": 0.08126588445156813, "entropy": 0.5272827949374914, "epoch": 0.00098, "grad_norm": 0.9256690144538879, "kl": 0.6228653602302074, "learning_rate": 9.03182979155548e-05, "loss": 0.0734, "step": 98, "step_time": 28.18560988900026 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.0016927084070630372, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.003113162994850427, "completions/clipped_ratio": 0.0, "completions/max_length": 7680.0, "completions/max_terminated_length": 7680.0, "completions/mean_length": 5903.28125, "completions/mean_terminated_length": 5903.28125, "completions/min_length": 1542.0, "completions/min_terminated_length": 1542.0, "entropy": 0.48266540840268135, "epoch": 0.00099, "frac_reward_zero_std": 0.0, "grad_norm": 0.7833912968635559, "kl": 0.44862770196050406, "learning_rate": 9.001822585594566e-05, "loss": 0.0352, "num_tokens": 11024946.0, "reward": 0.4953843951225281, "reward_std": 0.8200429677963257, "rewards/rollout_reward_func/mean": 0.4953843951225281, "rewards/rollout_reward_func/std": 0.9419152140617371, "sampling/importance_sampling_ratio/max": 1.3456898927688599, "sampling/importance_sampling_ratio/mean": 0.9980398416519165, "sampling/importance_sampling_ratio/min": 0.697952389717102, "sampling/sampling_logp_difference/max": 0.3596043586730957, "sampling/sampling_logp_difference/mean": 0.015101483091711998, "step": 99, "step_time": 72.09978903999581 }, { "clip_ratio/high_max": 0.04411832708865404, "clip_ratio/high_mean": 0.023925250861793756, "clip_ratio/low_mean": 0.019601643201895058, "clip_ratio/low_min": 0.006085638655349612, "clip_ratio/region_mean": 0.04352689447114244, "entropy": 0.49165011942386627, "epoch": 0.001, "grad_norm": 0.6402556300163269, "kl": 0.44617504347115755, "learning_rate": 8.971428635849876e-05, "loss": 0.029, "step": 100, "step_time": 29.76168408399826 }, { "clip_ratio/high_max": 0.0011574074160307646, "clip_ratio/high_mean": 0.0005787037080153823, "clip_ratio/low_mean": 0.0015539617743343115, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021326654823496938, "completions/clipped_ratio": 0.0, "completions/max_length": 7385.0, "completions/max_terminated_length": 7385.0, "completions/mean_length": 5582.59375, "completions/mean_terminated_length": 5582.59375, "completions/min_length": 1058.0, "completions/min_terminated_length": 1058.0, "entropy": 0.48347699642181396, "epoch": 0.00101, "frac_reward_zero_std": 0.0, "grad_norm": 0.9301091432571411, "kl": 0.4479141626507044, "learning_rate": 8.940652213916242e-05, "loss": -0.0409, "num_tokens": 11228856.0, "reward": 0.624025821685791, "reward_std": 0.7526465058326721, "rewards/rollout_reward_func/mean": 0.624025821685791, "rewards/rollout_reward_func/std": 0.8259312510490417, "sampling/importance_sampling_ratio/max": 1.4037631750106812, "sampling/importance_sampling_ratio/mean": 0.9999610185623169, "sampling/importance_sampling_ratio/min": 0.7249770760536194, "sampling/sampling_logp_difference/max": 0.3391566276550293, "sampling/sampling_logp_difference/mean": 0.01585567370057106, "step": 101, "step_time": 67.26555217100213 }, { "clip_ratio/high_max": 0.027707159286364913, "clip_ratio/high_mean": 0.016085722600109875, "clip_ratio/low_mean": 0.042682581464760005, "clip_ratio/low_min": 0.01653115637600422, "clip_ratio/region_mean": 0.058768304297700524, "entropy": 0.47125959396362305, "epoch": 0.00102, "grad_norm": 0.6703765392303467, "kl": 0.44803958013653755, "learning_rate": 8.9094976451415e-05, "loss": -0.0548, "step": 102, "step_time": 27.171368667000934 }, { "clip_ratio/high_max": 0.005793396616354585, "clip_ratio/high_mean": 0.0034175317268818617, "clip_ratio/low_mean": 0.0030743213137611747, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006491853040643036, "completions/clipped_ratio": 0.0, "completions/max_length": 7518.0, "completions/max_terminated_length": 7518.0, "completions/mean_length": 6146.1875, "completions/mean_terminated_length": 6146.1875, "completions/min_length": 1986.0, "completions/min_terminated_length": 1986.0, "entropy": 0.45371031016111374, "epoch": 0.00103, "frac_reward_zero_std": 0.0, "grad_norm": 1.0939282178878784, "kl": 0.4932640716433525, "learning_rate": 8.877969308018608e-05, "loss": 0.1766, "num_tokens": 11450912.0, "reward": 0.735064685344696, "reward_std": 0.8077653050422668, "rewards/rollout_reward_func/mean": 0.735064685344696, "rewards/rollout_reward_func/std": 0.7813030481338501, "sampling/importance_sampling_ratio/max": 1.2896977663040161, "sampling/importance_sampling_ratio/mean": 0.9993119835853577, "sampling/importance_sampling_ratio/min": 0.4944234788417816, "sampling/sampling_logp_difference/max": 0.7043628692626953, "sampling/sampling_logp_difference/mean": 0.015341417863965034, "step": 103, "step_time": 71.5954989170059 }, { "clip_ratio/high_max": 0.03924982901662588, "clip_ratio/high_mean": 0.022087035584263504, "clip_ratio/low_mean": 0.05070123152108863, "clip_ratio/low_min": 0.025220642797648907, "clip_ratio/region_mean": 0.07278826658148319, "entropy": 0.43081194534897804, "epoch": 0.00104, "grad_norm": 1.0702099800109863, "kl": 0.5175904352217913, "learning_rate": 8.846071633570285e-05, "loss": 0.1765, "step": 104, "step_time": 28.647956558012083 }, { "clip_ratio/high_max": 0.007951958104968071, "clip_ratio/high_mean": 0.0039759790524840355, "clip_ratio/low_mean": 0.004548150027403608, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.008524129079887643, "completions/clipped_ratio": 0.0, "completions/max_length": 7541.0, "completions/max_terminated_length": 7541.0, "completions/mean_length": 6141.96875, "completions/mean_terminated_length": 6141.96875, "completions/min_length": 2012.0, "completions/min_terminated_length": 2012.0, "entropy": 0.48021323420107365, "epoch": 0.00105, "frac_reward_zero_std": 0.0, "grad_norm": 0.8135919570922852, "kl": 0.5053455037996173, "learning_rate": 8.81380910472627e-05, "loss": 0.01, "num_tokens": 11672810.0, "reward": 0.5856585502624512, "reward_std": 0.9218465089797974, "rewards/rollout_reward_func/mean": 0.5856585502624512, "rewards/rollout_reward_func/std": 0.9378327131271362, "sampling/importance_sampling_ratio/max": 1.5461221933364868, "sampling/importance_sampling_ratio/mean": 1.0002919435501099, "sampling/importance_sampling_ratio/min": 0.7441051006317139, "sampling/sampling_logp_difference/max": 0.43575000762939453, "sampling/sampling_logp_difference/mean": 0.016296371817588806, "step": 105, "step_time": 71.01448985200477 }, { "clip_ratio/high_max": 0.033413808792829514, "clip_ratio/high_mean": 0.02011335955467075, "clip_ratio/low_mean": 0.054817499592900276, "clip_ratio/low_min": 0.02631788863800466, "clip_ratio/region_mean": 0.07493085949681699, "entropy": 0.4995097406208515, "epoch": 0.00106, "grad_norm": 0.708191454410553, "kl": 0.5532285096123815, "learning_rate": 8.78118625569329e-05, "loss": 0.0063, "step": 106, "step_time": 29.326790068993432 }, { "clip_ratio/high_max": 0.01274776179343462, "clip_ratio/high_mean": 0.006922126514837146, "clip_ratio/low_mean": 0.004202116280794144, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01112424279563129, "completions/clipped_ratio": 0.0, "completions/max_length": 7701.0, "completions/max_terminated_length": 7701.0, "completions/mean_length": 5437.40625, "completions/mean_terminated_length": 5437.40625, "completions/min_length": 1218.0, "completions/min_terminated_length": 1218.0, "entropy": 0.44049233943223953, "epoch": 0.00107, "frac_reward_zero_std": 0.0, "grad_norm": 0.9201698899269104, "kl": 0.5217599831521511, "learning_rate": 8.748207671317818e-05, "loss": 0.1518, "num_tokens": 11872077.0, "reward": 0.9152963161468506, "reward_std": 0.8030579090118408, "rewards/rollout_reward_func/mean": 0.9152963161468506, "rewards/rollout_reward_func/std": 0.7965114712715149, "sampling/importance_sampling_ratio/max": 1.3733481168746948, "sampling/importance_sampling_ratio/mean": 1.002716302871704, "sampling/importance_sampling_ratio/min": 0.7098954916000366, "sampling/sampling_logp_difference/max": 0.3426375389099121, "sampling/sampling_logp_difference/mean": 0.014386315830051899, "step": 107, "step_time": 70.06009158701272 }, { "clip_ratio/high_max": 0.025378714548423886, "clip_ratio/high_mean": 0.013891280279494822, "clip_ratio/low_mean": 0.03500447154510766, "clip_ratio/low_min": 0.012247157632373273, "clip_ratio/region_mean": 0.04889575263950974, "entropy": 0.4588348101824522, "epoch": 0.00108, "grad_norm": 0.686837911605835, "kl": 0.5159835238009691, "learning_rate": 8.714877986441713e-05, "loss": 0.1395, "step": 108, "step_time": 28.99311024200506 }, { "clip_ratio/high_max": 0.005978302680887282, "clip_ratio/high_mean": 0.002989151340443641, "clip_ratio/low_mean": 0.004562929039821029, "clip_ratio/low_min": 0.0009920635493472219, "clip_ratio/region_mean": 0.00755208038026467, "completions/clipped_ratio": 0.0, "completions/max_length": 7412.0, "completions/max_terminated_length": 7412.0, "completions/mean_length": 5322.71875, "completions/mean_terminated_length": 5322.71875, "completions/min_length": 1166.0, "completions/min_terminated_length": 1166.0, "entropy": 0.543195066973567, "epoch": 0.00109, "frac_reward_zero_std": 0.0, "grad_norm": 0.8315640687942505, "kl": 0.5399283822625875, "learning_rate": 8.681201885250831e-05, "loss": 0.1882, "num_tokens": 12067677.0, "reward": 0.6111374497413635, "reward_std": 0.8293532133102417, "rewards/rollout_reward_func/mean": 0.6111374497413635, "rewards/rollout_reward_func/std": 0.8299290537834167, "sampling/importance_sampling_ratio/max": 1.3433170318603516, "sampling/importance_sampling_ratio/mean": 0.9988371133804321, "sampling/importance_sampling_ratio/min": 0.29309409856796265, "sampling/sampling_logp_difference/max": 1.2272615432739258, "sampling/sampling_logp_difference/mean": 0.01797650009393692, "step": 109, "step_time": 67.26937316900876 }, { "clip_ratio/high_max": 0.043637046590447426, "clip_ratio/high_mean": 0.02358935633674264, "clip_ratio/low_mean": 0.04582322621718049, "clip_ratio/low_min": 0.02066367061343044, "clip_ratio/region_mean": 0.06941258243750781, "entropy": 0.4882536642253399, "epoch": 0.0011, "grad_norm": 0.6777316331863403, "kl": 0.5754590313881636, "learning_rate": 8.647184100616712e-05, "loss": 0.1783, "step": 110, "step_time": 27.47480677299609 }, { "clip_ratio/high_max": 0.005861102021299303, "clip_ratio/high_mean": 0.0029305510106496513, "clip_ratio/low_mean": 0.005353779561119154, "clip_ratio/low_min": 0.0006793478387407959, "clip_ratio/region_mean": 0.008284330542664975, "completions/clipped_ratio": 0.0, "completions/max_length": 7779.0, "completions/max_terminated_length": 7779.0, "completions/mean_length": 6675.46875, "completions/mean_terminated_length": 6675.46875, "completions/min_length": 2522.0, "completions/min_terminated_length": 2522.0, "entropy": 0.4639120027422905, "epoch": 0.00111, "frac_reward_zero_std": 0.0, "grad_norm": 0.9155011773109436, "kl": 0.5613513607531786, "learning_rate": 8.612829413431418e-05, "loss": 0.1975, "num_tokens": 12306650.0, "reward": 0.5449673533439636, "reward_std": 0.8542670011520386, "rewards/rollout_reward_func/mean": 0.5449673533439636, "rewards/rollout_reward_func/std": 0.8721972703933716, "sampling/importance_sampling_ratio/max": 2.0089263916015625, "sampling/importance_sampling_ratio/mean": 1.0015928745269775, "sampling/importance_sampling_ratio/min": 0.7398360967636108, "sampling/sampling_logp_difference/max": 0.6976003646850586, "sampling/sampling_logp_difference/mean": 0.016591046005487442, "step": 111, "step_time": 74.73249227600536 }, { "clip_ratio/high_max": 0.024599458323791623, "clip_ratio/high_mean": 0.013276291661895812, "clip_ratio/low_mean": 0.05496020085411146, "clip_ratio/low_min": 0.023701343685388565, "clip_ratio/region_mean": 0.06823649210855365, "entropy": 0.4363186452537775, "epoch": 0.00112, "grad_norm": 0.7674146294593811, "kl": 0.5711267096921802, "learning_rate": 8.578142651935609e-05, "loss": 0.1863, "step": 112, "step_time": 29.034983964997082 }, { "clip_ratio/high_max": 0.00884536630474031, "clip_ratio/high_mean": 0.004422683152370155, "clip_ratio/low_mean": 0.004356278921477497, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.00877896195743233, "completions/clipped_ratio": 0.0, "completions/max_length": 7737.0, "completions/max_terminated_length": 7737.0, "completions/mean_length": 6377.46875, "completions/mean_terminated_length": 6377.46875, "completions/min_length": 1992.0, "completions/min_terminated_length": 1992.0, "entropy": 0.42672330886125565, "epoch": 0.00113, "frac_reward_zero_std": 0.0, "grad_norm": 0.9103497862815857, "kl": 0.521999379619956, "learning_rate": 8.543128691039995e-05, "loss": 0.0964, "num_tokens": 12536129.0, "reward": 0.6508508920669556, "reward_std": 0.8222292065620422, "rewards/rollout_reward_func/mean": 0.6508508920669556, "rewards/rollout_reward_func/std": 0.8533644080162048, "sampling/importance_sampling_ratio/max": 1.415414810180664, "sampling/importance_sampling_ratio/mean": 0.9990522861480713, "sampling/importance_sampling_ratio/min": 0.5567319989204407, "sampling/sampling_logp_difference/max": 0.5856713056564331, "sampling/sampling_logp_difference/mean": 0.015725748613476753, "step": 113, "step_time": 70.06058273700546 }, { "clip_ratio/high_max": 0.032130034756846726, "clip_ratio/high_mean": 0.018545176077168435, "clip_ratio/low_mean": 0.04433334164787084, "clip_ratio/low_min": 0.019422042882069945, "clip_ratio/region_mean": 0.06287851766683161, "entropy": 0.42895147763192654, "epoch": 0.00114, "grad_norm": 0.7597185373306274, "kl": 0.5722134839743376, "learning_rate": 8.507792451640202e-05, "loss": 0.0878, "step": 114, "step_time": 29.248457103003602 }, { "clip_ratio/high_max": 0.00443491549231112, "clip_ratio/high_mean": 0.00221745774615556, "clip_ratio/low_mean": 0.006689558154903352, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008907015959266573, "completions/clipped_ratio": 0.0, "completions/max_length": 7539.0, "completions/max_terminated_length": 7539.0, "completions/mean_length": 5665.25, "completions/mean_terminated_length": 5665.25, "completions/min_length": 1246.0, "completions/min_terminated_length": 1246.0, "entropy": 0.4168203379958868, "epoch": 0.00115, "frac_reward_zero_std": 0.0, "grad_norm": 0.8924338221549988, "kl": 0.4583917185664177, "learning_rate": 8.472138899925184e-05, "loss": 0.1573, "num_tokens": 12742738.0, "reward": 0.7131021618843079, "reward_std": 0.8330323696136475, "rewards/rollout_reward_func/mean": 0.7131021618843079, "rewards/rollout_reward_func/std": 0.8250030279159546, "sampling/importance_sampling_ratio/max": 1.416430950164795, "sampling/importance_sampling_ratio/mean": 1.0009844303131104, "sampling/importance_sampling_ratio/min": 0.3399147391319275, "sampling/sampling_logp_difference/max": 1.079060435295105, "sampling/sampling_logp_difference/mean": 0.015324528329074383, "step": 115, "step_time": 69.33891673600374 }, { "clip_ratio/high_max": 0.0371712117921561, "clip_ratio/high_mean": 0.019627272500656545, "clip_ratio/low_mean": 0.034254872356541455, "clip_ratio/low_min": 0.010120547842234373, "clip_ratio/region_mean": 0.053882144624367356, "entropy": 0.43328261002898216, "epoch": 0.00116, "grad_norm": 0.5150614976882935, "kl": 0.4780856240540743, "learning_rate": 8.43617304667927e-05, "loss": 0.1444, "step": 116, "step_time": 28.656909434997942 }, { "clip_ratio/high_max": 0.003824300831183791, "clip_ratio/high_mean": 0.0019121504155918956, "clip_ratio/low_mean": 0.004263552749762312, "clip_ratio/low_min": 0.0016874123248271644, "clip_ratio/region_mean": 0.006175703136250377, "completions/clipped_ratio": 0.0, "completions/max_length": 7602.0, "completions/max_terminated_length": 7602.0, "completions/mean_length": 6215.375, "completions/mean_terminated_length": 6215.375, "completions/min_length": 1141.0, "completions/min_terminated_length": 1141.0, "entropy": 0.4935767464339733, "epoch": 0.00117, "frac_reward_zero_std": 0.0, "grad_norm": 1.0525767803192139, "kl": 0.551841301843524, "learning_rate": 8.399899946577953e-05, "loss": 0.2166, "num_tokens": 12966946.0, "reward": 0.646992027759552, "reward_std": 0.8566222190856934, "rewards/rollout_reward_func/mean": 0.646992027759552, "rewards/rollout_reward_func/std": 0.8302506804466248, "sampling/importance_sampling_ratio/max": 1.3433096408843994, "sampling/importance_sampling_ratio/mean": 0.99952632188797, "sampling/importance_sampling_ratio/min": 0.6213764548301697, "sampling/sampling_logp_difference/max": 0.4758181571960449, "sampling/sampling_logp_difference/mean": 0.016288109123706818, "step": 117, "step_time": 73.19683206100308 }, { "clip_ratio/high_max": 0.025173015194013715, "clip_ratio/high_mean": 0.014006962068378925, "clip_ratio/low_mean": 0.0478188656270504, "clip_ratio/low_min": 0.018147400522138923, "clip_ratio/region_mean": 0.06182582816109061, "entropy": 0.5328387320041656, "epoch": 0.00118, "grad_norm": 0.8987992405891418, "kl": 0.56526855006814, "learning_rate": 8.363324697477484e-05, "loss": 0.2067, "step": 118, "step_time": 28.764480029007245 }, { "clip_ratio/high_max": 0.003912496555130929, "clip_ratio/high_mean": 0.0019562482775654644, "clip_ratio/low_mean": 0.006855760788312182, "clip_ratio/low_min": 0.0020833334419876337, "clip_ratio/region_mean": 0.008812009065877646, "completions/clipped_ratio": 0.0, "completions/max_length": 7676.0, "completions/max_terminated_length": 7676.0, "completions/mean_length": 6365.46875, "completions/mean_terminated_length": 6365.46875, "completions/min_length": 1100.0, "completions/min_terminated_length": 1100.0, "entropy": 0.44694129936397076, "epoch": 0.00119, "frac_reward_zero_std": 0.0, "grad_norm": 0.8498477339744568, "kl": 0.6010912526398897, "learning_rate": 8.326452439698433e-05, "loss": 0.0731, "num_tokens": 13196008.0, "reward": 0.6467089653015137, "reward_std": 0.8979178071022034, "rewards/rollout_reward_func/mean": 0.6467089653015137, "rewards/rollout_reward_func/std": 0.8902814984321594, "sampling/importance_sampling_ratio/max": 1.2894235849380493, "sampling/importance_sampling_ratio/mean": 1.0002028942108154, "sampling/importance_sampling_ratio/min": 0.5250231623649597, "sampling/sampling_logp_difference/max": 0.644312858581543, "sampling/sampling_logp_difference/mean": 0.014511598274111748, "step": 119, "step_time": 72.29536485899735 }, { "clip_ratio/high_max": 0.02890018466860056, "clip_ratio/high_mean": 0.016896430868655443, "clip_ratio/low_mean": 0.03878700197674334, "clip_ratio/low_min": 0.02219674689695239, "clip_ratio/region_mean": 0.05568343272898346, "entropy": 0.39638725109398365, "epoch": 0.0012, "grad_norm": 0.6877231597900391, "kl": 0.6096882112324238, "learning_rate": 8.289288355303245e-05, "loss": 0.0643, "step": 120, "step_time": 28.796860850994562 }, { "clip_ratio/high_max": 0.002578596060629934, "clip_ratio/high_mean": 0.001289298030314967, "clip_ratio/low_mean": 0.004018607141915709, "clip_ratio/low_min": 0.0010080644860863686, "clip_ratio/region_mean": 0.005307905172230676, "completions/clipped_ratio": 0.0, "completions/max_length": 7638.0, "completions/max_terminated_length": 7638.0, "completions/mean_length": 6142.34375, "completions/mean_terminated_length": 6142.34375, "completions/min_length": 1005.0, "completions/min_terminated_length": 1005.0, "entropy": 0.3638766473159194, "epoch": 0.00121, "frac_reward_zero_std": 0.0, "grad_norm": 1.1139280796051025, "kl": 0.668537937104702, "learning_rate": 8.251837667367966e-05, "loss": 0.3865, "num_tokens": 13417834.0, "reward": 0.5333791375160217, "reward_std": 0.9890317916870117, "rewards/rollout_reward_func/mean": 0.5333791375160217, "rewards/rollout_reward_func/std": 1.0230786800384521, "sampling/importance_sampling_ratio/max": 1.3071253299713135, "sampling/importance_sampling_ratio/mean": 0.9976951479911804, "sampling/importance_sampling_ratio/min": 0.0005499590770341456, "sampling/sampling_logp_difference/max": 7.505666732788086, "sampling/sampling_logp_difference/mean": 0.019261103123426437, "step": 121, "step_time": 72.51705048901204 }, { "clip_ratio/high_max": 0.019009737065061927, "clip_ratio/high_mean": 0.010083572356961668, "clip_ratio/low_mean": 0.022900644864421338, "clip_ratio/low_min": 0.00845568731892854, "clip_ratio/region_mean": 0.03298421751242131, "entropy": 0.3559605022892356, "epoch": 0.00122, "grad_norm": 0.5722959041595459, "kl": 0.5966317411512136, "learning_rate": 8.214105639248173e-05, "loss": 0.3712, "step": 122, "step_time": 29.955552338997222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.006080239196307957, "clip_ratio/low_min": 0.0030262937070801854, "clip_ratio/region_mean": 0.006080239196307957, "completions/clipped_ratio": 0.03125, "completions/max_length": 7776.0, "completions/max_terminated_length": 7776.0, "completions/mean_length": 6753.40625, "completions/mean_terminated_length": 6766.1611328125, "completions/min_length": 1699.0, "completions/min_terminated_length": 1699.0, "entropy": 0.3724847650155425, "epoch": 0.00123, "frac_reward_zero_std": 0.0, "grad_norm": 0.9938178658485413, "kl": 0.5677086412906647, "learning_rate": 8.176097573839265e-05, "loss": -0.0441, "num_tokens": 13659314.0, "reward": 0.6316717863082886, "reward_std": 0.8159384727478027, "rewards/rollout_reward_func/mean": 0.6316717863082886, "rewards/rollout_reward_func/std": 0.8054634928703308, "sampling/importance_sampling_ratio/max": 1.5290156602859497, "sampling/importance_sampling_ratio/mean": 1.0016262531280518, "sampling/importance_sampling_ratio/min": 0.7136997580528259, "sampling/sampling_logp_difference/max": 0.4246242046356201, "sampling/sampling_logp_difference/mean": 0.012188512831926346, "step": 123, "step_time": 74.15964120999342 }, { "clip_ratio/high_max": 0.03847794118337333, "clip_ratio/high_mean": 0.02044089382980019, "clip_ratio/low_mean": 0.03697901719715446, "clip_ratio/low_min": 0.01608842983841896, "clip_ratio/region_mean": 0.05741991149261594, "entropy": 0.3926802035421133, "epoch": 0.00124, "grad_norm": 0.8093276619911194, "kl": 0.575550040230155, "learning_rate": 8.137818812831182e-05, "loss": -0.0542, "step": 124, "step_time": 29.78917570100748 }, { "clip_ratio/high_max": 0.004402436723466963, "clip_ratio/high_mean": 0.0022012183617334813, "clip_ratio/low_mean": 0.0031128493137657642, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005314067617291585, "completions/clipped_ratio": 0.0, "completions/max_length": 7443.0, "completions/max_terminated_length": 7443.0, "completions/mean_length": 6071.1875, "completions/mean_terminated_length": 6071.1875, "completions/min_length": 942.0, "completions/min_terminated_length": 942.0, "entropy": 0.3401971282437444, "epoch": 0.00125, "frac_reward_zero_std": 0.0, "grad_norm": 1.2700196504592896, "kl": 0.5313704609870911, "learning_rate": 8.09927473595769e-05, "loss": 0.2605, "num_tokens": 13878890.0, "reward": 1.1590375900268555, "reward_std": 0.7083783745765686, "rewards/rollout_reward_func/mean": 1.1590375900268555, "rewards/rollout_reward_func/std": 0.7582368850708008, "sampling/importance_sampling_ratio/max": 1.706364631652832, "sampling/importance_sampling_ratio/mean": 1.0003165006637573, "sampling/importance_sampling_ratio/min": 0.48443490266799927, "sampling/sampling_logp_difference/max": 0.7247722148895264, "sampling/sampling_logp_difference/mean": 0.011903348378837109, "step": 125, "step_time": 70.73400082099397 }, { "clip_ratio/high_max": 0.025903045781888068, "clip_ratio/high_mean": 0.01577775034820661, "clip_ratio/low_mean": 0.033360703877406195, "clip_ratio/low_min": 0.0010080644860863686, "clip_ratio/region_mean": 0.04913845378905535, "entropy": 0.3135298155248165, "epoch": 0.00126, "grad_norm": 1.7163009643554688, "kl": 0.5671845469623804, "learning_rate": 8.060470760240294e-05, "loss": 0.2567, "step": 126, "step_time": 28.063704603002407 }, { "clip_ratio/high_max": 0.006911287433467805, "clip_ratio/high_mean": 0.0034556437167339027, "clip_ratio/low_mean": 0.0018122286710422486, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005267872387776151, "completions/clipped_ratio": 0.03125, "completions/max_length": 7447.0, "completions/max_terminated_length": 7447.0, "completions/mean_length": 5454.71875, "completions/mean_terminated_length": 5407.54833984375, "completions/min_length": 1691.0, "completions/min_terminated_length": 1691.0, "entropy": 0.3306001052260399, "epoch": 0.00127, "frac_reward_zero_std": 0.0, "grad_norm": 0.8082352876663208, "kl": 0.49354379437863827, "learning_rate": 8.021412339226936e-05, "loss": 0.3053, "num_tokens": 14078707.0, "reward": 0.7845155000686646, "reward_std": 0.8515335321426392, "rewards/rollout_reward_func/mean": 0.7845155000686646, "rewards/rollout_reward_func/std": 0.8683934807777405, "sampling/importance_sampling_ratio/max": 1.5162270069122314, "sampling/importance_sampling_ratio/mean": 0.9989938735961914, "sampling/importance_sampling_ratio/min": 0.5404072403907776, "sampling/sampling_logp_difference/max": 0.6154322624206543, "sampling/sampling_logp_difference/mean": 0.011234473437070847, "step": 127, "step_time": 67.5287541339967 }, { "clip_ratio/high_max": 0.0369661042932421, "clip_ratio/high_mean": 0.02199867705348879, "clip_ratio/low_mean": 0.027023473172448575, "clip_ratio/low_min": 0.006877367617562413, "clip_ratio/region_mean": 0.04902215034235269, "entropy": 0.3221058417111635, "epoch": 0.00128, "grad_norm": 0.7026380896568298, "kl": 0.48632543347775936, "learning_rate": 7.982104962225541e-05, "loss": 0.299, "step": 128, "step_time": 27.83653207500538 }, { "clip_ratio/high_max": 0.0066298560705035925, "clip_ratio/high_mean": 0.003915889596100897, "clip_ratio/low_mean": 0.00361568623338826, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.007531575771281496, "completions/clipped_ratio": 0.03125, "completions/max_length": 7354.0, "completions/max_terminated_length": 7354.0, "completions/mean_length": 5788.78125, "completions/mean_terminated_length": 5809.935546875, "completions/min_length": 1143.0, "completions/min_terminated_length": 1143.0, "entropy": 0.32565185613930225, "epoch": 0.00129, "frac_reward_zero_std": 0.0, "grad_norm": 0.9964798092842102, "kl": 0.5289557594805956, "learning_rate": 7.94255415353255e-05, "loss": -0.0446, "num_tokens": 14289201.0, "reward": 0.7415599226951599, "reward_std": 0.7226687669754028, "rewards/rollout_reward_func/mean": 0.7415599226951599, "rewards/rollout_reward_func/std": 0.854263424873352, "sampling/importance_sampling_ratio/max": 1.313108205795288, "sampling/importance_sampling_ratio/mean": 1.0003950595855713, "sampling/importance_sampling_ratio/min": 0.5685809850692749, "sampling/sampling_logp_difference/max": 0.564611554145813, "sampling/sampling_logp_difference/mean": 0.010765906423330307, "step": 129, "step_time": 68.3636519500069 }, { "clip_ratio/high_max": 0.03583526232978329, "clip_ratio/high_mean": 0.021514865657081828, "clip_ratio/low_mean": 0.035437036596704274, "clip_ratio/low_min": 0.008266129298135638, "clip_ratio/region_mean": 0.05695190222468227, "entropy": 0.318532669916749, "epoch": 0.0013, "grad_norm": 0.6047857999801636, "kl": 0.5551189202815294, "learning_rate": 7.902765471656524e-05, "loss": -0.0546, "step": 130, "step_time": 28.343212672003574 }, { "clip_ratio/high_max": 0.004226116579957306, "clip_ratio/high_mean": 0.002113058289978653, "clip_ratio/low_mean": 0.0018771701725199819, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003990228462498635, "completions/clipped_ratio": 0.0, "completions/max_length": 7707.0, "completions/max_terminated_length": 7707.0, "completions/mean_length": 6247.1875, "completions/mean_terminated_length": 6247.1875, "completions/min_length": 2544.0, "completions/min_terminated_length": 2544.0, "entropy": 0.33738598320633173, "epoch": 0.00131, "frac_reward_zero_std": 0.0, "grad_norm": 0.9689266681671143, "kl": 0.4614291973412037, "learning_rate": 7.862744508536953e-05, "loss": 0.2768, "num_tokens": 14514533.0, "reward": 0.8323653340339661, "reward_std": 0.7810919284820557, "rewards/rollout_reward_func/mean": 0.8323653340339661, "rewards/rollout_reward_func/std": 0.8276123404502869, "sampling/importance_sampling_ratio/max": 1.3400417566299438, "sampling/importance_sampling_ratio/mean": 1.0000840425491333, "sampling/importance_sampling_ratio/min": 0.5728639960289001, "sampling/sampling_logp_difference/max": 0.5571069717407227, "sampling/sampling_logp_difference/mean": 0.01149269100278616, "step": 131, "step_time": 71.64506284500385 }, { "clip_ratio/high_max": 0.02095413487404585, "clip_ratio/high_mean": 0.012515111127868295, "clip_ratio/low_mean": 0.030454543535597622, "clip_ratio/low_min": 0.006220353185199201, "clip_ratio/region_mean": 0.04296965501271188, "entropy": 0.3390439376235008, "epoch": 0.00132, "grad_norm": 0.6430186629295349, "kl": 0.47557700239121914, "learning_rate": 7.822496888758351e-05, "loss": 0.2624, "step": 132, "step_time": 29.539602963992365 }, { "clip_ratio/high_max": 0.010458067583385855, "clip_ratio/high_mean": 0.0052290337916929275, "clip_ratio/low_mean": 0.005533854302484542, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01076288815238513, "completions/clipped_ratio": 0.0, "completions/max_length": 7308.0, "completions/max_terminated_length": 7308.0, "completions/mean_length": 5515.4375, "completions/mean_terminated_length": 5515.4375, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.31584723200649023, "epoch": 0.00133, "frac_reward_zero_std": 0.0, "grad_norm": 0.9093613624572754, "kl": 0.5023549776524305, "learning_rate": 7.782028268759781e-05, "loss": -0.0372, "num_tokens": 14716293.0, "reward": 0.8686923980712891, "reward_std": 0.9672844409942627, "rewards/rollout_reward_func/mean": 0.8686923980712891, "rewards/rollout_reward_func/std": 0.959867537021637, "sampling/importance_sampling_ratio/max": 1.4881139993667603, "sampling/importance_sampling_ratio/mean": 0.997056245803833, "sampling/importance_sampling_ratio/min": 0.4676284193992615, "sampling/sampling_logp_difference/max": 0.7600812911987305, "sampling/sampling_logp_difference/mean": 0.012620834633708, "step": 133, "step_time": 67.52569682700414 }, { "clip_ratio/high_max": 0.04854706733021885, "clip_ratio/high_mean": 0.030595372372772545, "clip_ratio/low_mean": 0.02791402261937037, "clip_ratio/low_min": 0.0052083334885537624, "clip_ratio/region_mean": 0.05850939458468929, "entropy": 0.34442267287522554, "epoch": 0.00134, "grad_norm": 0.7646601796150208, "kl": 0.5085277669131756, "learning_rate": 7.741344336039886e-05, "loss": -0.0474, "step": 134, "step_time": 27.043865278999874 }, { "clip_ratio/high_max": 0.002234993618912995, "clip_ratio/high_mean": 0.0011174968094564974, "clip_ratio/low_mean": 0.0029555283254012465, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004073025134857744, "completions/clipped_ratio": 0.03125, "completions/max_length": 7616.0, "completions/max_terminated_length": 7616.0, "completions/mean_length": 5907.03125, "completions/mean_terminated_length": 6038.2900390625, "completions/min_length": 1838.0, "completions/min_terminated_length": 3011.0, "entropy": 0.3462471142411232, "epoch": 0.00135, "frac_reward_zero_std": 0.0, "grad_norm": 0.8879774212837219, "kl": 0.44894287176430225, "learning_rate": 7.700450808357564e-05, "loss": 0.1538, "num_tokens": 14930628.0, "reward": 0.8469696044921875, "reward_std": 0.8384706974029541, "rewards/rollout_reward_func/mean": 0.8469696044921875, "rewards/rollout_reward_func/std": 0.8586955666542053, "sampling/importance_sampling_ratio/max": 1.276516318321228, "sampling/importance_sampling_ratio/mean": 1.0005093812942505, "sampling/importance_sampling_ratio/min": 0.6518867015838623, "sampling/sampling_logp_difference/max": 0.42788445949554443, "sampling/sampling_logp_difference/mean": 0.011005260050296783, "step": 135, "step_time": 69.67023418099416 }, { "clip_ratio/high_max": 0.03133134281961247, "clip_ratio/high_mean": 0.01620446445303969, "clip_ratio/low_mean": 0.023176160117145628, "clip_ratio/low_min": 0.005357143119908869, "clip_ratio/region_mean": 0.039380624919431284, "entropy": 0.3385225599631667, "epoch": 0.00136, "grad_norm": 0.5967376828193665, "kl": 0.45589919202029705, "learning_rate": 7.659353432928393e-05, "loss": 0.1403, "step": 136, "step_time": 28.593880535005155 }, { "clip_ratio/high_max": 0.008799962582997978, "clip_ratio/high_mean": 0.004399981291498989, "clip_ratio/low_mean": 0.0013813406403642148, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005781321931863204, "completions/clipped_ratio": 0.0, "completions/max_length": 7525.0, "completions/max_terminated_length": 7525.0, "completions/mean_length": 6321.4375, "completions/mean_terminated_length": 6321.4375, "completions/min_length": 2098.0, "completions/min_terminated_length": 2098.0, "entropy": 0.37285320088267326, "epoch": 0.00137, "frac_reward_zero_std": 0.0, "grad_norm": 0.9983375668525696, "kl": 0.5240616258233786, "learning_rate": 7.618057985616908e-05, "loss": -0.0201, "num_tokens": 15158250.0, "reward": 0.4480811357498169, "reward_std": 0.657256007194519, "rewards/rollout_reward_func/mean": 0.4480811357498169, "rewards/rollout_reward_func/std": 0.7046656012535095, "sampling/importance_sampling_ratio/max": 1.2847075462341309, "sampling/importance_sampling_ratio/mean": 1.0012481212615967, "sampling/importance_sampling_ratio/min": 0.7260105609893799, "sampling/sampling_logp_difference/max": 0.3201906681060791, "sampling/sampling_logp_difference/mean": 0.013318941928446293, "step": 137, "step_time": 70.96136008300527 }, { "clip_ratio/high_max": 0.029504462843760848, "clip_ratio/high_mean": 0.015793898259289563, "clip_ratio/low_mean": 0.034958052972797304, "clip_ratio/low_min": 0.01629464339930564, "clip_ratio/region_mean": 0.05075195140670985, "entropy": 0.3441360704600811, "epoch": 0.00138, "grad_norm": 0.6168625354766846, "kl": 0.5276490319520235, "learning_rate": 7.576570270124853e-05, "loss": -0.0297, "step": 138, "step_time": 29.604033201991115 }, { "clip_ratio/high_max": 0.006114718853496015, "clip_ratio/high_mean": 0.0030573594267480075, "clip_ratio/low_mean": 0.0020497312070801854, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005107090692035854, "completions/clipped_ratio": 0.0, "completions/max_length": 7739.0, "completions/max_terminated_length": 7739.0, "completions/mean_length": 6245.46875, "completions/mean_terminated_length": 6245.46875, "completions/min_length": 2136.0, "completions/min_terminated_length": 2136.0, "entropy": 0.32586858980357647, "epoch": 0.00139, "frac_reward_zero_std": 0.0, "grad_norm": 0.9882996678352356, "kl": 0.4955175220966339, "learning_rate": 7.53489611717553e-05, "loss": 0.2145, "num_tokens": 15383456.0, "reward": 0.6962569952011108, "reward_std": 0.7316734790802002, "rewards/rollout_reward_func/mean": 0.6962569952011108, "rewards/rollout_reward_func/std": 0.810741126537323, "sampling/importance_sampling_ratio/max": 1.3529454469680786, "sampling/importance_sampling_ratio/mean": 1.0002624988555908, "sampling/importance_sampling_ratio/min": 0.4332502782344818, "sampling/sampling_logp_difference/max": 0.8364397287368774, "sampling/sampling_logp_difference/mean": 0.012437964789569378, "step": 139, "step_time": 69.7632728529934 }, { "clip_ratio/high_max": 0.02750345692038536, "clip_ratio/high_mean": 0.015277026104740798, "clip_ratio/low_mean": 0.05159119644667953, "clip_ratio/low_min": 0.020618571783415973, "clip_ratio/region_mean": 0.06686822231858969, "entropy": 0.3140969406813383, "epoch": 0.0014, "grad_norm": 0.8289701342582703, "kl": 0.5235141273587942, "learning_rate": 7.49304138369434e-05, "loss": 0.2061, "step": 140, "step_time": 29.39824160400167 }, { "clip_ratio/high_max": 0.0018382353009656072, "clip_ratio/high_mean": 0.0009191176504828036, "clip_ratio/low_mean": 0.0036122312885709107, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.004531348939053714, "completions/clipped_ratio": 0.0, "completions/max_length": 7709.0, "completions/max_terminated_length": 7709.0, "completions/mean_length": 5595.46875, "completions/mean_terminated_length": 5595.46875, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.3261274313554168, "epoch": 0.00141, "frac_reward_zero_std": 0.0, "grad_norm": 0.8228873610496521, "kl": 0.49092673417180777, "learning_rate": 7.45101195198564e-05, "loss": 0.1597, "num_tokens": 15587886.0, "reward": 0.7169510722160339, "reward_std": 0.8378109335899353, "rewards/rollout_reward_func/mean": 0.7169510722160339, "rewards/rollout_reward_func/std": 0.8184298872947693, "sampling/importance_sampling_ratio/max": 1.593092679977417, "sampling/importance_sampling_ratio/mean": 0.9996085166931152, "sampling/importance_sampling_ratio/min": 0.631175696849823, "sampling/sampling_logp_difference/max": 0.46567726135253906, "sampling/sampling_logp_difference/mean": 0.01350921019911766, "step": 141, "step_time": 70.0164830040012 }, { "clip_ratio/high_max": 0.03266194008756429, "clip_ratio/high_mean": 0.017112219997216016, "clip_ratio/low_mean": 0.03861235536169261, "clip_ratio/low_min": 0.013411458814516664, "clip_ratio/region_mean": 0.05572457564994693, "entropy": 0.31402376666665077, "epoch": 0.00142, "grad_norm": 0.7742940783500671, "kl": 0.5270266449078918, "learning_rate": 7.408813728906053e-05, "loss": 0.1517, "step": 142, "step_time": 28.786760197992407 }, { "clip_ratio/high_max": 0.0035714287078008056, "clip_ratio/high_mean": 0.0017857143539004028, "clip_ratio/low_mean": 0.005538754659937695, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0073244690138380975, "completions/clipped_ratio": 0.0, "completions/max_length": 7626.0, "completions/max_terminated_length": 7626.0, "completions/mean_length": 6188.5, "completions/mean_terminated_length": 6188.5, "completions/min_length": 1468.0, "completions/min_terminated_length": 1468.0, "entropy": 0.29089806228876114, "epoch": 0.00143, "frac_reward_zero_std": 0.0, "grad_norm": 0.9141284227371216, "kl": 0.5138957593590021, "learning_rate": 7.366452645034293e-05, "loss": 0.2343, "num_tokens": 15811235.0, "reward": 0.7642644643783569, "reward_std": 0.7893732190132141, "rewards/rollout_reward_func/mean": 0.7642644643783569, "rewards/rollout_reward_func/std": 0.802289605140686, "sampling/importance_sampling_ratio/max": 1.6691169738769531, "sampling/importance_sampling_ratio/mean": 1.0009257793426514, "sampling/importance_sampling_ratio/min": 0.6748902797698975, "sampling/sampling_logp_difference/max": 0.5122947692871094, "sampling/sampling_logp_difference/mean": 0.013666888698935509, "step": 143, "step_time": 72.21533830900808 }, { "clip_ratio/high_max": 0.028732813196256757, "clip_ratio/high_mean": 0.016009034705348313, "clip_ratio/low_mean": 0.03939049592008814, "clip_ratio/low_min": 0.01478174643125385, "clip_ratio/region_mean": 0.05539953080005944, "entropy": 0.27141018118709326, "epoch": 0.00144, "grad_norm": 0.7495607733726501, "kl": 0.5547631531953812, "learning_rate": 7.32393465383769e-05, "loss": 0.2277, "step": 144, "step_time": 28.97441118698771 }, { "clip_ratio/high_max": 0.004058659775182605, "clip_ratio/high_mean": 0.0020293298875913024, "clip_ratio/low_mean": 0.0024314599577337503, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004460789845325053, "completions/clipped_ratio": 0.0, "completions/max_length": 7514.0, "completions/max_terminated_length": 7514.0, "completions/mean_length": 5912.46875, "completions/mean_terminated_length": 5912.46875, "completions/min_length": 1740.0, "completions/min_terminated_length": 1740.0, "entropy": 0.28330227453261614, "epoch": 0.00145, "frac_reward_zero_std": 0.0, "grad_norm": 0.9225184321403503, "kl": 0.5672691380605102, "learning_rate": 7.281265730835482e-05, "loss": 0.0358, "num_tokens": 16025722.0, "reward": 0.7273563146591187, "reward_std": 0.827768087387085, "rewards/rollout_reward_func/mean": 0.7273563146591187, "rewards/rollout_reward_func/std": 0.8488644361495972, "sampling/importance_sampling_ratio/max": 1.2558616399765015, "sampling/importance_sampling_ratio/mean": 0.9987654685974121, "sampling/importance_sampling_ratio/min": 0.6893196105957031, "sampling/sampling_logp_difference/max": 0.37205028533935547, "sampling/sampling_logp_difference/mean": 0.011272422969341278, "step": 145, "step_time": 68.02180921599938 }, { "clip_ratio/high_max": 0.03880192746873945, "clip_ratio/high_mean": 0.020878117240499705, "clip_ratio/low_mean": 0.031095593963982537, "clip_ratio/low_min": 0.014693304081447423, "clip_ratio/region_mean": 0.051973711437312886, "entropy": 0.28110581636428833, "epoch": 0.00146, "grad_norm": 0.6973389387130737, "kl": 0.5735289147123694, "learning_rate": 7.238451872759005e-05, "loss": 0.0283, "step": 146, "step_time": 28.319236400006048 }, { "clip_ratio/high_max": 0.008203125325962901, "clip_ratio/high_mean": 0.0041015626629814506, "clip_ratio/low_mean": 0.002241847920231521, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.006343410583212972, "completions/clipped_ratio": 0.0, "completions/max_length": 7636.0, "completions/max_terminated_length": 7636.0, "completions/mean_length": 6364.5625, "completions/mean_terminated_length": 6364.5625, "completions/min_length": 1850.0, "completions/min_terminated_length": 1850.0, "entropy": 0.29459603875875473, "epoch": 0.00147, "frac_reward_zero_std": 0.0, "grad_norm": 0.9231361746788025, "kl": 0.49721979536116123, "learning_rate": 7.195499096708908e-05, "loss": 0.0797, "num_tokens": 16254725.0, "reward": 0.4217974543571472, "reward_std": 1.1589643955230713, "rewards/rollout_reward_func/mean": 0.4217974543571472, "rewards/rollout_reward_func/std": 1.3444899320602417, "sampling/importance_sampling_ratio/max": 1.3134464025497437, "sampling/importance_sampling_ratio/mean": 0.9986108541488647, "sampling/importance_sampling_ratio/min": 0.7232284545898438, "sampling/sampling_logp_difference/max": 0.32403016090393066, "sampling/sampling_logp_difference/mean": 0.014123032800853252, "step": 147, "step_time": 72.42693970400433 }, { "clip_ratio/high_max": 0.025631660828366876, "clip_ratio/high_mean": 0.017242913716472685, "clip_ratio/low_mean": 0.03189136309083551, "clip_ratio/low_min": 0.016134876175783575, "clip_ratio/region_mean": 0.0491342768073082, "entropy": 0.28126809652894735, "epoch": 0.00148, "grad_norm": 0.5397176146507263, "kl": 0.5006464179605246, "learning_rate": 7.152413439309507e-05, "loss": 0.0672, "step": 148, "step_time": 28.85056770799565 }, { "clip_ratio/high_max": 0.003979700966738164, "clip_ratio/high_mean": 0.001989850483369082, "clip_ratio/low_mean": 0.005208333604969084, "clip_ratio/low_min": 0.0020833334419876337, "clip_ratio/region_mean": 0.0071981840883381665, "completions/clipped_ratio": 0.0, "completions/max_length": 7348.0, "completions/max_terminated_length": 7348.0, "completions/mean_length": 6015.6875, "completions/mean_terminated_length": 6015.6875, "completions/min_length": 880.0, "completions/min_terminated_length": 880.0, "entropy": 0.23935529962182045, "epoch": 0.00149, "frac_reward_zero_std": 0.0, "grad_norm": 0.9824679493904114, "kl": 0.7582256607711315, "learning_rate": 7.109200955860391e-05, "loss": 0.1286, "num_tokens": 16472526.0, "reward": 0.9733051061630249, "reward_std": 0.8397026658058167, "rewards/rollout_reward_func/mean": 0.9733051061630249, "rewards/rollout_reward_func/std": 0.8470094203948975, "sampling/importance_sampling_ratio/max": 1.3819187879562378, "sampling/importance_sampling_ratio/mean": 1.000042200088501, "sampling/importance_sampling_ratio/min": 0.7275699377059937, "sampling/sampling_logp_difference/max": 0.3234729766845703, "sampling/sampling_logp_difference/mean": 0.011042654514312744, "step": 149, "step_time": 70.02410554599555 }, { "clip_ratio/high_max": 0.023193919798359275, "clip_ratio/high_mean": 0.01438713859533891, "clip_ratio/low_mean": 0.02135416737291962, "clip_ratio/low_min": 0.008333333767950535, "clip_ratio/region_mean": 0.03574130591005087, "entropy": 0.24100138247013092, "epoch": 0.0015, "grad_norm": 0.7320052981376648, "kl": 0.5946865044534206, "learning_rate": 7.065867719485405e-05, "loss": 0.1188, "step": 150, "step_time": 27.489905474001716 }, { "clip_ratio/high_max": 0.0039502165745943785, "clip_ratio/high_mean": 0.0019751082872971892, "clip_ratio/low_mean": 0.0020665323245339096, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004041640611831099, "completions/clipped_ratio": 0.0, "completions/max_length": 7426.0, "completions/max_terminated_length": 7426.0, "completions/mean_length": 6180.90625, "completions/mean_terminated_length": 6180.90625, "completions/min_length": 1219.0, "completions/min_terminated_length": 1219.0, "entropy": 0.26912419125437737, "epoch": 0.00151, "frac_reward_zero_std": 0.0, "grad_norm": 0.8326308727264404, "kl": 0.533622158691287, "learning_rate": 7.022419820279119e-05, "loss": 0.1611, "num_tokens": 16695583.0, "reward": 0.7767990231513977, "reward_std": 0.7000998854637146, "rewards/rollout_reward_func/mean": 0.7767990231513977, "rewards/rollout_reward_func/std": 0.6950080394744873, "sampling/importance_sampling_ratio/max": 1.3788485527038574, "sampling/importance_sampling_ratio/mean": 0.9994899034500122, "sampling/importance_sampling_ratio/min": 0.6952024698257446, "sampling/sampling_logp_difference/max": 0.3635520935058594, "sampling/sampling_logp_difference/mean": 0.01406054012477398, "step": 151, "step_time": 69.23069016999943 }, { "clip_ratio/high_max": 0.029578342917375267, "clip_ratio/high_mean": 0.015310004877392203, "clip_ratio/low_mean": 0.020830729219596833, "clip_ratio/low_min": 0.004099462414160371, "clip_ratio/region_mean": 0.036140734155196697, "entropy": 0.27803486306220293, "epoch": 0.00152, "grad_norm": 0.6658856868743896, "kl": 0.5457711387425661, "learning_rate": 6.978863364450934e-05, "loss": 0.1518, "step": 152, "step_time": 28.74678311099342 }, { "clip_ratio/high_max": 0.006119791883975267, "clip_ratio/high_mean": 0.0030598959419876337, "clip_ratio/low_mean": 0.005351788946427405, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.008411684888415039, "completions/clipped_ratio": 0.0, "completions/max_length": 7724.0, "completions/max_terminated_length": 7724.0, "completions/mean_length": 6350.90625, "completions/mean_terminated_length": 6350.90625, "completions/min_length": 2573.0, "completions/min_terminated_length": 2573.0, "entropy": 0.29807998798787594, "epoch": 0.00153, "frac_reward_zero_std": 0.0, "grad_norm": 0.8597056269645691, "kl": 0.5509033557027578, "learning_rate": 6.935204473466904e-05, "loss": 0.1474, "num_tokens": 16924150.0, "reward": 0.3234422504901886, "reward_std": 0.7605398893356323, "rewards/rollout_reward_func/mean": 0.3234422504901886, "rewards/rollout_reward_func/std": 0.8037971258163452, "sampling/importance_sampling_ratio/max": 1.389380931854248, "sampling/importance_sampling_ratio/mean": 1.000666856765747, "sampling/importance_sampling_ratio/min": 0.6106446385383606, "sampling/sampling_logp_difference/max": 0.4932401180267334, "sampling/sampling_logp_difference/mean": 0.014613812789320946, "step": 153, "step_time": 73.08823051700892 }, { "clip_ratio/high_max": 0.02814065630082041, "clip_ratio/high_mean": 0.0160885572549887, "clip_ratio/low_mean": 0.034217736625578254, "clip_ratio/low_min": 0.014583333861082792, "clip_ratio/region_mean": 0.050306293822359294, "entropy": 0.30299864523112774, "epoch": 0.00154, "grad_norm": 0.6016488671302795, "kl": 0.5590750798583031, "learning_rate": 6.891449283189408e-05, "loss": 0.1385, "step": 154, "step_time": 29.6156636170017 }, { "clip_ratio/high_max": 0.007074652938172221, "clip_ratio/high_mean": 0.0035373264690861106, "clip_ratio/low_mean": 0.0015625000814907253, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005099826608784497, "completions/clipped_ratio": 0.0, "completions/max_length": 7692.0, "completions/max_terminated_length": 7692.0, "completions/mean_length": 6332.84375, "completions/mean_terminated_length": 6332.84375, "completions/min_length": 2368.0, "completions/min_terminated_length": 2368.0, "entropy": 0.2881121728569269, "epoch": 0.00155, "frac_reward_zero_std": 0.0, "grad_norm": 0.9251049757003784, "kl": 0.623315941542387, "learning_rate": 6.847603943014831e-05, "loss": 0.1741, "num_tokens": 17152096.0, "reward": 0.5484405755996704, "reward_std": 0.7380176186561584, "rewards/rollout_reward_func/mean": 0.5484405755996704, "rewards/rollout_reward_func/std": 0.7993680238723755, "sampling/importance_sampling_ratio/max": 1.2862424850463867, "sampling/importance_sampling_ratio/mean": 0.998192310333252, "sampling/importance_sampling_ratio/min": 0.5312603712081909, "sampling/sampling_logp_difference/max": 0.6325030326843262, "sampling/sampling_logp_difference/mean": 0.01470865961164236, "step": 155, "step_time": 71.35737086301015 }, { "clip_ratio/high_max": 0.04081530566327274, "clip_ratio/high_mean": 0.02040765283163637, "clip_ratio/low_mean": 0.023363096232060343, "clip_ratio/low_min": 0.0020833334419876337, "clip_ratio/region_mean": 0.043770749180112034, "entropy": 0.2862389935180545, "epoch": 0.00156, "grad_norm": 0.6376045346260071, "kl": 0.6118534859269857, "learning_rate": 6.803674615009306e-05, "loss": 0.1645, "step": 156, "step_time": 28.640151496005274 }, { "clip_ratio/high_max": 0.0015625000232830644, "clip_ratio/high_mean": 0.0007812500116415322, "clip_ratio/low_mean": 0.005803571664728224, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0065848216763697565, "completions/clipped_ratio": 0.0, "completions/max_length": 7738.0, "completions/max_terminated_length": 7738.0, "completions/mean_length": 5755.34375, "completions/mean_terminated_length": 5755.34375, "completions/min_length": 853.0, "completions/min_terminated_length": 853.0, "entropy": 0.27548809815198183, "epoch": 0.00157, "frac_reward_zero_std": 0.0, "grad_norm": 0.9188845753669739, "kl": 0.5965822078287601, "learning_rate": 6.759667473042693e-05, "loss": 0.0735, "num_tokens": 17361596.0, "reward": 0.6325401663780212, "reward_std": 0.7946115732192993, "rewards/rollout_reward_func/mean": 0.6325401663780212, "rewards/rollout_reward_func/std": 0.7904440760612488, "sampling/importance_sampling_ratio/max": 1.312840223312378, "sampling/importance_sampling_ratio/mean": 0.9986227750778198, "sampling/importance_sampling_ratio/min": 0.689022958278656, "sampling/sampling_logp_difference/max": 0.3724806308746338, "sampling/sampling_logp_difference/mean": 0.014214975759387016, "step": 157, "step_time": 70.59402103300454 }, { "clip_ratio/high_max": 0.034199481597170234, "clip_ratio/high_mean": 0.017620574100874364, "clip_ratio/low_mean": 0.03187600488308817, "clip_ratio/low_min": 0.013257576269097626, "clip_ratio/region_mean": 0.04949657875113189, "entropy": 0.2730365050956607, "epoch": 0.00158, "grad_norm": 0.5060749650001526, "kl": 0.62849629111588, "learning_rate": 6.71558870192091e-05, "loss": 0.0634, "step": 158, "step_time": 28.775418558994716 }, { "clip_ratio/high_max": 0.0027777778450399637, "clip_ratio/high_mean": 0.0013888889225199819, "clip_ratio/low_mean": 0.0031081990455277264, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.004497087968047708, "completions/clipped_ratio": 0.0, "completions/max_length": 7859.0, "completions/max_terminated_length": 7859.0, "completions/mean_length": 5964.53125, "completions/mean_terminated_length": 5964.53125, "completions/min_length": 665.0, "completions/min_terminated_length": 665.0, "entropy": 0.2799002369865775, "epoch": 0.00159, "frac_reward_zero_std": 0.0, "grad_norm": 0.8523491024971008, "kl": 0.6272502318024635, "learning_rate": 6.671444496516697e-05, "loss": -0.0236, "num_tokens": 17577800.0, "reward": 0.6606459617614746, "reward_std": 0.9057881832122803, "rewards/rollout_reward_func/mean": 0.6606459617614746, "rewards/rollout_reward_func/std": 0.8971738815307617, "sampling/importance_sampling_ratio/max": 1.4213851690292358, "sampling/importance_sampling_ratio/mean": 1.0007035732269287, "sampling/importance_sampling_ratio/min": 0.7810884714126587, "sampling/sampling_logp_difference/max": 0.35163187980651855, "sampling/sampling_logp_difference/mean": 0.013490458950400352, "step": 159, "step_time": 69.8240715100037 }, { "clip_ratio/high_max": 0.03540253220126033, "clip_ratio/high_mean": 0.01844531379174441, "clip_ratio/low_mean": 0.020762905420269817, "clip_ratio/low_min": 0.0072244624607264996, "clip_ratio/region_mean": 0.03920821927022189, "entropy": 0.2861761711537838, "epoch": 0.0016, "grad_norm": 0.7416832447052002, "kl": 0.6190724149346352, "learning_rate": 6.627241060898992e-05, "loss": -0.0338, "step": 160, "step_time": 29.33544054599406 }, { "clip_ratio/high_max": 0.0032407408580183983, "clip_ratio/high_mean": 0.0016203704290091991, "clip_ratio/low_mean": 0.003645833523478359, "clip_ratio/low_min": 0.0020833334419876337, "clip_ratio/region_mean": 0.005266204010695219, "completions/clipped_ratio": 0.0, "completions/max_length": 7612.0, "completions/max_terminated_length": 7612.0, "completions/mean_length": 6328.1875, "completions/mean_terminated_length": 6328.1875, "completions/min_length": 2070.0, "completions/min_terminated_length": 2070.0, "entropy": 0.30483807157725096, "epoch": 0.00161, "frac_reward_zero_std": 0.0, "grad_norm": 1.0490154027938843, "kl": 0.6775335241109133, "learning_rate": 6.582984607461005e-05, "loss": 0.0245, "num_tokens": 17805669.0, "reward": 0.5039433240890503, "reward_std": 0.933875560760498, "rewards/rollout_reward_func/mean": 0.5039433240890503, "rewards/rollout_reward_func/std": 0.917387068271637, "sampling/importance_sampling_ratio/max": 1.4645049571990967, "sampling/importance_sampling_ratio/mean": 0.9998276233673096, "sampling/importance_sampling_ratio/min": 0.7015359401702881, "sampling/sampling_logp_difference/max": 0.38151729106903076, "sampling/sampling_logp_difference/mean": 0.014585288241505623, "step": 161, "step_time": 69.48021378600242 }, { "clip_ratio/high_max": 0.04808438045438379, "clip_ratio/high_mean": 0.027353202051017433, "clip_ratio/low_mean": 0.038303779903799295, "clip_ratio/low_min": 0.01969086076132953, "clip_ratio/region_mean": 0.06565698212943971, "entropy": 0.30937677901238203, "epoch": 0.00162, "grad_norm": 1.1206525564193726, "kl": 0.8011088538914919, "learning_rate": 6.538681356047126e-05, "loss": 0.0203, "step": 162, "step_time": 29.011947536011576 }, { "clip_ratio/high_max": 0.005066724261268973, "clip_ratio/high_mean": 0.0025333621306344867, "clip_ratio/low_mean": 0.00679270934779197, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.009326071478426456, "completions/clipped_ratio": 0.0, "completions/max_length": 7431.0, "completions/max_terminated_length": 7431.0, "completions/mean_length": 6034.9375, "completions/mean_terminated_length": 6034.9375, "completions/min_length": 1067.0, "completions/min_terminated_length": 1067.0, "entropy": 0.29086963180452585, "epoch": 0.00163, "frac_reward_zero_std": 0.0, "grad_norm": 0.9237681031227112, "kl": 0.5985952522605658, "learning_rate": 6.494337533078768e-05, "loss": 0.1025, "num_tokens": 18024118.0, "reward": 0.6223878860473633, "reward_std": 0.7501951456069946, "rewards/rollout_reward_func/mean": 0.6223878860473633, "rewards/rollout_reward_func/std": 0.7293554544448853, "sampling/importance_sampling_ratio/max": 1.4762824773788452, "sampling/importance_sampling_ratio/mean": 1.0020084381103516, "sampling/importance_sampling_ratio/min": 0.6562693119049072, "sampling/sampling_logp_difference/max": 0.42118406295776367, "sampling/sampling_logp_difference/mean": 0.014804542064666748, "step": 163, "step_time": 69.07197276299848 }, { "clip_ratio/high_max": 0.02471592219080776, "clip_ratio/high_mean": 0.01235796109540388, "clip_ratio/low_mean": 0.03379677759949118, "clip_ratio/low_min": 0.010416667209938169, "clip_ratio/region_mean": 0.04615473881131038, "entropy": 0.28300073370337486, "epoch": 0.00164, "grad_norm": 0.7105070948600769, "kl": 0.6013880036771297, "learning_rate": 6.449959370679315e-05, "loss": 0.0925, "step": 164, "step_time": 27.784895152995887 }, { "clip_ratio/high_max": 0.006360324448905885, "clip_ratio/high_mean": 0.0031801622244529426, "clip_ratio/low_mean": 0.004687500186264515, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007867662410717458, "completions/clipped_ratio": 0.0, "completions/max_length": 7566.0, "completions/max_terminated_length": 7566.0, "completions/mean_length": 5741.40625, "completions/mean_terminated_length": 5741.40625, "completions/min_length": 661.0, "completions/min_terminated_length": 661.0, "entropy": 0.28069377038627863, "epoch": 0.00165, "frac_reward_zero_std": 0.0, "grad_norm": 0.8547999858856201, "kl": 0.546027634292841, "learning_rate": 6.40555310579825e-05, "loss": 0.0727, "num_tokens": 18233180.0, "reward": 0.7879390120506287, "reward_std": 0.8327975869178772, "rewards/rollout_reward_func/mean": 0.7879390120506287, "rewards/rollout_reward_func/std": 0.8136789202690125, "sampling/importance_sampling_ratio/max": 1.43551766872406, "sampling/importance_sampling_ratio/mean": 0.9994142055511475, "sampling/importance_sampling_ratio/min": 0.4435466229915619, "sampling/sampling_logp_difference/max": 0.8129523992538452, "sampling/sampling_logp_difference/mean": 0.013429960235953331, "step": 165, "step_time": 67.35486821599989 }, { "clip_ratio/high_max": 0.021107278647832572, "clip_ratio/high_mean": 0.013027597859036177, "clip_ratio/low_mean": 0.02430555672617629, "clip_ratio/low_min": 0.008333333767950535, "clip_ratio/region_mean": 0.03733315458521247, "entropy": 0.27624416071921587, "epoch": 0.00166, "grad_norm": 0.5641725659370422, "kl": 0.522529436275363, "learning_rate": 6.3611249793346e-05, "loss": 0.0633, "step": 166, "step_time": 28.600335343009647 }, { "clip_ratio/high_max": 0.0024621213087812066, "clip_ratio/high_mean": 0.0012310606543906033, "clip_ratio/low_mean": 0.0031250002211891115, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.004356060875579715, "completions/clipped_ratio": 0.0, "completions/max_length": 7563.0, "completions/max_terminated_length": 7563.0, "completions/mean_length": 6258.15625, "completions/mean_terminated_length": 6258.15625, "completions/min_length": 2334.0, "completions/min_terminated_length": 2334.0, "entropy": 0.26805614586919546, "epoch": 0.00167, "frac_reward_zero_std": 0.0, "grad_norm": 0.994601309299469, "kl": 0.5861261282116175, "learning_rate": 6.316681235259841e-05, "loss": 0.0892, "num_tokens": 18458753.0, "reward": 0.8967909812927246, "reward_std": 0.7031261920928955, "rewards/rollout_reward_func/mean": 0.8967909812927246, "rewards/rollout_reward_func/std": 0.7807169556617737, "sampling/importance_sampling_ratio/max": 1.390047311782837, "sampling/importance_sampling_ratio/mean": 0.9995607137680054, "sampling/importance_sampling_ratio/min": 0.5164026021957397, "sampling/sampling_logp_difference/max": 0.6608686447143555, "sampling/sampling_logp_difference/mean": 0.012276836670935154, "step": 167, "step_time": 71.96837405599581 }, { "clip_ratio/high_max": 0.02419637725688517, "clip_ratio/high_mean": 0.013518643216229975, "clip_ratio/low_mean": 0.015087366045918316, "clip_ratio/low_min": 0.0031250001629814506, "clip_ratio/region_mean": 0.028606009087525308, "entropy": 0.2717358134686947, "epoch": 0.00168, "grad_norm": 0.570343554019928, "kl": 0.5793642234057188, "learning_rate": 6.272228119740365e-05, "loss": 0.0794, "step": 168, "step_time": 28.98830055399958 }, { "clip_ratio/high_max": 0.00424107164144516, "clip_ratio/high_mean": 0.00212053582072258, "clip_ratio/low_mean": 0.006196829839609563, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.008317365660332143, "completions/clipped_ratio": 0.0, "completions/max_length": 7434.0, "completions/max_terminated_length": 7434.0, "completions/mean_length": 5708.5, "completions/mean_terminated_length": 5708.5, "completions/min_length": 1426.0, "completions/min_terminated_length": 1426.0, "entropy": 0.2691904818639159, "epoch": 0.00169, "frac_reward_zero_std": 0.0, "grad_norm": 0.9576648473739624, "kl": 0.5605732705444098, "learning_rate": 6.227771880259637e-05, "loss": 0.0467, "num_tokens": 18666687.0, "reward": 0.7240573167800903, "reward_std": 0.908635139465332, "rewards/rollout_reward_func/mean": 0.7240573167800903, "rewards/rollout_reward_func/std": 0.8919249773025513, "sampling/importance_sampling_ratio/max": 1.3410836458206177, "sampling/importance_sampling_ratio/mean": 0.9990981817245483, "sampling/importance_sampling_ratio/min": 0.5677777528762817, "sampling/sampling_logp_difference/max": 0.5660252571105957, "sampling/sampling_logp_difference/mean": 0.014057480730116367, "step": 169, "step_time": 69.4165788779901 }, { "clip_ratio/high_max": 0.028829088958445936, "clip_ratio/high_mean": 0.016072117170551792, "clip_ratio/low_mean": 0.03149133158149198, "clip_ratio/low_min": 0.010416667093522847, "clip_ratio/region_mean": 0.04756344889756292, "entropy": 0.2652217363938689, "epoch": 0.0017, "grad_norm": 0.8454261422157288, "kl": 0.5829532584175467, "learning_rate": 6.183318764740161e-05, "loss": 0.0362, "step": 170, "step_time": 27.761449992998678 }, { "clip_ratio/high_max": 0.006866056472063065, "clip_ratio/high_mean": 0.005265366402454674, "clip_ratio/low_mean": 0.002587365685030818, "clip_ratio/low_min": 0.0010080644860863686, "clip_ratio/region_mean": 0.007852732087485492, "completions/clipped_ratio": 0.0, "completions/max_length": 7247.0, "completions/max_terminated_length": 7247.0, "completions/mean_length": 5939.71875, "completions/mean_terminated_length": 5939.71875, "completions/min_length": 3103.0, "completions/min_terminated_length": 3103.0, "entropy": 0.28063530940562487, "epoch": 0.00171, "frac_reward_zero_std": 0.0, "grad_norm": 0.9780874252319336, "kl": 0.5503423009067774, "learning_rate": 6.138875020665402e-05, "loss": 0.1914, "num_tokens": 18882018.0, "reward": 1.0371155738830566, "reward_std": 0.7871289253234863, "rewards/rollout_reward_func/mean": 1.0371155738830566, "rewards/rollout_reward_func/std": 0.7574010491371155, "sampling/importance_sampling_ratio/max": 1.255274772644043, "sampling/importance_sampling_ratio/mean": 0.9985589385032654, "sampling/importance_sampling_ratio/min": 0.7150009870529175, "sampling/sampling_logp_difference/max": 0.33547139167785645, "sampling/sampling_logp_difference/mean": 0.014519904740154743, "step": 171, "step_time": 68.46184607100076 }, { "clip_ratio/high_max": 0.034278102801181376, "clip_ratio/high_mean": 0.020379792083986104, "clip_ratio/low_mean": 0.031115592049900442, "clip_ratio/low_min": 0.007291666930541396, "clip_ratio/region_mean": 0.05149538366822526, "entropy": 0.2781274449080229, "epoch": 0.00172, "grad_norm": 0.6253865361213684, "kl": 0.5425704941153526, "learning_rate": 6.0944468942017506e-05, "loss": 0.1774, "step": 172, "step_time": 26.846779451996554 }, { "clip_ratio/high_max": 0.004486013087444007, "clip_ratio/high_mean": 0.0022430065437220037, "clip_ratio/low_mean": 0.005729166965465993, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.007972173392772675, "completions/clipped_ratio": 0.0, "completions/max_length": 7605.0, "completions/max_terminated_length": 7605.0, "completions/mean_length": 6002.125, "completions/mean_terminated_length": 6002.125, "completions/min_length": 724.0, "completions/min_terminated_length": 724.0, "entropy": 0.27004879992455244, "epoch": 0.00173, "frac_reward_zero_std": 0.0, "grad_norm": 1.2557798624038696, "kl": 1.0267800018191338, "learning_rate": 6.050040629320685e-05, "loss": -0.0453, "num_tokens": 19099407.0, "reward": 0.6001454591751099, "reward_std": 0.934731125831604, "rewards/rollout_reward_func/mean": 0.6001454591751099, "rewards/rollout_reward_func/std": 0.9008826017379761, "sampling/importance_sampling_ratio/max": 1.4542118310928345, "sampling/importance_sampling_ratio/mean": 0.9997845888137817, "sampling/importance_sampling_ratio/min": 0.5840144157409668, "sampling/sampling_logp_difference/max": 0.5378295183181763, "sampling/sampling_logp_difference/mean": 0.013949964195489883, "step": 173, "step_time": 71.66625639000631 }, { "clip_ratio/high_max": 0.04564255801960826, "clip_ratio/high_mean": 0.023342112312093377, "clip_ratio/low_mean": 0.016356170235667378, "clip_ratio/low_min": 0.0020833334419876337, "clip_ratio/region_mean": 0.039698282547760755, "entropy": 0.28857745230197906, "epoch": 0.00174, "grad_norm": 0.7947912812232971, "kl": 0.6493016052991152, "learning_rate": 6.0056624669212335e-05, "loss": -0.0562, "step": 174, "step_time": 29.18363615700582 }, { "clip_ratio/high_max": 0.010533417691476643, "clip_ratio/high_mean": 0.00578754220623523, "clip_ratio/low_mean": 0.005338541930541396, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011126084194984287, "completions/clipped_ratio": 0.0, "completions/max_length": 7680.0, "completions/max_terminated_length": 7680.0, "completions/mean_length": 6049.59375, "completions/mean_terminated_length": 6049.59375, "completions/min_length": 2501.0, "completions/min_terminated_length": 2501.0, "entropy": 0.3034949544817209, "epoch": 0.00175, "frac_reward_zero_std": 0.0, "grad_norm": 1.1590583324432373, "kl": 0.6270948369055986, "learning_rate": 5.961318643952876e-05, "loss": 0.1315, "num_tokens": 19318324.0, "reward": 0.708774209022522, "reward_std": 0.705255925655365, "rewards/rollout_reward_func/mean": 0.708774209022522, "rewards/rollout_reward_func/std": 0.8596416115760803, "sampling/importance_sampling_ratio/max": 1.284186840057373, "sampling/importance_sampling_ratio/mean": 0.9993244409561157, "sampling/importance_sampling_ratio/min": 0.677297055721283, "sampling/sampling_logp_difference/max": 0.3896453380584717, "sampling/sampling_logp_difference/mean": 0.016125477850437164, "step": 175, "step_time": 72.21724637100124 }, { "clip_ratio/high_max": 0.05774388450663537, "clip_ratio/high_mean": 0.035284611803945154, "clip_ratio/low_mean": 0.048567709629423916, "clip_ratio/low_min": 0.015625000465661287, "clip_ratio/region_mean": 0.08385232114233077, "entropy": 0.27816114015877247, "epoch": 0.00176, "grad_norm": 1.0890446901321411, "kl": 0.7165942899882793, "learning_rate": 5.917015392538995e-05, "loss": 0.1264, "step": 176, "step_time": 29.499630223002896 }, { "clip_ratio/high_max": 0.00241815485060215, "clip_ratio/high_mean": 0.001209077425301075, "clip_ratio/low_mean": 0.0020833334419876337, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032924108672887087, "completions/clipped_ratio": 0.0, "completions/max_length": 7648.0, "completions/max_terminated_length": 7648.0, "completions/mean_length": 6062.65625, "completions/mean_terminated_length": 6062.65625, "completions/min_length": 1688.0, "completions/min_terminated_length": 1688.0, "entropy": 0.2841997453942895, "epoch": 0.00177, "frac_reward_zero_std": 0.0, "grad_norm": 0.811324954032898, "kl": 0.5354560874402523, "learning_rate": 5.872758939101011e-05, "loss": 0.2319, "num_tokens": 19537598.0, "reward": 0.8880721926689148, "reward_std": 0.7332305908203125, "rewards/rollout_reward_func/mean": 0.8880721926689148, "rewards/rollout_reward_func/std": 0.7316773533821106, "sampling/importance_sampling_ratio/max": 1.3328274488449097, "sampling/importance_sampling_ratio/mean": 0.9995454549789429, "sampling/importance_sampling_ratio/min": 0.7428115010261536, "sampling/sampling_logp_difference/max": 0.29731297492980957, "sampling/sampling_logp_difference/mean": 0.013784060254693031, "step": 177, "step_time": 70.70282067000153 }, { "clip_ratio/high_max": 0.04166789213195443, "clip_ratio/high_mean": 0.02208394603803754, "clip_ratio/low_mean": 0.01822916738456115, "clip_ratio/low_min": 0.0031250001629814506, "clip_ratio/region_mean": 0.04031311359722167, "entropy": 0.3112387331202626, "epoch": 0.00178, "grad_norm": 0.9581050872802734, "kl": 0.49249773286283016, "learning_rate": 5.828555503483305e-05, "loss": 0.2274, "step": 178, "step_time": 28.874133450000954 }, { "clip_ratio/high_max": 0.007784179062582552, "clip_ratio/high_mean": 0.004412922891788185, "clip_ratio/low_mean": 0.003757440601475537, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008170363551471382, "completions/clipped_ratio": 0.0, "completions/max_length": 7598.0, "completions/max_terminated_length": 7598.0, "completions/mean_length": 6360.53125, "completions/mean_terminated_length": 6360.53125, "completions/min_length": 692.0, "completions/min_terminated_length": 692.0, "entropy": 0.30882556922733784, "epoch": 0.00179, "frac_reward_zero_std": 0.0, "grad_norm": 1.170659065246582, "kl": 0.5295849964022636, "learning_rate": 5.784411298079091e-05, "loss": 0.4692, "num_tokens": 19766534.0, "reward": 0.6715573072433472, "reward_std": 0.7578749656677246, "rewards/rollout_reward_func/mean": 0.6715573072433472, "rewards/rollout_reward_func/std": 0.8563293218612671, "sampling/importance_sampling_ratio/max": 1.504078984260559, "sampling/importance_sampling_ratio/mean": 0.9988590478897095, "sampling/importance_sampling_ratio/min": 0.5416574478149414, "sampling/sampling_logp_difference/max": 0.613121509552002, "sampling/sampling_logp_difference/mean": 0.015492187812924385, "step": 179, "step_time": 72.7305891329961 }, { "clip_ratio/high_max": 0.03115731629077345, "clip_ratio/high_mean": 0.018313032982405275, "clip_ratio/low_mean": 0.0324878747924231, "clip_ratio/low_min": 0.0177479253616184, "clip_ratio/region_mean": 0.05080090824048966, "entropy": 0.29126913379877806, "epoch": 0.0018, "grad_norm": 0.7735331654548645, "kl": 0.550991365686059, "learning_rate": 5.740332526957307e-05, "loss": 0.464, "step": 180, "step_time": 28.72574914699726 }, { "clip_ratio/high_max": 0.010051407385617495, "clip_ratio/high_mean": 0.005025703692808747, "clip_ratio/low_mean": 0.0019097222830168903, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006935425975825638, "completions/clipped_ratio": 0.0, "completions/max_length": 7421.0, "completions/max_terminated_length": 7421.0, "completions/mean_length": 5659.875, "completions/mean_terminated_length": 5659.875, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.3196950303390622, "epoch": 0.00181, "frac_reward_zero_std": 0.0, "grad_norm": 1.0407955646514893, "kl": 0.5521001368761063, "learning_rate": 5.696325384990696e-05, "loss": -0.1261, "num_tokens": 19972922.0, "reward": 0.4764799475669861, "reward_std": 0.7681980133056641, "rewards/rollout_reward_func/mean": 0.4764799475669861, "rewards/rollout_reward_func/std": 0.7579792141914368, "sampling/importance_sampling_ratio/max": 1.5037927627563477, "sampling/importance_sampling_ratio/mean": 1.0015438795089722, "sampling/importance_sampling_ratio/min": 0.6809799075126648, "sampling/sampling_logp_difference/max": 0.4079904556274414, "sampling/sampling_logp_difference/mean": 0.015923358500003815, "step": 181, "step_time": 69.72042241999588 }, { "clip_ratio/high_max": 0.027349640266038477, "clip_ratio/high_mean": 0.014716486912220716, "clip_ratio/low_mean": 0.03972279216395691, "clip_ratio/low_min": 0.0031250000465661287, "clip_ratio/region_mean": 0.05443927925080061, "entropy": 0.2953004348091781, "epoch": 0.00182, "grad_norm": 0.5870159864425659, "kl": 0.6095700291916728, "learning_rate": 5.6523960569851695e-05, "loss": -0.1335, "step": 182, "step_time": 28.29985440000746 }, { "clip_ratio/high_max": 0.01590735709760338, "clip_ratio/high_mean": 0.00795367854880169, "clip_ratio/low_mean": 0.006216397974640131, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014170076523441821, "completions/clipped_ratio": 0.0, "completions/max_length": 7706.0, "completions/max_terminated_length": 7706.0, "completions/mean_length": 6120.40625, "completions/mean_terminated_length": 6120.40625, "completions/min_length": 1032.0, "completions/min_terminated_length": 1032.0, "entropy": 0.24601693032309413, "epoch": 0.00183, "frac_reward_zero_std": 0.0, "grad_norm": 0.9645286202430725, "kl": 0.5330642554908991, "learning_rate": 5.6085507168105936e-05, "loss": 0.1724, "num_tokens": 20194155.0, "reward": 0.877181887626648, "reward_std": 0.9203190803527832, "rewards/rollout_reward_func/mean": 0.877181887626648, "rewards/rollout_reward_func/std": 0.9252534508705139, "sampling/importance_sampling_ratio/max": 1.3765305280685425, "sampling/importance_sampling_ratio/mean": 0.9986370801925659, "sampling/importance_sampling_ratio/min": 0.7135393619537354, "sampling/sampling_logp_difference/max": 0.33751773834228516, "sampling/sampling_logp_difference/mean": 0.013944385573267937, "step": 183, "step_time": 68.9665804849974 }, { "clip_ratio/high_max": 0.03462804667651653, "clip_ratio/high_mean": 0.019898157916031778, "clip_ratio/low_mean": 0.02600806555710733, "clip_ratio/low_min": 0.009375000256113708, "clip_ratio/region_mean": 0.04590622376417741, "entropy": 0.2408104445785284, "epoch": 0.00184, "grad_norm": 0.7852590084075928, "kl": 0.5228715240955353, "learning_rate": 5.5647955265330974e-05, "loss": 0.1636, "step": 184, "step_time": 29.392674487990007 }, { "clip_ratio/high_max": 0.004898403189145029, "clip_ratio/high_mean": 0.0034908682573586702, "clip_ratio/low_mean": 0.0031250001629814506, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00661586836213246, "completions/clipped_ratio": 0.0, "completions/max_length": 7668.0, "completions/max_terminated_length": 7668.0, "completions/mean_length": 6003.40625, "completions/mean_terminated_length": 6003.40625, "completions/min_length": 686.0, "completions/min_terminated_length": 686.0, "entropy": 0.2209624256938696, "epoch": 0.00185, "frac_reward_zero_std": 0.0, "grad_norm": 1.004656434059143, "kl": 0.5534517057240009, "learning_rate": 5.5211366355490666e-05, "loss": -0.1851, "num_tokens": 20411612.0, "reward": 0.37751665711402893, "reward_std": 0.8726356625556946, "rewards/rollout_reward_func/mean": 0.37751665711402893, "rewards/rollout_reward_func/std": 0.9877573251724243, "sampling/importance_sampling_ratio/max": 1.2858830690383911, "sampling/importance_sampling_ratio/mean": 1.0000656843185425, "sampling/importance_sampling_ratio/min": 0.508312463760376, "sampling/sampling_logp_difference/max": 0.6766588687896729, "sampling/sampling_logp_difference/mean": 0.012145766988396645, "step": 185, "step_time": 70.45575987099437 }, { "clip_ratio/high_max": 0.04339219001121819, "clip_ratio/high_mean": 0.025341928470879793, "clip_ratio/low_mean": 0.0316860334132798, "clip_ratio/low_min": 0.017136270878836513, "clip_ratio/region_mean": 0.05702796170953661, "entropy": 0.23074535839259624, "epoch": 0.00186, "grad_norm": 0.6589720249176025, "kl": 0.5275174546986818, "learning_rate": 5.4775801797208824e-05, "loss": -0.1918, "step": 186, "step_time": 29.320655040999554 }, { "clip_ratio/high_max": 0.0058858084958046675, "clip_ratio/high_mean": 0.003567904233932495, "clip_ratio/low_mean": 0.0052905703778378665, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008858474669978023, "completions/clipped_ratio": 0.0, "completions/max_length": 7645.0, "completions/max_terminated_length": 7645.0, "completions/mean_length": 6466.25, "completions/mean_terminated_length": 6466.25, "completions/min_length": 1790.0, "completions/min_terminated_length": 1790.0, "entropy": 0.25866161845624447, "epoch": 0.00187, "frac_reward_zero_std": 0.0, "grad_norm": 1.0847647190093994, "kl": 0.47890415973961353, "learning_rate": 5.434132280514597e-05, "loss": 0.1125, "num_tokens": 20643899.0, "reward": 0.645482063293457, "reward_std": 0.8381398320198059, "rewards/rollout_reward_func/mean": 0.645482063293457, "rewards/rollout_reward_func/std": 0.8086071014404297, "sampling/importance_sampling_ratio/max": 1.5297152996063232, "sampling/importance_sampling_ratio/mean": 0.9987109303474426, "sampling/importance_sampling_ratio/min": 0.5598245859146118, "sampling/sampling_logp_difference/max": 0.5801317691802979, "sampling/sampling_logp_difference/mean": 0.013672111555933952, "step": 187, "step_time": 70.79536088500026 }, { "clip_ratio/high_max": 0.028969057835638523, "clip_ratio/high_mean": 0.018026195699349046, "clip_ratio/low_mean": 0.026771816366817802, "clip_ratio/low_min": 0.01659946294967085, "clip_ratio/region_mean": 0.04479801212437451, "entropy": 0.25937927328050137, "epoch": 0.00188, "grad_norm": 0.627483069896698, "kl": 0.47550554014742374, "learning_rate": 5.39079904413961e-05, "loss": 0.104, "step": 188, "step_time": 28.858083520000946 }, { "clip_ratio/high_max": 0.01187304116319865, "clip_ratio/high_mean": 0.007052592060063034, "clip_ratio/low_mean": 0.006754032452590764, "clip_ratio/low_min": 0.0020497312070801854, "clip_ratio/region_mean": 0.01380662462906912, "completions/clipped_ratio": 0.0, "completions/max_length": 7706.0, "completions/max_terminated_length": 7706.0, "completions/mean_length": 6484.65625, "completions/mean_terminated_length": 6484.65625, "completions/min_length": 1833.0, "completions/min_terminated_length": 1833.0, "entropy": 0.2494481299072504, "epoch": 0.00189, "frac_reward_zero_std": 0.0, "grad_norm": 0.8301334381103516, "kl": 0.47373599000275135, "learning_rate": 5.347586560690494e-05, "loss": 0.0563, "num_tokens": 20876766.0, "reward": 0.4350264072418213, "reward_std": 0.6912021040916443, "rewards/rollout_reward_func/mean": 0.4350264072418213, "rewards/rollout_reward_func/std": 0.7497764229774475, "sampling/importance_sampling_ratio/max": 1.540845513343811, "sampling/importance_sampling_ratio/mean": 0.9991546869277954, "sampling/importance_sampling_ratio/min": 0.6135069131851196, "sampling/sampling_logp_difference/max": 0.48856377601623535, "sampling/sampling_logp_difference/mean": 0.013833342120051384, "step": 189, "step_time": 73.73829232699791 }, { "clip_ratio/high_max": 0.030814534868113697, "clip_ratio/high_mean": 0.017099975782912225, "clip_ratio/low_mean": 0.020989125652704388, "clip_ratio/low_min": 0.006250000209547579, "clip_ratio/region_mean": 0.03808910143561661, "entropy": 0.24590318091213703, "epoch": 0.0019, "grad_norm": 0.7487269043922424, "kl": 0.4668748192489147, "learning_rate": 5.304500903291094e-05, "loss": 0.0534, "step": 190, "step_time": 29.55292062499211 }, { "clip_ratio/high_max": 0.014036744949407876, "clip_ratio/high_mean": 0.007018372474703938, "clip_ratio/low_mean": 0.0031250001629814506, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010143372579477727, "completions/clipped_ratio": 0.0, "completions/max_length": 7614.0, "completions/max_terminated_length": 7614.0, "completions/mean_length": 6102.625, "completions/mean_terminated_length": 6102.625, "completions/min_length": 1048.0, "completions/min_terminated_length": 1048.0, "entropy": 0.23831991758197546, "epoch": 0.00191, "frac_reward_zero_std": 0.0, "grad_norm": 1.1395350694656372, "kl": 0.7575500924140215, "learning_rate": 5.261548127240997e-05, "loss": 0.1598, "num_tokens": 21097415.0, "reward": 0.8718339800834656, "reward_std": 0.7443748712539673, "rewards/rollout_reward_func/mean": 0.8718339800834656, "rewards/rollout_reward_func/std": 0.8685494065284729, "sampling/importance_sampling_ratio/max": 1.7140424251556396, "sampling/importance_sampling_ratio/mean": 0.9987540245056152, "sampling/importance_sampling_ratio/min": 0.599521815776825, "sampling/sampling_logp_difference/max": 0.5388545989990234, "sampling/sampling_logp_difference/mean": 0.015344742685556412, "step": 191, "step_time": 71.47624410400022 }, { "clip_ratio/high_max": 0.04486134205944836, "clip_ratio/high_mean": 0.027018010965548456, "clip_ratio/low_mean": 0.019618056481704116, "clip_ratio/low_min": 0.007986111450009048, "clip_ratio/region_mean": 0.046636067098006606, "entropy": 0.24816366471350193, "epoch": 0.00192, "grad_norm": 0.7564718127250671, "kl": 0.6349804196506739, "learning_rate": 5.218734269164519e-05, "loss": 0.1509, "step": 192, "step_time": 28.939137056000618 }, { "clip_ratio/high_max": 0.007143640425056219, "clip_ratio/high_mean": 0.0035718202125281096, "clip_ratio/low_mean": 0.0031081990455277264, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006680019258055836, "completions/clipped_ratio": 0.0, "completions/max_length": 7668.0, "completions/max_terminated_length": 7668.0, "completions/mean_length": 6179.1875, "completions/mean_terminated_length": 6179.1875, "completions/min_length": 3385.0, "completions/min_terminated_length": 3385.0, "entropy": 0.2629840597510338, "epoch": 0.00193, "frac_reward_zero_std": 0.0, "grad_norm": 1.4336174726486206, "kl": 0.5205130334943533, "learning_rate": 5.176065346162311e-05, "loss": 0.2042, "num_tokens": 21320486.0, "reward": 1.0318678617477417, "reward_std": 0.7901626825332642, "rewards/rollout_reward_func/mean": 1.0318678617477417, "rewards/rollout_reward_func/std": 0.7859672904014587, "sampling/importance_sampling_ratio/max": 1.2904080152511597, "sampling/importance_sampling_ratio/mean": 1.0004006624221802, "sampling/importance_sampling_ratio/min": 0.7252420783042908, "sampling/sampling_logp_difference/max": 0.32124972343444824, "sampling/sampling_logp_difference/mean": 0.01481054350733757, "step": 193, "step_time": 71.46769104299892 }, { "clip_ratio/high_max": 0.027688406757079065, "clip_ratio/high_mean": 0.01458825106965378, "clip_ratio/low_mean": 0.04250672156922519, "clip_ratio/low_min": 0.010416667209938169, "clip_ratio/region_mean": 0.05709497298812494, "entropy": 0.25064905919134617, "epoch": 0.00194, "grad_norm": 1.285180687904358, "kl": 0.5832781717181206, "learning_rate": 5.1335473549657084e-05, "loss": 0.1943, "step": 194, "step_time": 28.59637527200539 }, { "clip_ratio/high_max": 0.0034262193366885185, "clip_ratio/high_mean": 0.0022339430288411677, "clip_ratio/low_mean": 0.004166666883975267, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006400609971024096, "completions/clipped_ratio": 0.0, "completions/max_length": 7624.0, "completions/max_terminated_length": 7624.0, "completions/mean_length": 6338.78125, "completions/mean_terminated_length": 6338.78125, "completions/min_length": 3043.0, "completions/min_terminated_length": 3043.0, "entropy": 0.26590211875736713, "epoch": 0.00195, "frac_reward_zero_std": 0.0, "grad_norm": 0.9436479806900024, "kl": 0.46912239491939545, "learning_rate": 5.0911862710939485e-05, "loss": 0.1109, "num_tokens": 21548636.0, "reward": 0.685299277305603, "reward_std": 0.8237334489822388, "rewards/rollout_reward_func/mean": 0.685299277305603, "rewards/rollout_reward_func/std": 0.8433352112770081, "sampling/importance_sampling_ratio/max": 1.4059780836105347, "sampling/importance_sampling_ratio/mean": 0.9996352195739746, "sampling/importance_sampling_ratio/min": 0.6509221792221069, "sampling/sampling_logp_difference/max": 0.4293651580810547, "sampling/sampling_logp_difference/mean": 0.013887550681829453, "step": 195, "step_time": 72.383208223986 } ], "logging_steps": 1.0, "max_steps": 300, "num_input_tokens_seen": 21548636, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }