{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.0165, "eval_steps": 500, "global_step": 825, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 15.90625, "completions/mean_terminated_length": 15.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1101458184421062, "epoch": 2e-05, "frac_reward_zero_std": 0.75, "grad_norm": 0.35523131489753723, "kl": 0.0, "learning_rate": 0.0, "loss": 0.0014, "num_tokens": 44024.0, "reward": -0.9368749856948853, "reward_std": 0.17854446172714233, "rewards/rollout_reward_func/mean": -0.9368749856948853, "rewards/rollout_reward_func/std": 0.3570888936519623, "sampling/importance_sampling_ratio/max": 1.4553872346878052, "sampling/importance_sampling_ratio/mean": 1.0262402296066284, "sampling/importance_sampling_ratio/min": 0.839340090751648, "sampling/sampling_logp_difference/max": 0.37515878677368164, "sampling/sampling_logp_difference/mean": 0.045643799006938934, "step": 1, "step_time": 11.735762504002196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2012700140476227, "epoch": 4e-05, "frac_reward_zero_std": 0.75, "grad_norm": 0.4595509469509125, "kl": 0.0, "learning_rate": 2.8571428571428575e-07, "loss": 0.0001, "num_tokens": 88505.0, "reward": -0.9368749856948853, "reward_std": 0.17854446172714233, "rewards/rollout_reward_func/mean": -0.9368749856948853, "rewards/rollout_reward_func/std": 0.3570888936519623, "sampling/importance_sampling_ratio/max": 1.2831617593765259, "sampling/importance_sampling_ratio/mean": 0.9785476922988892, "sampling/importance_sampling_ratio/min": 0.7469597458839417, "sampling/sampling_logp_difference/max": 0.29154348373413086, "sampling/sampling_logp_difference/mean": 0.04732952266931534, "step": 2, "step_time": 10.081924004996836 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 411.0, "completions/max_terminated_length": 411.0, "completions/mean_length": 14.78125, "completions/mean_terminated_length": 14.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.09858962520957, "epoch": 6e-05, "frac_reward_zero_std": 0.5, "grad_norm": 0.8724910020828247, "kl": 0.003875432536005974, "learning_rate": 5.714285714285715e-07, "loss": 0.0005, "num_tokens": 131377.0, "reward": -0.875, "reward_std": 0.3535533845424652, "rewards/rollout_reward_func/mean": -0.875, "rewards/rollout_reward_func/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3007621765136719, "sampling/importance_sampling_ratio/mean": 1.0253186225891113, "sampling/importance_sampling_ratio/min": 0.8196067214012146, "sampling/sampling_logp_difference/max": 0.2629384994506836, "sampling/sampling_logp_difference/mean": 0.041732046753168106, "step": 3, "step_time": 11.071516190000693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1617677714675665, "epoch": 8e-05, "frac_reward_zero_std": 1.0, "grad_norm": 0.0013095509493723512, "kl": 0.002941606566309929, "learning_rate": 8.571428571428572e-07, "loss": 0.0, "num_tokens": 174939.0, "reward": -1.0, "reward_std": 0.0, "rewards/rollout_reward_func/mean": -1.0, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.256412386894226, "sampling/importance_sampling_ratio/mean": 0.9781372547149658, "sampling/importance_sampling_ratio/min": 0.7918862700462341, "sampling/sampling_logp_difference/max": 0.23333168029785156, "sampling/sampling_logp_difference/mean": 0.04448595643043518, "step": 4, "step_time": 9.609989450997091 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.0, "completions/max_terminated_length": 294.0, "completions/mean_length": 22.0625, "completions/mean_terminated_length": 22.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1588657312095165, "epoch": 0.0001, "frac_reward_zero_std": 0.75, "grad_norm": 0.46203890442848206, "kl": 0.0021913163363933563, "learning_rate": 1.142857142857143e-06, "loss": -0.0001, "num_tokens": 219297.0, "reward": -0.9368749856948853, "reward_std": 0.17854446172714233, "rewards/rollout_reward_func/mean": -0.9368749856948853, "rewards/rollout_reward_func/std": 0.3570888936519623, "sampling/importance_sampling_ratio/max": 1.2034889459609985, "sampling/importance_sampling_ratio/mean": 1.0040606260299683, "sampling/importance_sampling_ratio/min": 0.7002345323562622, "sampling/sampling_logp_difference/max": 0.3564877510070801, "sampling/sampling_logp_difference/mean": 0.05030624568462372, "step": 5, "step_time": 11.22134409200953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.0, "completions/max_terminated_length": 294.0, "completions/mean_length": 11.125, "completions/mean_terminated_length": 11.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.161049485206604, "epoch": 0.00012, "frac_reward_zero_std": 1.0, "grad_norm": 0.0011111816857010126, "kl": 0.0021760454401373863, "learning_rate": 1.4285714285714286e-06, "loss": 0.0, "num_tokens": 263330.0, "reward": -1.0, "reward_std": 0.0, "rewards/rollout_reward_func/mean": -1.0, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2927371263504028, "sampling/importance_sampling_ratio/mean": 1.0184967517852783, "sampling/importance_sampling_ratio/min": 0.807214081287384, "sampling/sampling_logp_difference/max": 0.2567458152770996, "sampling/sampling_logp_difference/mean": 0.03831445425748825, "step": 6, "step_time": 10.691580012018676 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.126266971230507, "epoch": 0.00014, "frac_reward_zero_std": 1.0, "grad_norm": 0.0016149363946169615, "kl": 0.0025528715923428535, "learning_rate": 1.7142857142857145e-06, "loss": 0.0, "num_tokens": 306588.0, "reward": -1.0, "reward_std": 0.0, "rewards/rollout_reward_func/mean": -1.0, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5316190719604492, "sampling/importance_sampling_ratio/mean": 1.0121041536331177, "sampling/importance_sampling_ratio/min": 0.8342507481575012, "sampling/sampling_logp_difference/max": 0.4263129234313965, "sampling/sampling_logp_difference/mean": 0.04608137905597687, "step": 7, "step_time": 9.583956740993017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.148651011288166, "epoch": 0.00016, "frac_reward_zero_std": 1.0, "grad_norm": 0.0007359020528383553, "kl": 0.0022049425169825554, "learning_rate": 2.0000000000000003e-06, "loss": 0.0, "num_tokens": 350667.0, "reward": -1.0, "reward_std": 0.0, "rewards/rollout_reward_func/mean": -1.0, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.1606074571609497, "sampling/importance_sampling_ratio/mean": 0.9889074563980103, "sampling/importance_sampling_ratio/min": 0.87240070104599, "sampling/sampling_logp_difference/max": 0.14896917343139648, "sampling/sampling_logp_difference/mean": 0.03619501739740372, "step": 8, "step_time": 10.590144608984701 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1300547197461128, "epoch": 0.00018, "frac_reward_zero_std": 0.75, "grad_norm": 0.6199297904968262, "kl": 0.0030557047575712204, "learning_rate": 2.285714285714286e-06, "loss": -0.0026, "num_tokens": 392655.0, "reward": -0.8737499713897705, "reward_std": 0.23376956582069397, "rewards/rollout_reward_func/mean": -0.8737499713897705, "rewards/rollout_reward_func/std": 0.49678805470466614, "sampling/importance_sampling_ratio/max": 1.284065842628479, "sampling/importance_sampling_ratio/mean": 0.991536557674408, "sampling/importance_sampling_ratio/min": 0.7786229252815247, "sampling/sampling_logp_difference/max": 0.2504079341888428, "sampling/sampling_logp_difference/mean": 0.048865385353565216, "step": 9, "step_time": 9.659036624987493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1467384323477745, "epoch": 0.0002, "frac_reward_zero_std": 0.5, "grad_norm": 0.8426623940467834, "kl": 0.002311648800969124, "learning_rate": 2.571428571428571e-06, "loss": -0.0002, "num_tokens": 435541.0, "reward": -0.7475000023841858, "reward_std": 0.46753913164138794, "rewards/rollout_reward_func/mean": -0.7475000023841858, "rewards/rollout_reward_func/std": 0.6787416338920593, "sampling/importance_sampling_ratio/max": 1.2496191263198853, "sampling/importance_sampling_ratio/mean": 0.9841924905776978, "sampling/importance_sampling_ratio/min": 0.8210349082946777, "sampling/sampling_logp_difference/max": 0.22287797927856445, "sampling/sampling_logp_difference/mean": 0.033375613391399384, "step": 10, "step_time": 9.762223204008478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.0, "completions/max_terminated_length": 267.0, "completions/mean_length": 10.28125, "completions/mean_terminated_length": 10.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1633161827921867, "epoch": 0.00022, "frac_reward_zero_std": 1.0, "grad_norm": 0.000980835291557014, "kl": 0.0017784330993890762, "learning_rate": 2.8571428571428573e-06, "loss": 0.0, "num_tokens": 479696.0, "reward": -1.0, "reward_std": 0.0, "rewards/rollout_reward_func/mean": -1.0, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.20090913772583, "sampling/importance_sampling_ratio/mean": 0.9752998352050781, "sampling/importance_sampling_ratio/min": 0.7116908431053162, "sampling/sampling_logp_difference/max": 0.33983612060546875, "sampling/sampling_logp_difference/mean": 0.04168141260743141, "step": 11, "step_time": 11.680648902001849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 3.0, "completions/max_terminated_length": 3.0, "completions/mean_length": 2.03125, "completions/mean_terminated_length": 2.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1417035348713398, "epoch": 0.00024, "frac_reward_zero_std": 0.75, "grad_norm": 0.3536112904548645, "kl": 0.003446099039138062, "learning_rate": 3.142857142857143e-06, "loss": -0.0009, "num_tokens": 522204.0, "reward": -0.9368749856948853, "reward_std": 0.17854446172714233, "rewards/rollout_reward_func/mean": -0.9368749856948853, "rewards/rollout_reward_func/std": 0.3570888936519623, "sampling/importance_sampling_ratio/max": 1.2331899404525757, "sampling/importance_sampling_ratio/mean": 1.0035505294799805, "sampling/importance_sampling_ratio/min": 0.854897677898407, "sampling/sampling_logp_difference/max": 0.20957517623901367, "sampling/sampling_logp_difference/mean": 0.039066851139068604, "step": 12, "step_time": 9.797874875999696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 4.0, "completions/max_terminated_length": 4.0, "completions/mean_length": 2.0625, "completions/mean_terminated_length": 2.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1536507681012154, "epoch": 0.00026, "frac_reward_zero_std": 0.5, "grad_norm": 0.6397106051445007, "kl": 0.0033251410350203514, "learning_rate": 3.428571428571429e-06, "loss": 0.0012, "num_tokens": 566322.0, "reward": -0.8737499713897705, "reward_std": 0.35708892345428467, "rewards/rollout_reward_func/mean": -0.8737499713897705, "rewards/rollout_reward_func/std": 0.49678805470466614, "sampling/importance_sampling_ratio/max": 1.1669573783874512, "sampling/importance_sampling_ratio/mean": 0.9994304180145264, "sampling/importance_sampling_ratio/min": 0.7875416278839111, "sampling/sampling_logp_difference/max": 0.3974822759628296, "sampling/sampling_logp_difference/mean": 0.04457356780767441, "step": 13, "step_time": 9.809096927005157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 16.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.4375, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1349763460457325, "epoch": 0.00028, "frac_reward_zero_std": 0.5, "grad_norm": 1.3391581773757935, "kl": 0.003850462380796671, "learning_rate": 3.7142857142857146e-06, "loss": -0.0768, "num_tokens": 608502.0, "reward": -0.8737499713897705, "reward_std": 0.35708892345428467, "rewards/rollout_reward_func/mean": -0.8737499713897705, "rewards/rollout_reward_func/std": 0.49678805470466614, "sampling/importance_sampling_ratio/max": 1.2580833435058594, "sampling/importance_sampling_ratio/mean": 0.985958456993103, "sampling/importance_sampling_ratio/min": 0.7541505098342896, "sampling/sampling_logp_difference/max": 0.2821319103240967, "sampling/sampling_logp_difference/mean": 0.03971162438392639, "step": 14, "step_time": 10.725230993004516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 19.28125, "completions/mean_terminated_length": 19.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1142122745513916, "epoch": 0.0003, "frac_reward_zero_std": 0.75, "grad_norm": 0.3625663220882416, "kl": 0.006376393139362335, "learning_rate": 4.000000000000001e-06, "loss": -0.001, "num_tokens": 651761.0, "reward": -0.9368749856948853, "reward_std": 0.17854446172714233, "rewards/rollout_reward_func/mean": -0.9368749856948853, "rewards/rollout_reward_func/std": 0.3570888936519623, "sampling/importance_sampling_ratio/max": 1.1801482439041138, "sampling/importance_sampling_ratio/mean": 0.9660115838050842, "sampling/importance_sampling_ratio/min": 0.5928254127502441, "sampling/sampling_logp_difference/max": 0.5228145122528076, "sampling/sampling_logp_difference/mean": 0.05384121462702751, "step": 15, "step_time": 10.94547809499636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1735516637563705, "epoch": 0.00032, "frac_reward_zero_std": 0.5, "grad_norm": 0.7990303039550781, "kl": 0.011390226893126965, "learning_rate": 4.2857142857142855e-06, "loss": -0.0023, "num_tokens": 695845.0, "reward": -0.8106249570846558, "reward_std": 0.4123140275478363, "rewards/rollout_reward_func/mean": -0.8106249570846558, "rewards/rollout_reward_func/std": 0.5982120633125305, "sampling/importance_sampling_ratio/max": 1.500235915184021, "sampling/importance_sampling_ratio/mean": 1.0338366031646729, "sampling/importance_sampling_ratio/min": 0.8844273686408997, "sampling/sampling_logp_difference/max": 0.40564846992492676, "sampling/sampling_logp_difference/mean": 0.034501366317272186, "step": 16, "step_time": 9.545480836015486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 402.0, "completions/max_terminated_length": 402.0, "completions/mean_length": 14.5, "completions/mean_terminated_length": 14.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1844899877905846, "epoch": 0.00034, "frac_reward_zero_std": 0.5, "grad_norm": 1.1911578178405762, "kl": 0.015045786276459694, "learning_rate": 4.571428571428572e-06, "loss": -0.0111, "num_tokens": 739735.0, "reward": -0.7475000023841858, "reward_std": 0.46753913164138794, "rewards/rollout_reward_func/mean": -0.7475000023841858, "rewards/rollout_reward_func/std": 0.6787416338920593, "sampling/importance_sampling_ratio/max": 1.520254373550415, "sampling/importance_sampling_ratio/mean": 1.0293159484863281, "sampling/importance_sampling_ratio/min": 0.7751217484474182, "sampling/sampling_logp_difference/max": 0.4188828468322754, "sampling/sampling_logp_difference/mean": 0.06347618997097015, "step": 17, "step_time": 12.23414546600543 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.23242624476552, "epoch": 0.00036, "frac_reward_zero_std": 0.75, "grad_norm": 0.5176194310188293, "kl": 0.02058567199856043, "learning_rate": 4.857142857142858e-06, "loss": -0.0037, "num_tokens": 783973.0, "reward": -0.8737499713897705, "reward_std": 0.23376956582069397, "rewards/rollout_reward_func/mean": -0.8737499713897705, "rewards/rollout_reward_func/std": 0.49678805470466614, "sampling/importance_sampling_ratio/max": 1.4311583042144775, "sampling/importance_sampling_ratio/mean": 1.011650562286377, "sampling/importance_sampling_ratio/min": 0.7112131118774414, "sampling/sampling_logp_difference/max": 0.35846519470214844, "sampling/sampling_logp_difference/mean": 0.06897427886724472, "step": 18, "step_time": 9.663292413017189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1699943877756596, "epoch": 0.00038, "frac_reward_zero_std": 0.5, "grad_norm": 0.7768184542655945, "kl": 0.0399687048047781, "learning_rate": 5.142857142857142e-06, "loss": -0.0072, "num_tokens": 827715.0, "reward": -0.8112499713897705, "reward_std": 0.41116058826446533, "rewards/rollout_reward_func/mean": -0.8112499713897705, "rewards/rollout_reward_func/std": 0.5962449908256531, "sampling/importance_sampling_ratio/max": 1.392366647720337, "sampling/importance_sampling_ratio/mean": 1.0167264938354492, "sampling/importance_sampling_ratio/min": 0.6343091726303101, "sampling/sampling_logp_difference/max": 0.4550929069519043, "sampling/sampling_logp_difference/mean": 0.07035521417856216, "step": 19, "step_time": 9.467836144016474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 17.59375, "completions/mean_terminated_length": 17.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.139320019632578, "epoch": 0.0004, "frac_reward_zero_std": 0.0, "grad_norm": 1.7802847623825073, "kl": 0.09812878724187613, "learning_rate": 5.428571428571429e-06, "loss": -0.0248, "num_tokens": 871581.0, "reward": -0.559374988079071, "reward_std": 0.8769319653511047, "rewards/rollout_reward_func/mean": -0.559374988079071, "rewards/rollout_reward_func/std": 0.8460379838943481, "sampling/importance_sampling_ratio/max": 1.6023554801940918, "sampling/importance_sampling_ratio/mean": 1.0182526111602783, "sampling/importance_sampling_ratio/min": 0.703539252281189, "sampling/sampling_logp_difference/max": 0.47137153148651123, "sampling/sampling_logp_difference/mean": 0.08875077962875366, "step": 20, "step_time": 11.765440013012267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0721238888800144, "epoch": 0.00042, "frac_reward_zero_std": 0.25, "grad_norm": 1.031890630722046, "kl": 0.125044671818614, "learning_rate": 5.7142857142857145e-06, "loss": -0.0182, "num_tokens": 914885.0, "reward": -0.7475000023841858, "reward_std": 0.5908584594726562, "rewards/rollout_reward_func/mean": -0.7475000023841858, "rewards/rollout_reward_func/std": 0.6787416338920593, "sampling/importance_sampling_ratio/max": 1.706307291984558, "sampling/importance_sampling_ratio/mean": 0.9325054883956909, "sampling/importance_sampling_ratio/min": 0.4459281861782074, "sampling/sampling_logp_difference/max": 0.8076398372650146, "sampling/sampling_logp_difference/mean": 0.11276228725910187, "step": 21, "step_time": 9.62706638299278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1248116791248322, "epoch": 0.00044, "frac_reward_zero_std": 0.0, "grad_norm": 1.4247305393218994, "kl": 0.1778357094153762, "learning_rate": 6e-06, "loss": -0.0223, "num_tokens": 958775.0, "reward": -0.4325000047683716, "reward_std": 0.9153542518615723, "rewards/rollout_reward_func/mean": -0.4325000047683716, "rewards/rollout_reward_func/std": 0.9217339158058167, "sampling/importance_sampling_ratio/max": 1.966462254524231, "sampling/importance_sampling_ratio/mean": 1.1923273801803589, "sampling/importance_sampling_ratio/min": 0.6375418901443481, "sampling/sampling_logp_difference/max": 0.6761385202407837, "sampling/sampling_logp_difference/mean": 0.1340634971857071, "step": 22, "step_time": 9.77283102999354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0454005189239979, "epoch": 0.00046, "frac_reward_zero_std": 0.75, "grad_norm": 0.23615962266921997, "kl": 0.24954266473650932, "learning_rate": 6.285714285714286e-06, "loss": -0.003, "num_tokens": 1002629.0, "reward": -0.8737499713897705, "reward_std": 0.23376956582069397, "rewards/rollout_reward_func/mean": -0.8737499713897705, "rewards/rollout_reward_func/std": 0.49678805470466614, "sampling/importance_sampling_ratio/max": 1.5052162408828735, "sampling/importance_sampling_ratio/mean": 0.9249570369720459, "sampling/importance_sampling_ratio/min": 0.31692978739738464, "sampling/sampling_logp_difference/max": 1.148982286453247, "sampling/sampling_logp_difference/mean": 0.14720873534679413, "step": 23, "step_time": 10.54044373101351 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9169419910758734, "epoch": 0.00048, "frac_reward_zero_std": 0.25, "grad_norm": 0.39553141593933105, "kl": 0.5002009924501181, "learning_rate": 6.571428571428572e-06, "loss": -0.0138, "num_tokens": 1046262.0, "reward": -0.6212500333786011, "reward_std": 0.6736763715744019, "rewards/rollout_reward_func/mean": -0.6212500333786011, "rewards/rollout_reward_func/std": 0.8010466694831848, "sampling/importance_sampling_ratio/max": 1.5551542043685913, "sampling/importance_sampling_ratio/mean": 1.007424235343933, "sampling/importance_sampling_ratio/min": 0.29642626643180847, "sampling/sampling_logp_difference/max": 1.215821623802185, "sampling/sampling_logp_difference/mean": 0.1617252230644226, "step": 24, "step_time": 9.780042327001865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0625, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8262157365679741, "epoch": 0.0005, "frac_reward_zero_std": 0.0, "grad_norm": 0.6912961006164551, "kl": 0.48302776739001274, "learning_rate": 6.857142857142858e-06, "loss": -0.0304, "num_tokens": 1088625.0, "reward": -0.17937500774860382, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.303211212158203, "sampling/importance_sampling_ratio/mean": 1.1171104907989502, "sampling/importance_sampling_ratio/min": 0.32578226923942566, "sampling/sampling_logp_difference/max": 1.1214426755905151, "sampling/sampling_logp_difference/mean": 0.15045098960399628, "step": 25, "step_time": 9.779521933014621 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0389237366616726, "epoch": 0.00052, "frac_reward_zero_std": 0.25, "grad_norm": 0.31334877014160156, "kl": 0.5306602641940117, "learning_rate": 7.1428571428571436e-06, "loss": -0.0085, "num_tokens": 1133079.0, "reward": -0.6843750476837158, "reward_std": 0.6460835933685303, "rewards/rollout_reward_func/mean": -0.6843750476837158, "rewards/rollout_reward_func/std": 0.7451820969581604, "sampling/importance_sampling_ratio/max": 1.5233147144317627, "sampling/importance_sampling_ratio/mean": 1.0102949142456055, "sampling/importance_sampling_ratio/min": 0.2470434308052063, "sampling/sampling_logp_difference/max": 1.397956132888794, "sampling/sampling_logp_difference/mean": 0.14086109399795532, "step": 26, "step_time": 10.697294077013794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 21.53125, "completions/mean_terminated_length": 21.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.772206268273294, "epoch": 0.00054, "frac_reward_zero_std": 0.0, "grad_norm": 0.45416679978370667, "kl": 0.995326355099678, "learning_rate": 7.428571428571429e-06, "loss": -0.0143, "num_tokens": 1176364.0, "reward": -0.4318750202655792, "reward_std": 0.8522208333015442, "rewards/rollout_reward_func/mean": -0.4318750202655792, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.5411627292633057, "sampling/importance_sampling_ratio/mean": 0.9506964087486267, "sampling/importance_sampling_ratio/min": 0.17998860776424408, "sampling/sampling_logp_difference/max": 1.7147712707519531, "sampling/sampling_logp_difference/mean": 0.15160313248634338, "step": 27, "step_time": 11.396256268002617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 384.0, "completions/max_terminated_length": 384.0, "completions/mean_length": 13.9375, "completions/mean_terminated_length": 13.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7480363864451647, "epoch": 0.00056, "frac_reward_zero_std": 0.0, "grad_norm": 1.012612223625183, "kl": 0.44282771460711956, "learning_rate": 7.714285714285716e-06, "loss": -0.0022, "num_tokens": 1218916.0, "reward": -0.053125008940696716, "reward_std": 0.8883851170539856, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 2.502302885055542, "sampling/importance_sampling_ratio/mean": 1.1656153202056885, "sampling/importance_sampling_ratio/min": 0.21121083199977875, "sampling/sampling_logp_difference/max": 1.5548205375671387, "sampling/sampling_logp_difference/mean": 0.12281522154808044, "step": 28, "step_time": 10.43434568199882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 35.71875, "completions/mean_terminated_length": 35.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9166710667777807, "epoch": 0.00058, "frac_reward_zero_std": 0.0, "grad_norm": 0.7254788875579834, "kl": 3.1689953859895468, "learning_rate": 8.000000000000001e-06, "loss": -0.025, "num_tokens": 1264111.0, "reward": -0.5581250190734863, "reward_std": 0.8522208333015442, "rewards/rollout_reward_func/mean": -0.5581250190734863, "rewards/rollout_reward_func/std": 0.8484271168708801, "sampling/importance_sampling_ratio/max": 1.590344786643982, "sampling/importance_sampling_ratio/mean": 0.8667327165603638, "sampling/importance_sampling_ratio/min": 0.08781670778989792, "sampling/sampling_logp_difference/max": 2.432515859603882, "sampling/sampling_logp_difference/mean": 0.18564936518669128, "step": 29, "step_time": 12.224259169997822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 21.25, "completions/mean_terminated_length": 21.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8425687765702605, "epoch": 0.0006, "frac_reward_zero_std": 0.0, "grad_norm": 0.8417797684669495, "kl": 6.761422930285335, "learning_rate": 8.285714285714287e-06, "loss": -0.0027, "num_tokens": 1308608.0, "reward": -0.5581250190734863, "reward_std": 0.8522208333015442, "rewards/rollout_reward_func/mean": -0.5581250190734863, "rewards/rollout_reward_func/std": 0.8484271168708801, "sampling/importance_sampling_ratio/max": 1.3233213424682617, "sampling/importance_sampling_ratio/mean": 0.8790379166603088, "sampling/importance_sampling_ratio/min": 0.06082652136683464, "sampling/sampling_logp_difference/max": 2.79976487159729, "sampling/sampling_logp_difference/mean": 0.1907103955745697, "step": 30, "step_time": 11.424345586994605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 213.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 9.03125, "completions/mean_terminated_length": 8.806451797485352, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8697557772975415, "epoch": 0.00062, "frac_reward_zero_std": 0.0, "grad_norm": 0.7362136244773865, "kl": 2.256235963664949, "learning_rate": 8.571428571428571e-06, "loss": 0.0129, "num_tokens": 1352503.0, "reward": -0.36937499046325684, "reward_std": 0.9429470300674438, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.2871812582015991, "sampling/importance_sampling_ratio/mean": 0.9355064034461975, "sampling/importance_sampling_ratio/min": 0.08177375048398972, "sampling/sampling_logp_difference/max": 2.503854274749756, "sampling/sampling_logp_difference/mean": 0.12451677769422531, "step": 31, "step_time": 10.265051972011861 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 13.65625, "completions/mean_terminated_length": 13.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9568654457107186, "epoch": 0.00064, "frac_reward_zero_std": 0.25, "grad_norm": 0.6355522871017456, "kl": 0.8942806646227837, "learning_rate": 8.857142857142858e-06, "loss": -0.0078, "num_tokens": 1397474.0, "reward": -0.5581250190734863, "reward_std": 0.6822478771209717, "rewards/rollout_reward_func/mean": -0.5581250190734863, "rewards/rollout_reward_func/std": 0.8484271168708801, "sampling/importance_sampling_ratio/max": 2.646205186843872, "sampling/importance_sampling_ratio/mean": 1.0772745609283447, "sampling/importance_sampling_ratio/min": 0.1262790411710739, "sampling/sampling_logp_difference/max": 2.0691747665405273, "sampling/sampling_logp_difference/mean": 0.1210440993309021, "step": 32, "step_time": 11.332661205000477 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009765625, "completions/clipped_ratio": 0.03125, "completions/max_length": 321.0, "completions/max_terminated_length": 321.0, "completions/mean_length": 12.40625, "completions/mean_terminated_length": 12.290322303771973, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7108103199861944, "epoch": 0.00066, "frac_reward_zero_std": 0.0, "grad_norm": 0.7612587809562683, "kl": 4.541434457525611, "learning_rate": 9.142857142857144e-06, "loss": 0.0053, "num_tokens": 1440764.0, "reward": -0.24312500655651093, "reward_std": 0.9791113138198853, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.6250250339508057, "sampling/importance_sampling_ratio/mean": 1.027604579925537, "sampling/importance_sampling_ratio/min": 0.09694358706474304, "sampling/sampling_logp_difference/max": 2.333430767059326, "sampling/sampling_logp_difference/mean": 0.09147238731384277, "step": 33, "step_time": 10.31220469300024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 18.71875, "completions/mean_terminated_length": 18.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7961781020276248, "epoch": 0.00068, "frac_reward_zero_std": 0.0, "grad_norm": 1.3770873546600342, "kl": 10.769548650830984, "learning_rate": 9.42857142857143e-06, "loss": 0.008, "num_tokens": 1485171.0, "reward": -0.24312500655651093, "reward_std": 1.0067436695098877, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.4364633560180664, "sampling/importance_sampling_ratio/mean": 0.9861564040184021, "sampling/importance_sampling_ratio/min": 0.08366281539201736, "sampling/sampling_logp_difference/max": 2.480712413787842, "sampling/sampling_logp_difference/mean": 0.10766791552305222, "step": 34, "step_time": 11.224970709990885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 25.03125, "completions/mean_terminated_length": 25.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7858833733480424, "epoch": 0.0007, "frac_reward_zero_std": 0.0, "grad_norm": 1.0330138206481934, "kl": 8.307057168334723, "learning_rate": 9.714285714285715e-06, "loss": -0.012, "num_tokens": 1530383.0, "reward": -0.43187499046325684, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 2.353869676589966, "sampling/importance_sampling_ratio/mean": 1.0010517835617065, "sampling/importance_sampling_ratio/min": 0.10979831963777542, "sampling/sampling_logp_difference/max": 2.2089667320251465, "sampling/sampling_logp_difference/mean": 0.1532958298921585, "step": 35, "step_time": 11.12699087398505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 186.0, "completions/max_terminated_length": 186.0, "completions/mean_length": 7.75, "completions/mean_terminated_length": 7.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6422080947086215, "epoch": 0.00072, "frac_reward_zero_std": 0.0, "grad_norm": 0.8231570720672607, "kl": 5.247677996754646, "learning_rate": 1e-05, "loss": -0.0073, "num_tokens": 1573681.0, "reward": -0.36937499046325684, "reward_std": 0.9700891375541687, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.70852530002594, "sampling/importance_sampling_ratio/mean": 0.996002197265625, "sampling/importance_sampling_ratio/min": 0.16062688827514648, "sampling/sampling_logp_difference/max": 1.8286256790161133, "sampling/sampling_logp_difference/mean": 0.14497563242912292, "step": 36, "step_time": 9.763581996994617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 357.0, "completions/max_terminated_length": 357.0, "completions/mean_length": 13.09375, "completions/mean_terminated_length": 13.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9733941908925772, "epoch": 0.00074, "frac_reward_zero_std": 0.0, "grad_norm": 0.7395598888397217, "kl": 0.5669558551162481, "learning_rate": 9.999975267482496e-06, "loss": -0.0213, "num_tokens": 1617968.0, "reward": -0.36875003576278687, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.5473345518112183, "sampling/importance_sampling_ratio/mean": 1.0303630828857422, "sampling/importance_sampling_ratio/min": 0.5258460640907288, "sampling/sampling_logp_difference/max": 0.6426796913146973, "sampling/sampling_logp_difference/mean": 0.09076067060232162, "step": 37, "step_time": 10.716105106999748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 44.0, "completions/mean_terminated_length": 44.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.027046786621213, "epoch": 0.00076, "frac_reward_zero_std": 0.25, "grad_norm": 0.38944578170776367, "kl": 1.336287866346538, "learning_rate": 9.99990107025622e-06, "loss": -0.0197, "num_tokens": 1663655.0, "reward": -0.4950000047683716, "reward_std": 0.7564942836761475, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 1.7334187030792236, "sampling/importance_sampling_ratio/mean": 0.9044437408447266, "sampling/importance_sampling_ratio/min": 0.26329341530799866, "sampling/sampling_logp_difference/max": 1.3340981006622314, "sampling/sampling_logp_difference/mean": 0.1301455795764923, "step": 38, "step_time": 11.769310128001962 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6.0, "completions/max_terminated_length": 6.0, "completions/mean_length": 2.125, "completions/mean_terminated_length": 2.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7196492217481136, "epoch": 0.00078, "frac_reward_zero_std": 0.0, "grad_norm": 0.831965982913971, "kl": 0.5431868443265557, "learning_rate": 9.99977740929988e-06, "loss": 0.0082, "num_tokens": 1706643.0, "reward": -0.11687500774860382, "reward_std": 1.0449373722076416, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.6070606708526611, "sampling/importance_sampling_ratio/mean": 1.0358202457427979, "sampling/importance_sampling_ratio/min": 0.5587164759635925, "sampling/sampling_logp_difference/max": 0.5820729732513428, "sampling/sampling_logp_difference/mean": 0.0934421494603157, "step": 39, "step_time": 9.868151147988101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.017578125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017578125, "completions/clipped_ratio": 0.03125, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 28.28125, "completions/mean_terminated_length": 28.677417755126953, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7809339687228203, "epoch": 0.0008, "frac_reward_zero_std": 0.0, "grad_norm": 0.4963785707950592, "kl": 1.1810962613672018, "learning_rate": 9.999604286244655e-06, "loss": -0.0048, "num_tokens": 1751126.0, "reward": -0.43187499046325684, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.3726229667663574, "sampling/importance_sampling_ratio/mean": 0.9366868734359741, "sampling/importance_sampling_ratio/min": 0.3469223380088806, "sampling/sampling_logp_difference/max": 1.1994409561157227, "sampling/sampling_logp_difference/mean": 0.11903870105743408, "step": 40, "step_time": 12.812802573993395 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 49.125, "completions/mean_terminated_length": 49.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7560571078211069, "epoch": 0.00082, "frac_reward_zero_std": 0.0, "grad_norm": 0.42093878984451294, "kl": 0.6491071693599224, "learning_rate": 9.999381703374151e-06, "loss": -0.0013, "num_tokens": 1796011.0, "reward": -0.24250000715255737, "reward_std": 0.860792338848114, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 2.025148391723633, "sampling/importance_sampling_ratio/mean": 1.0319427251815796, "sampling/importance_sampling_ratio/min": 0.4150913655757904, "sampling/sampling_logp_difference/max": 1.1338307857513428, "sampling/sampling_logp_difference/mean": 0.12987510859966278, "step": 41, "step_time": 11.5935614380287 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 204.0, "completions/max_terminated_length": 204.0, "completions/mean_length": 13.625, "completions/mean_terminated_length": 13.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4092940101400018, "epoch": 0.00084, "frac_reward_zero_std": 0.0, "grad_norm": 0.696532666683197, "kl": 0.7095080874860287, "learning_rate": 9.999109663624387e-06, "loss": -0.0021, "num_tokens": 1836930.0, "reward": 0.2618749737739563, "reward_std": 1.0449373722076416, "rewards/rollout_reward_func/mean": 0.2618749737739563, "rewards/rollout_reward_func/std": 0.9930903911590576, "sampling/importance_sampling_ratio/max": 1.8468196392059326, "sampling/importance_sampling_ratio/mean": 1.018528938293457, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1263619661331177, "sampling/sampling_logp_difference/mean": 0.10224512219429016, "step": 42, "step_time": 9.890783256014402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.0, "completions/max_terminated_length": 330.0, "completions/mean_length": 12.375, "completions/mean_terminated_length": 12.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6816630037501454, "epoch": 0.00086, "frac_reward_zero_std": 0.25, "grad_norm": 0.4352133870124817, "kl": 1.6185806225985289, "learning_rate": 9.99878817058375e-06, "loss": -0.0039, "num_tokens": 1881390.0, "reward": -0.4950000047683716, "reward_std": 0.6736763715744019, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 2.8228118419647217, "sampling/importance_sampling_ratio/mean": 1.0034019947052002, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4378703832626343, "sampling/sampling_logp_difference/mean": 0.17666828632354736, "step": 43, "step_time": 11.125497522014484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 602.0, "completions/max_terminated_length": 602.0, "completions/mean_length": 37.4375, "completions/mean_terminated_length": 37.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7677044998854399, "epoch": 0.00088, "frac_reward_zero_std": 0.0, "grad_norm": 0.5419669151306152, "kl": 1.1407826859503984, "learning_rate": 9.998417228492952e-06, "loss": -0.0126, "num_tokens": 1927199.0, "reward": -0.4950000047683716, "reward_std": 0.9350782632827759, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 1.5034878253936768, "sampling/importance_sampling_ratio/mean": 0.846903920173645, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.02264404296875, "sampling/sampling_logp_difference/mean": 0.19105038046836853, "step": 44, "step_time": 11.418263587984256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 42.65625, "completions/mean_terminated_length": 42.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7085529053583741, "epoch": 0.0009, "frac_reward_zero_std": 0.0, "grad_norm": 0.8932253122329712, "kl": 0.6081150267273188, "learning_rate": 9.99799684224497e-06, "loss": -0.0129, "num_tokens": 1972117.0, "reward": -0.11687500774860382, "reward_std": 1.042907953262329, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 2.5155322551727295, "sampling/importance_sampling_ratio/mean": 0.9644193649291992, "sampling/importance_sampling_ratio/min": 0.35833725333213806, "sampling/sampling_logp_difference/max": 1.0257883071899414, "sampling/sampling_logp_difference/mean": 0.19078195095062256, "step": 45, "step_time": 11.079020894998393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 16.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.4375, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.61114920116961, "epoch": 0.00092, "frac_reward_zero_std": 0.0, "grad_norm": 0.4450460970401764, "kl": 1.0686105769127607, "learning_rate": 9.997527017384983e-06, "loss": 0.0018, "num_tokens": 2015421.0, "reward": -0.6212500333786011, "reward_std": 0.8246280550956726, "rewards/rollout_reward_func/mean": -0.6212500333786011, "rewards/rollout_reward_func/std": 0.8010466694831848, "sampling/importance_sampling_ratio/max": 1.378600835800171, "sampling/importance_sampling_ratio/mean": 0.9049894213676453, "sampling/importance_sampling_ratio/min": 0.2500297427177429, "sampling/sampling_logp_difference/max": 1.3859829902648926, "sampling/sampling_logp_difference/mean": 0.14530596137046814, "step": 46, "step_time": 10.412889323000854 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 366.0, "completions/max_terminated_length": 366.0, "completions/mean_length": 27.46875, "completions/mean_terminated_length": 27.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.45126095227897167, "epoch": 0.00094, "frac_reward_zero_std": 0.0, "grad_norm": 0.7834930419921875, "kl": 3.0919060595333576, "learning_rate": 9.997007760110298e-06, "loss": 0.0009, "num_tokens": 2058522.0, "reward": -0.24250000715255737, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 2.396512508392334, "sampling/importance_sampling_ratio/mean": 0.922390878200531, "sampling/importance_sampling_ratio/min": 0.22439061105251312, "sampling/sampling_logp_difference/max": 1.494025707244873, "sampling/sampling_logp_difference/mean": 0.187630295753479, "step": 47, "step_time": 10.452874400019937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 330.0, "completions/max_terminated_length": 330.0, "completions/mean_length": 20.3125, "completions/mean_terminated_length": 20.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.48484386689960957, "epoch": 0.00096, "frac_reward_zero_std": 0.0, "grad_norm": 0.227596715092659, "kl": 1.3651910685002804, "learning_rate": 9.996439077270269e-06, "loss": -0.0125, "num_tokens": 2102471.0, "reward": -0.49562501907348633, "reward_std": 0.9056781530380249, "rewards/rollout_reward_func/mean": -0.49562501907348633, "rewards/rollout_reward_func/std": 0.8875879645347595, "sampling/importance_sampling_ratio/max": 1.672074794769287, "sampling/importance_sampling_ratio/mean": 0.9739598035812378, "sampling/importance_sampling_ratio/min": 0.3719911277294159, "sampling/sampling_logp_difference/max": 1.1275887489318848, "sampling/sampling_logp_difference/mean": 0.14412572979927063, "step": 48, "step_time": 10.337521443994774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 61.6875, "completions/mean_terminated_length": 61.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.43444518465548754, "epoch": 0.00098, "frac_reward_zero_std": 0.25, "grad_norm": 0.32844191789627075, "kl": 0.6293791271746159, "learning_rate": 9.995820976366208e-06, "loss": -0.0263, "num_tokens": 2149080.0, "reward": -0.3687500059604645, "reward_std": 0.7564942836761475, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.4385292530059814, "sampling/importance_sampling_ratio/mean": 1.0379738807678223, "sampling/importance_sampling_ratio/min": 0.37481850385665894, "sampling/sampling_logp_difference/max": 0.8474884033203125, "sampling/sampling_logp_difference/mean": 0.09049835801124573, "step": 49, "step_time": 12.343467876999057 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 40.75, "completions/mean_terminated_length": 40.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.44607881642878056, "epoch": 0.001, "frac_reward_zero_std": 0.0, "grad_norm": 0.31479763984680176, "kl": 1.1030713468790054, "learning_rate": 9.995153465551283e-06, "loss": -0.0036, "num_tokens": 2194180.0, "reward": -0.3062500059604645, "reward_std": 0.9981721043586731, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 2.4394545555114746, "sampling/importance_sampling_ratio/mean": 0.988561749458313, "sampling/importance_sampling_ratio/min": 0.3176950514316559, "sampling/sampling_logp_difference/max": 1.1464309692382812, "sampling/sampling_logp_difference/mean": 0.1458698809146881, "step": 50, "step_time": 10.88682957101264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 63.21875, "completions/mean_terminated_length": 63.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3528860993683338, "epoch": 0.00102, "frac_reward_zero_std": 0.0, "grad_norm": 0.45462533831596375, "kl": 1.7221911679953337, "learning_rate": 9.994436553630411e-06, "loss": -0.0078, "num_tokens": 2239268.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.39462411403656, "sampling/importance_sampling_ratio/mean": 0.9643653035163879, "sampling/importance_sampling_ratio/min": 0.22743728756904602, "sampling/sampling_logp_difference/max": 1.480198860168457, "sampling/sampling_logp_difference/mean": 0.13230440020561218, "step": 51, "step_time": 11.287576889000775 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.020833333488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125000046566129, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 91.15625, "completions/mean_terminated_length": 91.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3076598169282079, "epoch": 0.00104, "frac_reward_zero_std": 0.0, "grad_norm": 0.7462118268013, "kl": 1.4334196504205465, "learning_rate": 9.993670250060152e-06, "loss": -0.0156, "num_tokens": 2285340.0, "reward": 0.1356249898672104, "reward_std": 1.0449373722076416, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.5402473211288452, "sampling/importance_sampling_ratio/mean": 1.0772759914398193, "sampling/importance_sampling_ratio/min": 0.271751344203949, "sampling/sampling_logp_difference/max": 1.3026318550109863, "sampling/sampling_logp_difference/mean": 0.1296808123588562, "step": 52, "step_time": 12.503497617988614 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 82.34375, "completions/mean_terminated_length": 82.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2588592730462551, "epoch": 0.00106, "frac_reward_zero_std": 0.0, "grad_norm": 0.12822526693344116, "kl": 1.4068788420408964, "learning_rate": 9.99285456494856e-06, "loss": -0.0224, "num_tokens": 2332423.0, "reward": -0.5581250190734863, "reward_std": 0.8798531293869019, "rewards/rollout_reward_func/mean": -0.5581250190734863, "rewards/rollout_reward_func/std": 0.8484271168708801, "sampling/importance_sampling_ratio/max": 1.3991150856018066, "sampling/importance_sampling_ratio/mean": 1.0619444847106934, "sampling/importance_sampling_ratio/min": 0.3118636906147003, "sampling/sampling_logp_difference/max": 1.1649456024169922, "sampling/sampling_logp_difference/mean": 0.10711416602134705, "step": 53, "step_time": 11.104716867004754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 50.75, "completions/mean_terminated_length": 50.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1865886189043522, "epoch": 0.00108, "frac_reward_zero_std": 0.0, "grad_norm": 0.21232527494430542, "kl": 0.7287630047649145, "learning_rate": 9.991989509055075e-06, "loss": 0.0028, "num_tokens": 2375210.0, "reward": 0.13624998927116394, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.3144935369491577, "sampling/importance_sampling_ratio/mean": 1.091733455657959, "sampling/importance_sampling_ratio/min": 0.8157200217247009, "sampling/sampling_logp_difference/max": 0.21102550625801086, "sampling/sampling_logp_difference/mean": 0.04455772787332535, "step": 54, "step_time": 12.380230396986008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 114.53125, "completions/mean_terminated_length": 114.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21473789727315307, "epoch": 0.0011, "frac_reward_zero_std": 0.0, "grad_norm": 1.461179256439209, "kl": 9.061162114143372, "learning_rate": 9.991075093790372e-06, "loss": 0.0026, "num_tokens": 2422990.0, "reward": -0.4325000047683716, "reward_std": 0.9341806769371033, "rewards/rollout_reward_func/mean": -0.4325000047683716, "rewards/rollout_reward_func/std": 0.9217339754104614, "sampling/importance_sampling_ratio/max": 1.5466614961624146, "sampling/importance_sampling_ratio/mean": 0.9394662380218506, "sampling/importance_sampling_ratio/min": 0.16025322675704956, "sampling/sampling_logp_difference/max": 1.4823026657104492, "sampling/sampling_logp_difference/mean": 0.1521511971950531, "step": 55, "step_time": 11.735733306995826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 92.59375, "completions/mean_terminated_length": 92.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2059540688060224, "epoch": 0.00112, "frac_reward_zero_std": 0.0, "grad_norm": 0.15598154067993164, "kl": 0.8241954669356346, "learning_rate": 9.990111331216204e-06, "loss": -0.0118, "num_tokens": 2469689.0, "reward": -0.18000000715255737, "reward_std": 0.9981721639633179, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.3613200187683105, "sampling/importance_sampling_ratio/mean": 1.026908040046692, "sampling/importance_sampling_ratio/min": 0.3549956679344177, "sampling/sampling_logp_difference/max": 1.1564700603485107, "sampling/sampling_logp_difference/mean": 0.07610966265201569, "step": 56, "step_time": 11.664252020993445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 43.21875, "completions/mean_terminated_length": 43.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21927001513540745, "epoch": 0.00114, "frac_reward_zero_std": 0.0, "grad_norm": 0.5107495784759521, "kl": 3.012550186365843, "learning_rate": 9.989098234045254e-06, "loss": -0.0189, "num_tokens": 2515273.0, "reward": -0.3687500059604645, "reward_std": 0.9626314640045166, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.4425526857376099, "sampling/importance_sampling_ratio/mean": 0.8721513748168945, "sampling/importance_sampling_ratio/min": 0.19710630178451538, "sampling/sampling_logp_difference/max": 1.6180987358093262, "sampling/sampling_logp_difference/mean": 0.17093579471111298, "step": 57, "step_time": 12.848486519011203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 92.03125, "completions/mean_terminated_length": 92.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19916435703635216, "epoch": 0.00116, "frac_reward_zero_std": 0.0, "grad_norm": 0.1804942488670349, "kl": 1.9680915549397469, "learning_rate": 9.988035815640953e-06, "loss": -0.009, "num_tokens": 2562125.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.3082994222640991, "sampling/importance_sampling_ratio/mean": 1.0168112516403198, "sampling/importance_sampling_ratio/min": 0.30249226093292236, "sampling/sampling_logp_difference/max": 1.195734977722168, "sampling/sampling_logp_difference/mean": 0.08057618886232376, "step": 58, "step_time": 11.611599996984296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 71.8125, "completions/mean_terminated_length": 71.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23370478674769402, "epoch": 0.00118, "frac_reward_zero_std": 0.25, "grad_norm": 0.07777382433414459, "kl": 0.6592869572341442, "learning_rate": 9.986924090017322e-06, "loss": -0.0081, "num_tokens": 2610011.0, "reward": -0.4950000047683716, "reward_std": 0.7374730110168457, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 1.190380573272705, "sampling/importance_sampling_ratio/mean": 1.014493703842163, "sampling/importance_sampling_ratio/min": 0.4697192311286926, "sampling/sampling_logp_difference/max": 0.7383666038513184, "sampling/sampling_logp_difference/mean": 0.06715209782123566, "step": 59, "step_time": 11.511684830002196 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.03125, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 55.25, "completions/mean_terminated_length": 56.51612854003906, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2742707747966051, "epoch": 0.0012, "frac_reward_zero_std": 0.0, "grad_norm": 0.7342955470085144, "kl": 1.1098666675388813, "learning_rate": 9.985763071838774e-06, "loss": 0.0116, "num_tokens": 2654866.0, "reward": -0.17937500774860382, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.3714877367019653, "sampling/importance_sampling_ratio/mean": 1.0018641948699951, "sampling/importance_sampling_ratio/min": 0.2198164463043213, "sampling/sampling_logp_difference/max": 1.1127009391784668, "sampling/sampling_logp_difference/mean": 0.0904836505651474, "step": 60, "step_time": 12.580888715012406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 52.28125, "completions/mean_terminated_length": 52.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2362475423142314, "epoch": 0.00122, "frac_reward_zero_std": 0.0, "grad_norm": 0.131918266415596, "kl": 1.6472347266972065, "learning_rate": 9.98455277641992e-06, "loss": -0.0059, "num_tokens": 2700267.0, "reward": -0.3687500059604645, "reward_std": 0.9712425470352173, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.1725562810897827, "sampling/importance_sampling_ratio/mean": 1.0029021501541138, "sampling/importance_sampling_ratio/min": 0.40411725640296936, "sampling/sampling_logp_difference/max": 0.8854658603668213, "sampling/sampling_logp_difference/mean": 0.08134079724550247, "step": 61, "step_time": 11.551218055006757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 61.125, "completions/mean_terminated_length": 61.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19655722845345736, "epoch": 0.00124, "frac_reward_zero_std": 0.0, "grad_norm": 0.06847658008337021, "kl": 1.0828284453600645, "learning_rate": 9.983293219725379e-06, "loss": -0.0182, "num_tokens": 2745561.0, "reward": -0.11625000834465027, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.2213244438171387, "sampling/importance_sampling_ratio/mean": 0.9811855554580688, "sampling/importance_sampling_ratio/min": 0.44130387902259827, "sampling/sampling_logp_difference/max": 0.817976713180542, "sampling/sampling_logp_difference/mean": 0.0791253075003624, "step": 62, "step_time": 10.90633414201875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 62.96875, "completions/mean_terminated_length": 62.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24199629947543144, "epoch": 0.00126, "frac_reward_zero_std": 0.25, "grad_norm": 0.12385766953229904, "kl": 1.0348778646439314, "learning_rate": 9.981984418369552e-06, "loss": -0.0015, "num_tokens": 2792011.0, "reward": -0.43187499046325684, "reward_std": 0.7289015054702759, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.2623605728149414, "sampling/importance_sampling_ratio/mean": 1.0169847011566162, "sampling/importance_sampling_ratio/min": 0.37802302837371826, "sampling/sampling_logp_difference/max": 0.8706917762756348, "sampling/sampling_logp_difference/mean": 0.08789366483688354, "step": 63, "step_time": 13.527496928996698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 46.21875, "completions/mean_terminated_length": 46.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2124620615504682, "epoch": 0.00128, "frac_reward_zero_std": 0.0, "grad_norm": 0.09499778598546982, "kl": 1.057374270632863, "learning_rate": 9.980626389616414e-06, "loss": -0.0319, "num_tokens": 2836776.0, "reward": -0.11625000834465027, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.190427541732788, "sampling/importance_sampling_ratio/mean": 1.0379104614257812, "sampling/importance_sampling_ratio/min": 0.5047110915184021, "sampling/sampling_logp_difference/max": 0.6647145748138428, "sampling/sampling_logp_difference/mean": 0.0609111487865448, "step": 64, "step_time": 10.673833380984433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 74.90625, "completions/mean_terminated_length": 74.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2393715539947152, "epoch": 0.0013, "frac_reward_zero_std": 0.0, "grad_norm": 1.8150489330291748, "kl": 4.895392782986164, "learning_rate": 9.979219151379285e-06, "loss": -0.004, "num_tokens": 2884125.0, "reward": -0.43187499046325684, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.4430902004241943, "sampling/importance_sampling_ratio/mean": 0.9783347249031067, "sampling/importance_sampling_ratio/min": 0.1597035974264145, "sampling/sampling_logp_difference/max": 1.9740710258483887, "sampling/sampling_logp_difference/mean": 0.12054869532585144, "step": 65, "step_time": 11.59554510802991 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 63.375, "completions/mean_terminated_length": 63.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23268678039312363, "epoch": 0.00132, "frac_reward_zero_std": 0.0, "grad_norm": 0.1058499664068222, "kl": 0.786825193092227, "learning_rate": 9.977762722220585e-06, "loss": -0.0141, "num_tokens": 2930272.0, "reward": -0.24250000715255737, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 1.2187714576721191, "sampling/importance_sampling_ratio/mean": 1.03284752368927, "sampling/importance_sampling_ratio/min": 0.33028465509414673, "sampling/sampling_logp_difference/max": 1.0961804389953613, "sampling/sampling_logp_difference/mean": 0.07454027235507965, "step": 66, "step_time": 12.362123988001258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 52.5625, "completions/mean_terminated_length": 52.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20529841911047697, "epoch": 0.00134, "frac_reward_zero_std": 0.0, "grad_norm": 0.5615201592445374, "kl": 2.315034555271268, "learning_rate": 9.976257121351602e-06, "loss": -0.0228, "num_tokens": 2974379.0, "reward": 0.009374991059303284, "reward_std": 1.0617344379425049, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.2011706829071045, "sampling/importance_sampling_ratio/mean": 0.9655159711837769, "sampling/importance_sampling_ratio/min": 0.19783371686935425, "sampling/sampling_logp_difference/max": 1.7083356380462646, "sampling/sampling_logp_difference/mean": 0.10217750817537308, "step": 67, "step_time": 11.475433154984785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 49.90625, "completions/mean_terminated_length": 49.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1948902104049921, "epoch": 0.00136, "frac_reward_zero_std": 0.0, "grad_norm": 0.1579577475786209, "kl": 1.5260688792914152, "learning_rate": 9.974702368632228e-06, "loss": 0.005, "num_tokens": 3018684.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.2112942934036255, "sampling/importance_sampling_ratio/mean": 1.0493426322937012, "sampling/importance_sampling_ratio/min": 0.43330538272857666, "sampling/sampling_logp_difference/max": 0.8360679149627686, "sampling/sampling_logp_difference/mean": 0.056468673050403595, "step": 68, "step_time": 11.55147792702337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 483.0, "completions/max_terminated_length": 483.0, "completions/mean_length": 46.65625, "completions/mean_terminated_length": 46.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23961756471544504, "epoch": 0.00138, "frac_reward_zero_std": 0.0, "grad_norm": 0.5269181728363037, "kl": 1.4896109476685524, "learning_rate": 9.973098484570702e-06, "loss": -0.0308, "num_tokens": 3064233.0, "reward": -0.11625000834465027, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.279442310333252, "sampling/importance_sampling_ratio/mean": 0.9789178371429443, "sampling/importance_sampling_ratio/min": 0.43163618445396423, "sampling/sampling_logp_difference/max": 0.7586016654968262, "sampling/sampling_logp_difference/mean": 0.08909390866756439, "step": 69, "step_time": 12.311888941003417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 79.25, "completions/mean_terminated_length": 79.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23127900203689933, "epoch": 0.0014, "frac_reward_zero_std": 0.0, "grad_norm": 0.1411987841129303, "kl": 1.4739936422556639, "learning_rate": 9.97144549032334e-06, "loss": -0.0264, "num_tokens": 3111086.0, "reward": -0.18000000715255737, "reward_std": 1.033846139907837, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.1825177669525146, "sampling/importance_sampling_ratio/mean": 0.9612064957618713, "sampling/importance_sampling_ratio/min": 0.35416167974472046, "sampling/sampling_logp_difference/max": 1.0379970073699951, "sampling/sampling_logp_difference/mean": 0.10613569617271423, "step": 70, "step_time": 11.738511798997934 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 348.0, "completions/max_terminated_length": 348.0, "completions/mean_length": 31.46875, "completions/mean_terminated_length": 31.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17388719599694014, "epoch": 0.00142, "frac_reward_zero_std": 0.0, "grad_norm": 0.13367214798927307, "kl": 1.1646664254367352, "learning_rate": 9.969743407694254e-06, "loss": -0.0152, "num_tokens": 3155023.0, "reward": 0.009374991059303284, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.1922001838684082, "sampling/importance_sampling_ratio/mean": 1.012123465538025, "sampling/importance_sampling_ratio/min": 0.27834123373031616, "sampling/sampling_logp_difference/max": 1.2789702415466309, "sampling/sampling_logp_difference/mean": 0.07257995009422302, "step": 71, "step_time": 10.926322246996278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 59.1875, "completions/mean_terminated_length": 59.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1976085165515542, "epoch": 0.00144, "frac_reward_zero_std": 0.0, "grad_norm": 0.2524814009666443, "kl": 0.9518985077738762, "learning_rate": 9.967992259135063e-06, "loss": -0.0143, "num_tokens": 3201275.0, "reward": -0.24250000715255737, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.1776372194290161, "sampling/importance_sampling_ratio/mean": 1.0004748106002808, "sampling/importance_sampling_ratio/min": 0.2958310544490814, "sampling/sampling_logp_difference/max": 1.2747445106506348, "sampling/sampling_logp_difference/mean": 0.07727165520191193, "step": 72, "step_time": 12.579483394998533 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 57.0625, "completions/mean_terminated_length": 57.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17659917450509965, "epoch": 0.00146, "frac_reward_zero_std": 0.0, "grad_norm": 0.265146404504776, "kl": 2.0242786835879087, "learning_rate": 9.9661920677446e-06, "loss": -0.003, "num_tokens": 3246432.0, "reward": 0.009374991059303284, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.5880414247512817, "sampling/importance_sampling_ratio/mean": 1.0384960174560547, "sampling/importance_sampling_ratio/min": 0.2562171518802643, "sampling/sampling_logp_difference/max": 1.3600177764892578, "sampling/sampling_logp_difference/mean": 0.06503751873970032, "step": 73, "step_time": 11.734008066006936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 83.75, "completions/mean_terminated_length": 83.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17888660356402397, "epoch": 0.00148, "frac_reward_zero_std": 0.0, "grad_norm": 0.13857032358646393, "kl": 1.2385688051581383, "learning_rate": 9.964342857268609e-06, "loss": -0.0155, "num_tokens": 3293523.0, "reward": -0.43187499046325684, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.1779422760009766, "sampling/importance_sampling_ratio/mean": 1.0275330543518066, "sampling/importance_sampling_ratio/min": 0.29330354928970337, "sampling/sampling_logp_difference/max": 1.307389497756958, "sampling/sampling_logp_difference/mean": 0.060125648975372314, "step": 74, "step_time": 12.11033266600134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 109.3125, "completions/mean_terminated_length": 109.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17255390342324972, "epoch": 0.0015, "frac_reward_zero_std": 0.0, "grad_norm": 0.4558054506778717, "kl": 4.556879173964262, "learning_rate": 9.962444652099422e-06, "loss": -0.0037, "num_tokens": 3341686.0, "reward": -0.36875003576278687, "reward_std": 0.9074459075927734, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767791748047, "sampling/importance_sampling_ratio/max": 1.333703875541687, "sampling/importance_sampling_ratio/mean": 0.9615335464477539, "sampling/importance_sampling_ratio/min": 0.2420041859149933, "sampling/sampling_logp_difference/max": 1.418318271636963, "sampling/sampling_logp_difference/mean": 0.10259148478507996, "step": 75, "step_time": 12.266180637969228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 222.0, "completions/max_terminated_length": 222.0, "completions/mean_length": 8.875, "completions/mean_terminated_length": 8.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20938470494002104, "epoch": 0.00152, "frac_reward_zero_std": 0.0, "grad_norm": 0.11724161356687546, "kl": 0.9100913088768721, "learning_rate": 9.960497477275653e-06, "loss": -0.0069, "num_tokens": 3386287.0, "reward": -0.43187499046325684, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.1125833988189697, "sampling/importance_sampling_ratio/mean": 0.9947004318237305, "sampling/importance_sampling_ratio/min": 0.27328377962112427, "sampling/sampling_logp_difference/max": 1.0277259349822998, "sampling/sampling_logp_difference/mean": 0.07346513867378235, "step": 76, "step_time": 10.249607939011184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 70.6875, "completions/mean_terminated_length": 70.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18831276567652822, "epoch": 0.00154, "frac_reward_zero_std": 0.0, "grad_norm": 0.12098287791013718, "kl": 1.4878373593091965, "learning_rate": 9.95850135848185e-06, "loss": -0.0105, "num_tokens": 3432393.0, "reward": -0.3056250214576721, "reward_std": 0.9521816968917847, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.164184808731079, "sampling/importance_sampling_ratio/mean": 0.9539521932601929, "sampling/importance_sampling_ratio/min": 0.27924293279647827, "sampling/sampling_logp_difference/max": 1.042396068572998, "sampling/sampling_logp_difference/mean": 0.09555620700120926, "step": 77, "step_time": 13.060628709004959 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 59.15625, "completions/mean_terminated_length": 59.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19426002772524953, "epoch": 0.00156, "frac_reward_zero_std": 0.0, "grad_norm": 0.2136894315481186, "kl": 2.3301228024065495, "learning_rate": 9.956456322048174e-06, "loss": -0.0321, "num_tokens": 3478637.0, "reward": -0.24250002205371857, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.24250002205371857, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 1.1924937963485718, "sampling/importance_sampling_ratio/mean": 1.0421090126037598, "sampling/importance_sampling_ratio/min": 0.2734510600566864, "sampling/sampling_logp_difference/max": 1.291327953338623, "sampling/sampling_logp_difference/mean": 0.050746623426675797, "step": 78, "step_time": 10.998594626988051 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 95.6875, "completions/mean_terminated_length": 95.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21478537563234568, "epoch": 0.00158, "frac_reward_zero_std": 0.0, "grad_norm": 0.09457482397556305, "kl": 0.8173107374459505, "learning_rate": 9.954362394950035e-06, "loss": -0.0131, "num_tokens": 3526403.0, "reward": -0.18062500655651093, "reward_std": 1.0334784984588623, "rewards/rollout_reward_func/mean": -0.18062500655651093, "rewards/rollout_reward_func/std": 1.006437063217163, "sampling/importance_sampling_ratio/max": 1.1699942350387573, "sampling/importance_sampling_ratio/mean": 0.9492309093475342, "sampling/importance_sampling_ratio/min": 0.2290450632572174, "sampling/sampling_logp_difference/max": 1.5505987405776978, "sampling/sampling_logp_difference/mean": 0.10652923583984375, "step": 79, "step_time": 11.69718050499796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 42.15625, "completions/mean_terminated_length": 42.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1941694226115942, "epoch": 0.0016, "frac_reward_zero_std": 0.0, "grad_norm": 0.1760215163230896, "kl": 2.7613948471844196, "learning_rate": 9.952219604807746e-06, "loss": -0.008, "num_tokens": 3570731.0, "reward": -0.17937500774860382, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.1483756303787231, "sampling/importance_sampling_ratio/mean": 1.0028650760650635, "sampling/importance_sampling_ratio/min": 0.27332472801208496, "sampling/sampling_logp_difference/max": 1.2879505157470703, "sampling/sampling_logp_difference/mean": 0.06740148365497589, "step": 80, "step_time": 13.095802609015664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 37.53125, "completions/mean_terminated_length": 37.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2147835912182927, "epoch": 0.00162, "frac_reward_zero_std": 0.0, "grad_norm": 0.15608933568000793, "kl": 0.7971496433019638, "learning_rate": 9.950027979886159e-06, "loss": -0.0011, "num_tokens": 3614938.0, "reward": -0.11812500655651093, "reward_std": 0.9424644708633423, "rewards/rollout_reward_func/mean": -0.11812500655651093, "rewards/rollout_reward_func/std": 1.0159674882888794, "sampling/importance_sampling_ratio/max": 1.3321868181228638, "sampling/importance_sampling_ratio/mean": 1.0439398288726807, "sampling/importance_sampling_ratio/min": 0.35446351766586304, "sampling/sampling_logp_difference/max": 0.9170780181884766, "sampling/sampling_logp_difference/mean": 0.05318332836031914, "step": 81, "step_time": 11.541422615991905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 420.0, "completions/max_terminated_length": 420.0, "completions/mean_length": 41.65625, "completions/mean_terminated_length": 41.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21180923236534, "epoch": 0.00164, "frac_reward_zero_std": 0.0, "grad_norm": 0.20777176320552826, "kl": 1.6776529215276241, "learning_rate": 9.947787549094288e-06, "loss": -0.0127, "num_tokens": 3658201.0, "reward": 0.009999990463256836, "reward_std": 1.0074068307876587, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.122735619544983, "sampling/importance_sampling_ratio/mean": 1.008034110069275, "sampling/importance_sampling_ratio/min": 0.2403235137462616, "sampling/sampling_logp_difference/max": 1.4257359504699707, "sampling/sampling_logp_difference/mean": 0.06991834938526154, "step": 82, "step_time": 10.729350555004203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 58.875, "completions/mean_terminated_length": 58.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21867159428074956, "epoch": 0.00166, "frac_reward_zero_std": 0.0, "grad_norm": 0.280441015958786, "kl": 2.2784926649183035, "learning_rate": 9.945498341984929e-06, "loss": -0.0057, "num_tokens": 3702721.0, "reward": 0.008749991655349731, "reward_std": 1.0060805082321167, "rewards/rollout_reward_func/mean": 0.008749991655349731, "rewards/rollout_reward_func/std": 1.0249019861221313, "sampling/importance_sampling_ratio/max": 1.1341757774353027, "sampling/importance_sampling_ratio/mean": 0.9979944229125977, "sampling/importance_sampling_ratio/min": 0.26346930861473083, "sampling/sampling_logp_difference/max": 1.3337531089782715, "sampling/sampling_logp_difference/mean": 0.07190564274787903, "step": 83, "step_time": 12.455428555003891 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 45.46875, "completions/mean_terminated_length": 45.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3492494188249111, "epoch": 0.00168, "frac_reward_zero_std": 0.0, "grad_norm": 0.3022581934928894, "kl": 1.899079117923975, "learning_rate": 9.943160388754274e-06, "loss": -0.0109, "num_tokens": 3748606.0, "reward": -0.3062500059604645, "reward_std": 0.9345266222953796, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 1.117174506187439, "sampling/importance_sampling_ratio/mean": 0.8734064698219299, "sampling/importance_sampling_ratio/min": 0.2590082287788391, "sampling/sampling_logp_difference/max": 1.2697927951812744, "sampling/sampling_logp_difference/mean": 0.16177377104759216, "step": 84, "step_time": 11.618174575000012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 599.0, "completions/max_terminated_length": 599.0, "completions/mean_length": 55.40625, "completions/mean_terminated_length": 55.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3942499291151762, "epoch": 0.0017, "frac_reward_zero_std": 0.0, "grad_norm": 0.3136917054653168, "kl": 1.284922305494547, "learning_rate": 9.940773720241503e-06, "loss": -0.0112, "num_tokens": 3794913.0, "reward": -0.24312500655651093, "reward_std": 0.9897517561912537, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.0986663103103638, "sampling/importance_sampling_ratio/mean": 0.8684815168380737, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8234405517578125, "sampling/sampling_logp_difference/mean": 0.15363314747810364, "step": 85, "step_time": 11.471524151005724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 92.59375, "completions/mean_terminated_length": 92.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31988708628341556, "epoch": 0.00172, "frac_reward_zero_std": 0.0, "grad_norm": 0.6907168626785278, "kl": 2.67919097840786, "learning_rate": 9.938338367928391e-06, "loss": 0.0091, "num_tokens": 3841148.0, "reward": -0.11687500774860382, "reward_std": 1.0259159803390503, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.2801426649093628, "sampling/importance_sampling_ratio/mean": 0.8710198402404785, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.865297555923462, "sampling/sampling_logp_difference/mean": 0.14682155847549438, "step": 86, "step_time": 12.752270435994433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 81.84375, "completions/mean_terminated_length": 81.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37421192694455385, "epoch": 0.00174, "frac_reward_zero_std": 0.0, "grad_norm": 0.14645454287528992, "kl": 1.5167259611189365, "learning_rate": 9.935854363938876e-06, "loss": -0.0162, "num_tokens": 3887750.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.1217045783996582, "sampling/importance_sampling_ratio/mean": 0.8462419509887695, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1907668113708496, "sampling/sampling_logp_difference/mean": 0.14148810505867004, "step": 87, "step_time": 11.52130465398659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 100.78125, "completions/mean_terminated_length": 100.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.42475425731390715, "epoch": 0.00176, "frac_reward_zero_std": 0.0, "grad_norm": 0.513873279094696, "kl": 1.4950469322502613, "learning_rate": 9.933321741038655e-06, "loss": 0.0274, "num_tokens": 3935697.0, "reward": -0.24312500655651093, "reward_std": 1.0252747535705566, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 2.9089114665985107, "sampling/importance_sampling_ratio/mean": 0.9046182036399841, "sampling/importance_sampling_ratio/min": 0.2879839241504669, "sampling/sampling_logp_difference/max": 1.022554874420166, "sampling/sampling_logp_difference/mean": 0.1499689221382141, "step": 88, "step_time": 12.044982764011365 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 664.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 34.90625, "completions/mean_terminated_length": 34.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.39967982890084386, "epoch": 0.00178, "frac_reward_zero_std": 0.0, "grad_norm": 0.3792923092842102, "kl": 2.050057142972946, "learning_rate": 9.930740532634733e-06, "loss": 0.0137, "num_tokens": 3980189.0, "reward": -0.24312500655651093, "reward_std": 0.9418424367904663, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 2.553908109664917, "sampling/importance_sampling_ratio/mean": 0.7941986322402954, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 20.556840896606445, "sampling/sampling_logp_difference/mean": 0.424967885017395, "step": 89, "step_time": 12.12830566100456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 83.78125, "completions/mean_terminated_length": 83.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2973370044492185, "epoch": 0.0018, "frac_reward_zero_std": 0.0, "grad_norm": 0.9235020279884338, "kl": 2.212584927678108, "learning_rate": 9.928110772774995e-06, "loss": -0.0368, "num_tokens": 4025981.0, "reward": 0.009374991059303284, "reward_std": 1.0617344379425049, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 2.9453330039978027, "sampling/importance_sampling_ratio/mean": 0.9084874391555786, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1897823810577393, "sampling/sampling_logp_difference/mean": 0.14653706550598145, "step": 90, "step_time": 11.021228947989584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0028409091755747795, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028409091755747795, "completions/clipped_ratio": 0.0, "completions/max_length": 499.0, "completions/max_terminated_length": 499.0, "completions/mean_length": 37.5, "completions/mean_terminated_length": 37.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25705015612766147, "epoch": 0.00182, "frac_reward_zero_std": 0.0, "grad_norm": 1.455798864364624, "kl": 2.6332956291735172, "learning_rate": 9.92543249614775e-06, "loss": 0.1255, "num_tokens": 4070719.0, "reward": -0.11687500774860382, "reward_std": 1.042907953262329, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 2.947312593460083, "sampling/importance_sampling_ratio/mean": 1.1030385494232178, "sampling/importance_sampling_ratio/min": 0.1105804592370987, "sampling/sampling_logp_difference/max": 2.293159246444702, "sampling/sampling_logp_difference/mean": 0.1870369166135788, "step": 91, "step_time": 11.593285730006755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 84.8125, "completions/mean_terminated_length": 87.03225708007812, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2822948880493641, "epoch": 0.00184, "frac_reward_zero_std": 0.0, "grad_norm": 1.9018090963363647, "kl": 4.800305031239986, "learning_rate": 9.922705738081279e-06, "loss": 0.051, "num_tokens": 4118007.0, "reward": -0.3075000047683716, "reward_std": 1.0149871110916138, "rewards/rollout_reward_func/mean": -0.3075000047683716, "rewards/rollout_reward_func/std": 0.9721508622169495, "sampling/importance_sampling_ratio/max": 2.3923892974853516, "sampling/importance_sampling_ratio/mean": 0.8866229057312012, "sampling/importance_sampling_ratio/min": 0.026840927079319954, "sampling/sampling_logp_difference/max": 3.007565975189209, "sampling/sampling_logp_difference/mean": 0.25455981492996216, "step": 92, "step_time": 12.27530051201029 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 330.0, "completions/max_terminated_length": 330.0, "completions/mean_length": 38.875, "completions/mean_terminated_length": 38.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1964356037788093, "epoch": 0.00186, "frac_reward_zero_std": 0.0, "grad_norm": 1.9499703645706177, "kl": 3.4241428952664137, "learning_rate": 9.919930534543365e-06, "loss": 0.1664, "num_tokens": 4162317.0, "reward": 0.009999990463256836, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 2.6458520889282227, "sampling/importance_sampling_ratio/mean": 1.1475515365600586, "sampling/importance_sampling_ratio/min": 0.07119249552488327, "sampling/sampling_logp_difference/max": 2.757072925567627, "sampling/sampling_logp_difference/mean": 0.21721240878105164, "step": 93, "step_time": 10.7222401839972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 81.125, "completions/mean_terminated_length": 81.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27563488902524114, "epoch": 0.00188, "frac_reward_zero_std": 0.0, "grad_norm": 0.7441632747650146, "kl": 1.6940552480518818, "learning_rate": 9.917106922140814e-06, "loss": 0.1233, "num_tokens": 4207978.0, "reward": -0.11625000834465027, "reward_std": 1.0435711145401, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.842447280883789, "sampling/importance_sampling_ratio/mean": 0.9243795275688171, "sampling/importance_sampling_ratio/min": 0.18035101890563965, "sampling/sampling_logp_difference/max": 1.7820159196853638, "sampling/sampling_logp_difference/mean": 0.18189460039138794, "step": 94, "step_time": 12.51795081000455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 77.90625, "completions/mean_terminated_length": 77.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3488580365665257, "epoch": 0.0019, "frac_reward_zero_std": 0.0, "grad_norm": 0.5429728627204895, "kl": 1.1483394913375378, "learning_rate": 9.914234938118988e-06, "loss": 0.0193, "num_tokens": 4253757.0, "reward": 0.009374991059303284, "reward_std": 1.0259159803390503, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.7057890892028809, "sampling/importance_sampling_ratio/mean": 0.7967541813850403, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4870154857635498, "sampling/sampling_logp_difference/mean": 0.1676263064146042, "step": 95, "step_time": 11.066309933012235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 84.40625, "completions/mean_terminated_length": 84.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3443884435109794, "epoch": 0.00192, "frac_reward_zero_std": 0.0, "grad_norm": 0.7581422328948975, "kl": 2.4960633609443903, "learning_rate": 9.911314620361294e-06, "loss": -0.0434, "num_tokens": 4299730.0, "reward": -0.3056250214576721, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 2.1668930053710938, "sampling/importance_sampling_ratio/mean": 0.9097690582275391, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3584489822387695, "sampling/sampling_logp_difference/mean": 0.13561227917671204, "step": 96, "step_time": 11.955056495011377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 85.1875, "completions/mean_terminated_length": 87.41934967041016, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3722026417963207, "epoch": 0.00194, "frac_reward_zero_std": 0.0, "grad_norm": 0.7946562170982361, "kl": 0.9476795988157392, "learning_rate": 9.908346007388698e-06, "loss": -0.0307, "num_tokens": 4345952.0, "reward": -0.17937500774860382, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.3337196111679077, "sampling/importance_sampling_ratio/mean": 0.8797585964202881, "sampling/importance_sampling_ratio/min": 0.30666008591651917, "sampling/sampling_logp_difference/max": 1.1692461967468262, "sampling/sampling_logp_difference/mean": 0.11259317398071289, "step": 97, "step_time": 12.419648253999185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 37.46875, "completions/mean_terminated_length": 38.16128921508789, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4387580412440002, "epoch": 0.00196, "frac_reward_zero_std": 0.0, "grad_norm": 0.8612997531890869, "kl": 1.1250556129962206, "learning_rate": 9.905329138359213e-06, "loss": 0.0298, "num_tokens": 4391328.0, "reward": -0.3056250214576721, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.6508891582489014, "sampling/importance_sampling_ratio/mean": 0.8393794298171997, "sampling/importance_sampling_ratio/min": 0.09522183239459991, "sampling/sampling_logp_difference/max": 1.3453352451324463, "sampling/sampling_logp_difference/mean": 0.16517946124076843, "step": 98, "step_time": 11.416660034039523 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1270.0, "completions/max_terminated_length": 1270.0, "completions/mean_length": 124.0, "completions/mean_terminated_length": 124.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32197059132158756, "epoch": 0.00198, "frac_reward_zero_std": 0.0, "grad_norm": 0.7138665318489075, "kl": 2.662873912602663, "learning_rate": 9.902264053067381e-06, "loss": 0.019, "num_tokens": 4438254.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.7483880519866943, "sampling/importance_sampling_ratio/mean": 0.8002989292144775, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.513014793395996, "sampling/sampling_logp_difference/mean": 0.15341144800186157, "step": 99, "step_time": 14.532287683003233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 110.59375, "completions/mean_terminated_length": 110.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26894899643957615, "epoch": 0.002, "frac_reward_zero_std": 0.0, "grad_norm": 0.37445732951164246, "kl": 2.8692740127444267, "learning_rate": 9.899150791943747e-06, "loss": 0.0184, "num_tokens": 4485282.0, "reward": 0.008125007152557373, "reward_std": 0.6762641072273254, "rewards/rollout_reward_func/mean": 0.008125007152557373, "rewards/rollout_reward_func/std": 1.0242713689804077, "sampling/importance_sampling_ratio/max": 2.9676363468170166, "sampling/importance_sampling_ratio/mean": 0.819832444190979, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3042054176330566, "sampling/sampling_logp_difference/mean": 0.1892186552286148, "step": 100, "step_time": 12.186941377985931 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1130.0, "completions/max_terminated_length": 1130.0, "completions/mean_length": 54.4375, "completions/mean_terminated_length": 54.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2981878872960806, "epoch": 0.00202, "frac_reward_zero_std": 0.0, "grad_norm": 0.6246065497398376, "kl": 1.0195802003145218, "learning_rate": 9.895989396054334e-06, "loss": -0.057, "num_tokens": 4531253.0, "reward": -0.24250000715255737, "reward_std": 1.0074068307876587, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 2.677860736846924, "sampling/importance_sampling_ratio/mean": 0.9210923314094543, "sampling/importance_sampling_ratio/min": 0.19866611063480377, "sampling/sampling_logp_difference/max": 1.563672661781311, "sampling/sampling_logp_difference/mean": 0.15435180068016052, "step": 101, "step_time": 14.026156646024901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 303.0, "completions/max_terminated_length": 303.0, "completions/mean_length": 26.25, "completions/mean_terminated_length": 26.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27894961927086115, "epoch": 0.00204, "frac_reward_zero_std": 0.0, "grad_norm": 0.33247414231300354, "kl": 2.688132666051388, "learning_rate": 9.892779907100084e-06, "loss": 0.0073, "num_tokens": 4576663.0, "reward": -0.3062500059604645, "reward_std": 0.9705398082733154, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 2.555898666381836, "sampling/importance_sampling_ratio/mean": 0.8995653390884399, "sampling/importance_sampling_ratio/min": 0.20048213005065918, "sampling/sampling_logp_difference/max": 1.6853617429733276, "sampling/sampling_logp_difference/mean": 0.25110191106796265, "step": 102, "step_time": 10.583589826994285 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.03125, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 47.03125, "completions/mean_terminated_length": 48.032257080078125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27224164456129074, "epoch": 0.00206, "frac_reward_zero_std": 0.0, "grad_norm": 0.5115644335746765, "kl": 3.3863462768495083, "learning_rate": 9.889522367416332e-06, "loss": 0.0338, "num_tokens": 4620500.0, "reward": 0.07187499105930328, "reward_std": 1.0528455972671509, "rewards/rollout_reward_func/mean": 0.07187499105930328, "rewards/rollout_reward_func/std": 1.0229730606079102, "sampling/importance_sampling_ratio/max": 2.0351221561431885, "sampling/importance_sampling_ratio/mean": 0.7632189989089966, "sampling/importance_sampling_ratio/min": 0.10083232820034027, "sampling/sampling_logp_difference/max": 1.8964018821716309, "sampling/sampling_logp_difference/mean": 0.26232925057411194, "step": 103, "step_time": 12.22992458500812 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 37.625, "completions/mean_terminated_length": 37.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.227071700617671, "epoch": 0.00208, "frac_reward_zero_std": 0.0, "grad_norm": 0.3304665684700012, "kl": 3.3552427291870117, "learning_rate": 9.886216819972226e-06, "loss": 0.0583, "num_tokens": 4665242.0, "reward": -0.24250000715255737, "reward_std": 1.0074068307876587, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.7746691703796387, "sampling/importance_sampling_ratio/mean": 0.8196139931678772, "sampling/importance_sampling_ratio/min": 0.094782255589962, "sampling/sampling_logp_difference/max": 1.872551441192627, "sampling/sampling_logp_difference/mean": 0.2469957172870636, "step": 104, "step_time": 10.795023541017144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 518.0, "completions/max_terminated_length": 518.0, "completions/mean_length": 44.1875, "completions/mean_terminated_length": 44.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19915345311164856, "epoch": 0.0021, "frac_reward_zero_std": 0.0, "grad_norm": 0.16865089535713196, "kl": 2.4443675596266985, "learning_rate": 9.882863308370175e-06, "loss": -0.0182, "num_tokens": 4710833.0, "reward": -0.3056250214576721, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 2.235647439956665, "sampling/importance_sampling_ratio/mean": 1.1281330585479736, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.663675308227539, "sampling/sampling_logp_difference/mean": 0.11810535192489624, "step": 105, "step_time": 11.772750498988898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 57.75, "completions/mean_terminated_length": 57.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.272039738483727, "epoch": 0.00212, "frac_reward_zero_std": 0.0, "grad_norm": 0.7141158580780029, "kl": 2.5134409684687853, "learning_rate": 9.879461876845255e-06, "loss": 0.0581, "num_tokens": 4756490.0, "reward": -0.18000000715255737, "reward_std": 1.0167031288146973, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 2.013583183288574, "sampling/importance_sampling_ratio/mean": 0.7953761219978333, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.163804769515991, "sampling/sampling_logp_difference/mean": 0.23695069551467896, "step": 106, "step_time": 11.599184758968477 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 626.0, "completions/max_terminated_length": 626.0, "completions/mean_length": 71.9375, "completions/mean_terminated_length": 71.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28179579600691795, "epoch": 0.00214, "frac_reward_zero_std": 0.0, "grad_norm": 0.49531349539756775, "kl": 1.7788424789905548, "learning_rate": 9.876012570264658e-06, "loss": 0.072, "num_tokens": 4802406.0, "reward": 0.009999990463256836, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 2.185286521911621, "sampling/importance_sampling_ratio/mean": 0.8717455863952637, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4053983688354492, "sampling/sampling_logp_difference/mean": 0.16888338327407837, "step": 107, "step_time": 11.696548897998582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 55.5625, "completions/mean_terminated_length": 55.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3542287587188184, "epoch": 0.00216, "frac_reward_zero_std": 0.0, "grad_norm": 0.23626558482646942, "kl": 1.370000183582306, "learning_rate": 9.872515434127064e-06, "loss": 0.0892, "num_tokens": 4847971.0, "reward": -0.24312500655651093, "reward_std": 1.0062534809112549, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 2.615412712097168, "sampling/importance_sampling_ratio/mean": 0.9008859395980835, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.15913724899292, "sampling/sampling_logp_difference/mean": 0.19387298822402954, "step": 108, "step_time": 11.95831564701075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 149.125, "completions/mean_terminated_length": 149.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37219258211553097, "epoch": 0.00218, "frac_reward_zero_std": 0.0, "grad_norm": 0.8568993806838989, "kl": 2.1927274130284786, "learning_rate": 9.86897051456206e-06, "loss": 0.0229, "num_tokens": 4896790.0, "reward": -0.053125008940696716, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0708640813827515, "sampling/importance_sampling_ratio/mean": 0.7352132797241211, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.324184775352478, "sampling/sampling_logp_difference/mean": 0.15352877974510193, "step": 109, "step_time": 12.288820362991828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 93.625, "completions/mean_terminated_length": 93.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5249111074954271, "epoch": 0.0022, "frac_reward_zero_std": 0.0, "grad_norm": 0.30540695786476135, "kl": 1.3222577441483736, "learning_rate": 9.865377858329539e-06, "loss": -0.0159, "num_tokens": 4944713.0, "reward": -0.3687500059604645, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.0548564195632935, "sampling/importance_sampling_ratio/mean": 0.741305947303772, "sampling/importance_sampling_ratio/min": 0.23253090679645538, "sampling/sampling_logp_difference/max": 1.0632665157318115, "sampling/sampling_logp_difference/mean": 0.19340813159942627, "step": 110, "step_time": 11.850246708017949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 49.90625, "completions/mean_terminated_length": 49.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37584489211440086, "epoch": 0.00222, "frac_reward_zero_std": 0.0, "grad_norm": 0.8776552677154541, "kl": 2.5413450691848993, "learning_rate": 9.861737512819058e-06, "loss": 0.0013, "num_tokens": 4989086.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.5133428573608398, "sampling/importance_sampling_ratio/mean": 0.8844367265701294, "sampling/importance_sampling_ratio/min": 0.3338417708873749, "sampling/sampling_logp_difference/max": 0.98297119140625, "sampling/sampling_logp_difference/mean": 0.13421420753002167, "step": 111, "step_time": 11.861282552017656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 36.84375, "completions/mean_terminated_length": 36.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3077101525850594, "epoch": 0.00224, "frac_reward_zero_std": 0.0, "grad_norm": 0.32781490683555603, "kl": 1.6339647620916367, "learning_rate": 9.858049526049241e-06, "loss": -0.0193, "num_tokens": 5032374.0, "reward": 0.009999990463256836, "reward_std": 1.0074068307876587, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.1319823265075684, "sampling/importance_sampling_ratio/mean": 0.8479960560798645, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.085207223892212, "sampling/sampling_logp_difference/mean": 0.12672817707061768, "step": 112, "step_time": 11.55301059399062 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 25.21875, "completions/mean_terminated_length": 25.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4774694303050637, "epoch": 0.00226, "frac_reward_zero_std": 0.0, "grad_norm": 0.5268510580062866, "kl": 0.8076906278729439, "learning_rate": 9.85431394666712e-06, "loss": -0.0112, "num_tokens": 5077948.0, "reward": -0.49562501907348633, "reward_std": 0.9062924981117249, "rewards/rollout_reward_func/mean": -0.49562501907348633, "rewards/rollout_reward_func/std": 0.8875880241394043, "sampling/importance_sampling_ratio/max": 1.0911785364151, "sampling/importance_sampling_ratio/mean": 0.8362442255020142, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4037001132965088, "sampling/sampling_logp_difference/mean": 0.14267539978027344, "step": 113, "step_time": 10.900702321006975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 89.5625, "completions/mean_terminated_length": 89.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4378659795038402, "epoch": 0.00228, "frac_reward_zero_std": 0.0, "grad_norm": 0.39807409048080444, "kl": 2.747623836621642, "learning_rate": 9.850530823947519e-06, "loss": -0.0167, "num_tokens": 5123866.0, "reward": -0.3687500059604645, "reward_std": 0.9626314640045166, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.0526474714279175, "sampling/importance_sampling_ratio/mean": 0.7074549198150635, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2166171073913574, "sampling/sampling_logp_difference/mean": 0.19656577706336975, "step": 114, "step_time": 12.505495866018464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 48.375, "completions/mean_terminated_length": 48.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3528332784771919, "epoch": 0.0023, "frac_reward_zero_std": 0.0, "grad_norm": 0.4329725503921509, "kl": 1.1773086618632078, "learning_rate": 9.846700207792381e-06, "loss": -0.0063, "num_tokens": 5168676.0, "reward": -0.24312500655651093, "reward_std": 0.9791113138198853, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 2.785158634185791, "sampling/importance_sampling_ratio/mean": 0.8373759388923645, "sampling/importance_sampling_ratio/min": 0.09064503759145737, "sampling/sampling_logp_difference/max": 1.8812295198440552, "sampling/sampling_logp_difference/mean": 0.21614298224449158, "step": 115, "step_time": 13.478201300975343 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 51.1875, "completions/mean_terminated_length": 51.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20618777116760612, "epoch": 0.00232, "frac_reward_zero_std": 0.0, "grad_norm": 1.1802395582199097, "kl": 1.1905044447630644, "learning_rate": 9.842822148730122e-06, "loss": 0.0651, "num_tokens": 5213468.0, "reward": 0.13624998927116394, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.302466630935669, "sampling/importance_sampling_ratio/mean": 1.286996841430664, "sampling/importance_sampling_ratio/min": 0.33089911937713623, "sampling/sampling_logp_difference/max": 1.208604097366333, "sampling/sampling_logp_difference/mean": 0.08306226134300232, "step": 116, "step_time": 11.64494886398461 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1060.0, "completions/max_terminated_length": 1060.0, "completions/mean_length": 74.03125, "completions/mean_terminated_length": 74.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18531569093465805, "epoch": 0.00234, "frac_reward_zero_std": 0.0, "grad_norm": 0.6382340788841248, "kl": 2.2594538293778896, "learning_rate": 9.838896697914964e-06, "loss": 0.1262, "num_tokens": 5258056.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.7555502653121948, "sampling/importance_sampling_ratio/mean": 1.0042020082473755, "sampling/importance_sampling_ratio/min": 0.08414474129676819, "sampling/sampling_logp_difference/max": 2.2823116779327393, "sampling/sampling_logp_difference/mean": 0.1047881543636322, "step": 117, "step_time": 15.19964321700536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 646.0, "completions/max_terminated_length": 646.0, "completions/mean_length": 48.0, "completions/mean_terminated_length": 48.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2524274163879454, "epoch": 0.00236, "frac_reward_zero_std": 0.25, "grad_norm": 0.45628973841667175, "kl": 2.0992937851697206, "learning_rate": 9.834923907126256e-06, "loss": 0.0583, "num_tokens": 5303815.0, "reward": -0.4950000047683716, "reward_std": 0.7564942836761475, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 1.7195571660995483, "sampling/importance_sampling_ratio/mean": 1.0690832138061523, "sampling/importance_sampling_ratio/min": 0.13204075396060944, "sampling/sampling_logp_difference/max": 1.3531625270843506, "sampling/sampling_logp_difference/mean": 0.1074824258685112, "step": 118, "step_time": 11.824089354020543 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 563.0, "completions/max_terminated_length": 563.0, "completions/mean_length": 36.75, "completions/mean_terminated_length": 36.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21259491983801126, "epoch": 0.00238, "frac_reward_zero_std": 0.0, "grad_norm": 0.42844924330711365, "kl": 4.084874643012881, "learning_rate": 9.830903828767796e-06, "loss": 0.1002, "num_tokens": 5347110.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.6037057638168335, "sampling/importance_sampling_ratio/mean": 0.96025151014328, "sampling/importance_sampling_ratio/min": 0.09635412693023682, "sampling/sampling_logp_difference/max": 1.4653894901275635, "sampling/sampling_logp_difference/mean": 0.12620049715042114, "step": 119, "step_time": 11.077849383989815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 357.0, "completions/max_terminated_length": 357.0, "completions/mean_length": 30.75, "completions/mean_terminated_length": 30.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2059748349711299, "epoch": 0.0024, "frac_reward_zero_std": 0.0, "grad_norm": 0.4314999580383301, "kl": 1.550351794809103, "learning_rate": 9.826836515867131e-06, "loss": 0.1231, "num_tokens": 5392271.0, "reward": -0.3056250214576721, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.59015691280365, "sampling/importance_sampling_ratio/mean": 1.0320637226104736, "sampling/importance_sampling_ratio/min": 0.0712207481265068, "sampling/sampling_logp_difference/max": 1.4650336503982544, "sampling/sampling_logp_difference/mean": 0.10363679379224777, "step": 120, "step_time": 11.156794940026884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 566.0, "completions/max_terminated_length": 566.0, "completions/mean_length": 54.6875, "completions/mean_terminated_length": 54.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2599881375208497, "epoch": 0.00242, "frac_reward_zero_std": 0.0, "grad_norm": 0.46165063977241516, "kl": 2.211166499182582, "learning_rate": 9.822722022074871e-06, "loss": -0.097, "num_tokens": 5437695.0, "reward": -0.18062500655651093, "reward_std": 0.9338013529777527, "rewards/rollout_reward_func/mean": -0.18062500655651093, "rewards/rollout_reward_func/std": 1.006437063217163, "sampling/importance_sampling_ratio/max": 2.469852924346924, "sampling/importance_sampling_ratio/mean": 1.019798994064331, "sampling/importance_sampling_ratio/min": 0.1831744909286499, "sampling/sampling_logp_difference/max": 1.3515386581420898, "sampling/sampling_logp_difference/mean": 0.14784559607505798, "step": 121, "step_time": 11.738479763000214 }, { "clip_ratio/high_max": 0.003289473708719015, "clip_ratio/high_mean": 0.0016447368543595076, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016447368543595076, "completions/clipped_ratio": 0.0, "completions/max_length": 995.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 86.375, "completions/mean_terminated_length": 86.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2453760397620499, "epoch": 0.00244, "frac_reward_zero_std": 0.0, "grad_norm": 0.7551835775375366, "kl": 1.6317713260650635, "learning_rate": 9.818560401663972e-06, "loss": 0.0214, "num_tokens": 5484587.0, "reward": -0.05375000834465027, "reward_std": 0.9148640632629395, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.8567055463790894, "sampling/importance_sampling_ratio/mean": 1.0583664178848267, "sampling/importance_sampling_ratio/min": 0.27397245168685913, "sampling/sampling_logp_difference/max": 1.3889257907867432, "sampling/sampling_logp_difference/mean": 0.11466899514198303, "step": 122, "step_time": 14.084691649979504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 95.6875, "completions/mean_terminated_length": 95.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2695980276912451, "epoch": 0.00246, "frac_reward_zero_std": 0.0, "grad_norm": 0.4494613707065582, "kl": 1.227709675207734, "learning_rate": 9.814351709529018e-06, "loss": 0.1161, "num_tokens": 5530686.0, "reward": -0.11812500655651093, "reward_std": 1.0415849685668945, "rewards/rollout_reward_func/mean": -0.11812500655651093, "rewards/rollout_reward_func/std": 1.0159801244735718, "sampling/importance_sampling_ratio/max": 2.0666143894195557, "sampling/importance_sampling_ratio/mean": 0.9471431970596313, "sampling/importance_sampling_ratio/min": 0.21053583920001984, "sampling/sampling_logp_difference/max": 1.2048933506011963, "sampling/sampling_logp_difference/mean": 0.12260060012340546, "step": 123, "step_time": 14.60425836801005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 64.78125, "completions/mean_terminated_length": 64.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23751549562439322, "epoch": 0.00248, "frac_reward_zero_std": 0.0, "grad_norm": 0.3557853400707245, "kl": 1.4206101652234793, "learning_rate": 9.8100960011855e-06, "loss": 0.1151, "num_tokens": 5575946.0, "reward": -0.18000000715255737, "reward_std": 1.016998529434204, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.8330148458480835, "sampling/importance_sampling_ratio/mean": 1.006387710571289, "sampling/importance_sampling_ratio/min": 0.17297130823135376, "sampling/sampling_logp_difference/max": 1.3445045948028564, "sampling/sampling_logp_difference/mean": 0.13639086484909058, "step": 124, "step_time": 13.38598619899858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 689.0, "completions/max_terminated_length": 689.0, "completions/mean_length": 86.5625, "completions/mean_terminated_length": 86.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26385422283783555, "epoch": 0.0025, "frac_reward_zero_std": 0.0, "grad_norm": 0.27940279245376587, "kl": 1.6837775371968746, "learning_rate": 9.805793332769095e-06, "loss": 0.032, "num_tokens": 5622304.0, "reward": -0.17937500774860382, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.322218179702759, "sampling/importance_sampling_ratio/mean": 0.9052063822746277, "sampling/importance_sampling_ratio/min": 0.24483010172843933, "sampling/sampling_logp_difference/max": 1.1316652297973633, "sampling/sampling_logp_difference/mean": 0.16086196899414062, "step": 125, "step_time": 12.382617391005624 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 37.53125, "completions/mean_terminated_length": 37.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32560787815600634, "epoch": 0.00252, "frac_reward_zero_std": 0.0, "grad_norm": 0.2996266186237335, "kl": 2.7286181524395943, "learning_rate": 9.801443761034903e-06, "loss": -0.0195, "num_tokens": 5666872.0, "reward": -0.17937500774860382, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.2323338985443115, "sampling/importance_sampling_ratio/mean": 0.8405665159225464, "sampling/importance_sampling_ratio/min": 0.09107934683561325, "sampling/sampling_logp_difference/max": 1.3470892906188965, "sampling/sampling_logp_difference/mean": 0.2130303978919983, "step": 126, "step_time": 11.161363475999678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.009765625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009765625, "completions/clipped_ratio": 0.03125, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 82.34375, "completions/mean_terminated_length": 84.48387145996094, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3307203147560358, "epoch": 0.00254, "frac_reward_zero_std": 0.0, "grad_norm": 0.480472207069397, "kl": 1.2242956943809986, "learning_rate": 9.797047343356718e-06, "loss": 0.007, "num_tokens": 5712999.0, "reward": 0.13624998927116394, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.4301207065582275, "sampling/importance_sampling_ratio/mean": 0.869860053062439, "sampling/importance_sampling_ratio/min": 0.23007239401340485, "sampling/sampling_logp_difference/max": 1.5741949081420898, "sampling/sampling_logp_difference/mean": 0.12969690561294556, "step": 127, "step_time": 11.416859739001666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 79.3125, "completions/mean_terminated_length": 79.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2841349449008703, "epoch": 0.00256, "frac_reward_zero_std": 0.0, "grad_norm": 0.7643325924873352, "kl": 3.847316777333617, "learning_rate": 9.792604137726259e-06, "loss": 0.0466, "num_tokens": 5760139.0, "reward": -0.5581250190734863, "reward_std": 0.8522208333015442, "rewards/rollout_reward_func/mean": -0.5581250190734863, "rewards/rollout_reward_func/std": 0.8484271168708801, "sampling/importance_sampling_ratio/max": 1.9410697221755981, "sampling/importance_sampling_ratio/mean": 0.9128574132919312, "sampling/importance_sampling_ratio/min": 0.09054891765117645, "sampling/sampling_logp_difference/max": 1.5627570152282715, "sampling/sampling_logp_difference/mean": 0.15728695690631866, "step": 128, "step_time": 11.775357975020597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 63.65625, "completions/mean_terminated_length": 63.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3892551213502884, "epoch": 0.00258, "frac_reward_zero_std": 0.0, "grad_norm": 0.802656888961792, "kl": 1.1098022684454918, "learning_rate": 9.788114202752415e-06, "loss": -0.0494, "num_tokens": 5805142.0, "reward": -0.11625000834465027, "reward_std": 1.0435711145401, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.8889260292053223, "sampling/importance_sampling_ratio/mean": 0.9514934420585632, "sampling/importance_sampling_ratio/min": 0.3037758469581604, "sampling/sampling_logp_difference/max": 0.9286909103393555, "sampling/sampling_logp_difference/mean": 0.14835013449192047, "step": 129, "step_time": 12.79046437900979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 671.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 108.4375, "completions/mean_terminated_length": 108.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3024310292676091, "epoch": 0.0026, "frac_reward_zero_std": 0.0, "grad_norm": 1.200369954109192, "kl": 2.44382019713521, "learning_rate": 9.78357759766046e-06, "loss": 0.0011, "num_tokens": 5851150.0, "reward": 0.009374991059303284, "reward_std": 1.0257649421691895, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 2.6161069869995117, "sampling/importance_sampling_ratio/mean": 1.036661148071289, "sampling/importance_sampling_ratio/min": 0.26789751648902893, "sampling/sampling_logp_difference/max": 1.1175562143325806, "sampling/sampling_logp_difference/mean": 0.1354573667049408, "step": 130, "step_time": 12.08522123498551 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 178.40625, "completions/mean_terminated_length": 178.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4303685035556555, "epoch": 0.00262, "frac_reward_zero_std": 0.0, "grad_norm": 0.39271625876426697, "kl": 2.552606862038374, "learning_rate": 9.778994382291283e-06, "loss": -0.0748, "num_tokens": 5903272.0, "reward": -0.43187499046325684, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 2.7264161109924316, "sampling/importance_sampling_ratio/mean": 0.7002993822097778, "sampling/importance_sampling_ratio/min": 0.0739365741610527, "sampling/sampling_logp_difference/max": 2.600172281265259, "sampling/sampling_logp_difference/mean": 0.27565237879753113, "step": 131, "step_time": 15.37801430401305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 653.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 47.53125, "completions/mean_terminated_length": 47.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.310549832880497, "epoch": 0.00264, "frac_reward_zero_std": 0.0, "grad_norm": 0.7810649871826172, "kl": 1.5843484364449978, "learning_rate": 9.774364617100596e-06, "loss": 0.0654, "num_tokens": 5947975.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 2.627734661102295, "sampling/importance_sampling_ratio/mean": 0.9406154155731201, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5751612186431885, "sampling/sampling_logp_difference/mean": 0.17543406784534454, "step": 132, "step_time": 11.860382834005577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 437.0, "completions/max_terminated_length": 437.0, "completions/mean_length": 66.78125, "completions/mean_terminated_length": 66.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2938839136622846, "epoch": 0.00266, "frac_reward_zero_std": 0.0, "grad_norm": 1.0023372173309326, "kl": 1.361091861501336, "learning_rate": 9.769688363158127e-06, "loss": 0.1023, "num_tokens": 5993551.0, "reward": 0.07249999046325684, "reward_std": 1.0531628131866455, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 2.5563645362854004, "sampling/importance_sampling_ratio/mean": 1.0014188289642334, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5521126985549927, "sampling/sampling_logp_difference/mean": 0.1569703221321106, "step": 133, "step_time": 10.905057140989811 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 401.0, "completions/max_terminated_length": 401.0, "completions/mean_length": 33.71875, "completions/mean_terminated_length": 33.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28926934814080596, "epoch": 0.00268, "frac_reward_zero_std": 0.0, "grad_norm": 1.3987756967544556, "kl": 1.2438727132976055, "learning_rate": 9.76496568214683e-06, "loss": 0.1107, "num_tokens": 6037766.0, "reward": 0.009999990463256836, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 2.5166683197021484, "sampling/importance_sampling_ratio/mean": 1.034893274307251, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9967845678329468, "sampling/sampling_logp_difference/mean": 0.16589856147766113, "step": 134, "step_time": 12.238847836008063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 102.03125, "completions/mean_terminated_length": 102.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3181192148476839, "epoch": 0.0027, "frac_reward_zero_std": 0.0, "grad_norm": 0.2547202408313751, "kl": 1.7958272360265255, "learning_rate": 9.760196636362058e-06, "loss": -0.0041, "num_tokens": 6085318.0, "reward": -0.17937500774860382, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.4926581382751465, "sampling/importance_sampling_ratio/mean": 0.8735089898109436, "sampling/importance_sampling_ratio/min": 0.1038321778178215, "sampling/sampling_logp_difference/max": 1.4157356023788452, "sampling/sampling_logp_difference/mean": 0.13989882171154022, "step": 135, "step_time": 11.767091369008995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 142.375, "completions/mean_terminated_length": 142.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23695008945651352, "epoch": 0.00272, "frac_reward_zero_std": 0.0, "grad_norm": 0.45866093039512634, "kl": 1.2066212426871061, "learning_rate": 9.755381288710751e-06, "loss": -0.0049, "num_tokens": 6132651.0, "reward": -0.11750000715255737, "reward_std": 1.0444223880767822, "rewards/rollout_reward_func/mean": -0.11750000715255737, "rewards/rollout_reward_func/std": 1.0166834592819214, "sampling/importance_sampling_ratio/max": 1.3376466035842896, "sampling/importance_sampling_ratio/mean": 0.857162594795227, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.351970911026001, "sampling/sampling_logp_difference/mean": 0.10474840551614761, "step": 136, "step_time": 13.158121502005088 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 68.90625, "completions/mean_terminated_length": 68.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2194082154892385, "epoch": 0.00274, "frac_reward_zero_std": 0.0, "grad_norm": 0.3399125933647156, "kl": 1.582700727507472, "learning_rate": 9.750519702710598e-06, "loss": 0.027, "num_tokens": 6178056.0, "reward": -0.18000000715255737, "reward_std": 0.970690906047821, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.2727537155151367, "sampling/importance_sampling_ratio/mean": 0.9772036671638489, "sampling/importance_sampling_ratio/min": 0.6319042444229126, "sampling/sampling_logp_difference/max": 0.8376603126525879, "sampling/sampling_logp_difference/mean": 0.0609738752245903, "step": 137, "step_time": 12.76087806999567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 53.125, "completions/mean_terminated_length": 53.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19577912148088217, "epoch": 0.00276, "frac_reward_zero_std": 0.0, "grad_norm": 0.22743730247020721, "kl": 1.513700246810913, "learning_rate": 9.745611942489202e-06, "loss": -0.0168, "num_tokens": 6223146.0, "reward": -0.11625000834465027, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0434455871582031, "sampling/importance_sampling_ratio/mean": 0.9537824392318726, "sampling/importance_sampling_ratio/min": 0.18954165279865265, "sampling/sampling_logp_difference/max": 1.3247261047363281, "sampling/sampling_logp_difference/mean": 0.0726810172200203, "step": 138, "step_time": 11.505913105000218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 35.0625, "completions/mean_terminated_length": 35.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18560637277550995, "epoch": 0.00278, "frac_reward_zero_std": 0.0, "grad_norm": 0.09234777092933655, "kl": 0.725341685116291, "learning_rate": 9.740658072783244e-06, "loss": -0.0027, "num_tokens": 6267891.0, "reward": -0.3687500059604645, "reward_std": 0.9074459075927734, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.0984472036361694, "sampling/importance_sampling_ratio/mean": 1.0010590553283691, "sampling/importance_sampling_ratio/min": 0.5643281936645508, "sampling/sampling_logp_difference/max": 0.9496660232543945, "sampling/sampling_logp_difference/mean": 0.04170164093375206, "step": 139, "step_time": 10.781692250995548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 63.25, "completions/mean_terminated_length": 63.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17877088580280542, "epoch": 0.0028, "frac_reward_zero_std": 0.0, "grad_norm": 0.6957454085350037, "kl": 3.7256632018834352, "learning_rate": 9.73565815893761e-06, "loss": -0.0016, "num_tokens": 6313621.0, "reward": -0.24312500655651093, "reward_std": 0.9894057512283325, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 2.952294111251831, "sampling/importance_sampling_ratio/mean": 1.0025808811187744, "sampling/importance_sampling_ratio/min": 0.22998271882534027, "sampling/sampling_logp_difference/max": 1.4691729545593262, "sampling/sampling_logp_difference/mean": 0.09014210104942322, "step": 140, "step_time": 14.498936352989404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 43.71875, "completions/mean_terminated_length": 43.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18275485560297966, "epoch": 0.00282, "frac_reward_zero_std": 0.0, "grad_norm": 0.1272275596857071, "kl": 1.4700044970959425, "learning_rate": 9.730612266904548e-06, "loss": -0.0098, "num_tokens": 6359105.0, "reward": -0.4950000047683716, "reward_std": 0.9074459075927734, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 1.1065235137939453, "sampling/importance_sampling_ratio/mean": 0.9884523749351501, "sampling/importance_sampling_ratio/min": 0.455690860748291, "sampling/sampling_logp_difference/max": 0.7262029647827148, "sampling/sampling_logp_difference/mean": 0.05001436918973923, "step": 141, "step_time": 11.53119324601721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 84.46875, "completions/mean_terminated_length": 84.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1638767346739769, "epoch": 0.00284, "frac_reward_zero_std": 0.0, "grad_norm": 0.19835495948791504, "kl": 0.7126652039587498, "learning_rate": 9.72552046324278e-06, "loss": 0.0031, "num_tokens": 6404598.0, "reward": -0.24250000715255737, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.3041815757751465, "sampling/importance_sampling_ratio/mean": 1.038973331451416, "sampling/importance_sampling_ratio/min": 0.9886639714241028, "sampling/sampling_logp_difference/max": 0.23390287160873413, "sampling/sampling_logp_difference/mean": 0.023053281009197235, "step": 142, "step_time": 12.112229278995073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 69.40625, "completions/mean_terminated_length": 69.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20867924485355616, "epoch": 0.00286, "frac_reward_zero_std": 0.0, "grad_norm": 0.28085410594940186, "kl": 1.5808676928281784, "learning_rate": 9.72038281511664e-06, "loss": -0.0225, "num_tokens": 6450754.0, "reward": -0.11625000834465027, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1178375482559204, "sampling/importance_sampling_ratio/mean": 0.9422897100448608, "sampling/importance_sampling_ratio/min": 0.36084070801734924, "sampling/sampling_logp_difference/max": 0.9531655311584473, "sampling/sampling_logp_difference/mean": 0.07105104625225067, "step": 143, "step_time": 12.790343963017222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 66.90625, "completions/mean_terminated_length": 66.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17012207605876029, "epoch": 0.00288, "frac_reward_zero_std": 0.0, "grad_norm": 0.18283578753471375, "kl": 0.6266191862523556, "learning_rate": 9.715199390295181e-06, "loss": 0.0016, "num_tokens": 6496485.0, "reward": -0.3056250214576721, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.94219172000885, "sampling/importance_sampling_ratio/mean": 1.0339903831481934, "sampling/importance_sampling_ratio/min": 0.3778478801250458, "sampling/sampling_logp_difference/max": 0.8216211795806885, "sampling/sampling_logp_difference/mean": 0.04430244863033295, "step": 144, "step_time": 11.612769864004804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 77.28125, "completions/mean_terminated_length": 77.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1968524237163365, "epoch": 0.0029, "frac_reward_zero_std": 0.0, "grad_norm": 0.34051626920700073, "kl": 0.602449607104063, "learning_rate": 9.70997025715128e-06, "loss": -0.0179, "num_tokens": 6543431.0, "reward": -0.11625000834465027, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0780783891677856, "sampling/importance_sampling_ratio/mean": 1.0084786415100098, "sampling/importance_sampling_ratio/min": 0.36615610122680664, "sampling/sampling_logp_difference/max": 0.9361898899078369, "sampling/sampling_logp_difference/mean": 0.043632447719573975, "step": 145, "step_time": 14.287368501987658 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 132.90625, "completions/mean_terminated_length": 132.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1564204185269773, "epoch": 0.00292, "frac_reward_zero_std": 0.0, "grad_norm": 0.29423171281814575, "kl": 0.7174077136442065, "learning_rate": 9.704695484660736e-06, "loss": -0.0115, "num_tokens": 6592003.0, "reward": -0.11625000834465027, "reward_std": 0.9712425470352173, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.170998454093933, "sampling/importance_sampling_ratio/mean": 0.9889670014381409, "sampling/importance_sampling_ratio/min": 0.3968275189399719, "sampling/sampling_logp_difference/max": 0.8324196338653564, "sampling/sampling_logp_difference/mean": 0.04503291845321655, "step": 146, "step_time": 13.410187317997043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 669.0, "completions/max_terminated_length": 669.0, "completions/mean_length": 46.71875, "completions/mean_terminated_length": 46.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23946205293759704, "epoch": 0.00294, "frac_reward_zero_std": 0.0, "grad_norm": 0.49801698327064514, "kl": 2.7128776190802455, "learning_rate": 9.699375142401365e-06, "loss": 0.0238, "num_tokens": 6635599.0, "reward": -0.3062500059604645, "reward_std": 0.9976819157600403, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 1.3567233085632324, "sampling/importance_sampling_ratio/mean": 0.9486961960792542, "sampling/importance_sampling_ratio/min": 0.3396696448326111, "sampling/sampling_logp_difference/max": 1.0798983573913574, "sampling/sampling_logp_difference/mean": 0.07685984671115875, "step": 147, "step_time": 11.579516266974679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 70.6875, "completions/mean_terminated_length": 70.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.149948158999905, "epoch": 0.00296, "frac_reward_zero_std": 0.0, "grad_norm": 0.0923597514629364, "kl": 0.6699654124677181, "learning_rate": 9.694009300552081e-06, "loss": -0.0137, "num_tokens": 6681718.0, "reward": -0.18000000715255737, "reward_std": 0.9983232021331787, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.1022456884384155, "sampling/importance_sampling_ratio/mean": 1.0058543682098389, "sampling/importance_sampling_ratio/min": 0.4625107944011688, "sampling/sampling_logp_difference/max": 0.7031056880950928, "sampling/sampling_logp_difference/mean": 0.03964696079492569, "step": 148, "step_time": 12.06831062198762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.0, "completions/max_terminated_length": 303.0, "completions/mean_length": 32.875, "completions/mean_terminated_length": 32.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15304092620499432, "epoch": 0.00298, "frac_reward_zero_std": 0.0, "grad_norm": 0.18771076202392578, "kl": 0.7898096516728401, "learning_rate": 9.68859802989196e-06, "loss": -0.0186, "num_tokens": 6726240.0, "reward": 0.009374991059303284, "reward_std": 1.0260486602783203, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.2173141241073608, "sampling/importance_sampling_ratio/mean": 1.032132625579834, "sampling/importance_sampling_ratio/min": 0.5564842820167542, "sampling/sampling_logp_difference/max": 0.5726118087768555, "sampling/sampling_logp_difference/mean": 0.030189834535121918, "step": 149, "step_time": 10.806676811989746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 64.5, "completions/mean_terminated_length": 64.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16663940390571952, "epoch": 0.003, "frac_reward_zero_std": 0.0, "grad_norm": 0.10819780081510544, "kl": 1.7372485473752022, "learning_rate": 9.683141401799326e-06, "loss": -0.0134, "num_tokens": 6772865.0, "reward": -0.36937499046325684, "reward_std": 0.9894058108329773, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.308474063873291, "sampling/importance_sampling_ratio/mean": 1.028259038925171, "sampling/importance_sampling_ratio/min": 0.3306693732738495, "sampling/sampling_logp_difference/max": 1.1066389083862305, "sampling/sampling_logp_difference/mean": 0.04343872517347336, "step": 150, "step_time": 11.029696088997298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.0, "completions/max_terminated_length": 258.0, "completions/mean_length": 10.0, "completions/mean_terminated_length": 10.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13836692203767598, "epoch": 0.00302, "frac_reward_zero_std": 0.0, "grad_norm": 0.06550052762031555, "kl": 0.5668622124940157, "learning_rate": 9.677639488250787e-06, "loss": -0.0037, "num_tokens": 6815514.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.1151353120803833, "sampling/importance_sampling_ratio/mean": 1.0160069465637207, "sampling/importance_sampling_ratio/min": 0.4674533009529114, "sampling/sampling_logp_difference/max": 0.6909241676330566, "sampling/sampling_logp_difference/mean": 0.027889825403690338, "step": 151, "step_time": 11.004616280988557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 43.4375, "completions/mean_terminated_length": 43.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1406450578942895, "epoch": 0.00304, "frac_reward_zero_std": 0.0, "grad_norm": 0.09509263932704926, "kl": 0.9030255246907473, "learning_rate": 9.672092361820306e-06, "loss": -0.0033, "num_tokens": 6859559.0, "reward": 0.07312498986721039, "reward_std": 0.9521816968917847, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0795868635177612, "sampling/importance_sampling_ratio/mean": 1.014825701713562, "sampling/importance_sampling_ratio/min": 0.2665277123451233, "sampling/sampling_logp_difference/max": 1.3188133239746094, "sampling/sampling_logp_difference/mean": 0.03676149621605873, "step": 152, "step_time": 10.973286445987469 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 104.9375, "completions/mean_terminated_length": 104.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16936813993379474, "epoch": 0.00306, "frac_reward_zero_std": 0.0, "grad_norm": 0.17134757339954376, "kl": 1.3658365216106176, "learning_rate": 9.666500095678226e-06, "loss": -0.0337, "num_tokens": 6907771.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.5085872411727905, "sampling/importance_sampling_ratio/mean": 1.0463781356811523, "sampling/importance_sampling_ratio/min": 0.19290883839130402, "sampling/sampling_logp_difference/max": 1.7064685821533203, "sampling/sampling_logp_difference/mean": 0.050287358462810516, "step": 153, "step_time": 11.043110722013807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 75.3125, "completions/mean_terminated_length": 75.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16115697636269033, "epoch": 0.00308, "frac_reward_zero_std": 0.0, "grad_norm": 0.09350550919771194, "kl": 0.9284354243427515, "learning_rate": 9.660862763590322e-06, "loss": -0.0146, "num_tokens": 6953585.0, "reward": 0.009999990463256836, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.2177451848983765, "sampling/importance_sampling_ratio/mean": 1.0112842321395874, "sampling/importance_sampling_ratio/min": 0.18724435567855835, "sampling/sampling_logp_difference/max": 1.7319846153259277, "sampling/sampling_logp_difference/mean": 0.05531879514455795, "step": 154, "step_time": 11.850151051992725 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 76.4375, "completions/mean_terminated_length": 76.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22864535404369235, "epoch": 0.0031, "frac_reward_zero_std": 0.0, "grad_norm": 0.2044961005449295, "kl": 0.6720114685595036, "learning_rate": 9.655180439916814e-06, "loss": -0.0135, "num_tokens": 7000682.0, "reward": -0.3687500059604645, "reward_std": 0.9626314640045166, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.4970910549163818, "sampling/importance_sampling_ratio/mean": 1.0253026485443115, "sampling/importance_sampling_ratio/min": 0.43901917338371277, "sampling/sampling_logp_difference/max": 0.6821396350860596, "sampling/sampling_logp_difference/mean": 0.051706261932849884, "step": 155, "step_time": 11.579812521013082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 98.09375, "completions/mean_terminated_length": 98.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22030511684715748, "epoch": 0.00312, "frac_reward_zero_std": 0.0, "grad_norm": 0.18362043797969818, "kl": 0.957327913492918, "learning_rate": 9.64945319961139e-06, "loss": -0.0194, "num_tokens": 7048270.0, "reward": -0.18000000715255737, "reward_std": 0.970690906047821, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.4378794431686401, "sampling/importance_sampling_ratio/mean": 1.0118696689605713, "sampling/importance_sampling_ratio/min": 0.4320341944694519, "sampling/sampling_logp_difference/max": 0.8669414520263672, "sampling/sampling_logp_difference/mean": 0.05959641933441162, "step": 156, "step_time": 11.613237180004944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 93.5625, "completions/mean_terminated_length": 93.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1777551758568734, "epoch": 0.00314, "frac_reward_zero_std": 0.0, "grad_norm": 0.30354347825050354, "kl": 1.4406844954937696, "learning_rate": 9.643681118220224e-06, "loss": -0.0338, "num_tokens": 7093810.0, "reward": 0.1356249898672104, "reward_std": 0.9795305728912354, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.4799975156784058, "sampling/importance_sampling_ratio/mean": 1.0121450424194336, "sampling/importance_sampling_ratio/min": 0.5123579502105713, "sampling/sampling_logp_difference/max": 0.6101036071777344, "sampling/sampling_logp_difference/mean": 0.04989094287157059, "step": 157, "step_time": 12.520105024996155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 89.9375, "completions/mean_terminated_length": 89.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.179415684659034, "epoch": 0.00316, "frac_reward_zero_std": 0.0, "grad_norm": 0.32263660430908203, "kl": 0.7662702407687902, "learning_rate": 9.637864271880972e-06, "loss": -0.0042, "num_tokens": 7138114.0, "reward": 0.1993749886751175, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.1652387380599976, "sampling/importance_sampling_ratio/mean": 0.9885199069976807, "sampling/importance_sampling_ratio/min": 0.26910945773124695, "sampling/sampling_logp_difference/max": 1.3703057765960693, "sampling/sampling_logp_difference/mean": 0.053899772465229034, "step": 158, "step_time": 11.63319377899461 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0022321429569274187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022321429569274187, "completions/clipped_ratio": 0.0, "completions/max_length": 420.0, "completions/max_terminated_length": 420.0, "completions/mean_length": 28.0625, "completions/mean_terminated_length": 28.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2645487152040005, "epoch": 0.00318, "frac_reward_zero_std": 0.0, "grad_norm": 0.7267633676528931, "kl": 1.5122839026153088, "learning_rate": 9.632002737321768e-06, "loss": 0.0361, "num_tokens": 7182176.0, "reward": -0.11750000715255737, "reward_std": 1.044079303741455, "rewards/rollout_reward_func/mean": -0.11750000715255737, "rewards/rollout_reward_func/std": 1.0166834592819214, "sampling/importance_sampling_ratio/max": 2.0008225440979004, "sampling/importance_sampling_ratio/mean": 1.0088067054748535, "sampling/importance_sampling_ratio/min": 0.48906853795051575, "sampling/sampling_logp_difference/max": 0.6804680824279785, "sampling/sampling_logp_difference/mean": 0.07040542364120483, "step": 159, "step_time": 10.622248316009063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 59.09375, "completions/mean_terminated_length": 59.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23152804374694824, "epoch": 0.0032, "frac_reward_zero_std": 0.0, "grad_norm": 0.26960328221321106, "kl": 1.2351203337311745, "learning_rate": 9.626096591860215e-06, "loss": 0.0059, "num_tokens": 7227105.0, "reward": 0.009374991059303284, "reward_std": 0.9430981278419495, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.4851778745651245, "sampling/importance_sampling_ratio/mean": 1.0127692222595215, "sampling/importance_sampling_ratio/min": 0.5108656287193298, "sampling/sampling_logp_difference/max": 0.5599406957626343, "sampling/sampling_logp_difference/mean": 0.06667565554380417, "step": 160, "step_time": 12.40886264899018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 63.25, "completions/mean_terminated_length": 63.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19768974371254444, "epoch": 0.00322, "frac_reward_zero_std": 0.0, "grad_norm": 0.16992945969104767, "kl": 1.368023170158267, "learning_rate": 9.620145913402366e-06, "loss": 0.0016, "num_tokens": 7272492.0, "reward": -0.05375000834465027, "reward_std": 1.0535087585449219, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.3943607807159424, "sampling/importance_sampling_ratio/mean": 1.0259286165237427, "sampling/importance_sampling_ratio/min": 0.3882415294647217, "sampling/sampling_logp_difference/max": 0.9183120727539062, "sampling/sampling_logp_difference/mean": 0.04979971423745155, "step": 161, "step_time": 11.449598853985663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 43.90625, "completions/mean_terminated_length": 43.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20229384093545377, "epoch": 0.00324, "frac_reward_zero_std": 0.0, "grad_norm": 0.20290720462799072, "kl": 1.8924048468470573, "learning_rate": 9.614150780441687e-06, "loss": -0.0113, "num_tokens": 7316936.0, "reward": -0.18000000715255737, "reward_std": 1.0531628131866455, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.240686058998108, "sampling/importance_sampling_ratio/mean": 0.9873522520065308, "sampling/importance_sampling_ratio/min": 0.35065600275993347, "sampling/sampling_logp_difference/max": 1.0449576377868652, "sampling/sampling_logp_difference/mean": 0.06558661162853241, "step": 162, "step_time": 11.086146696005017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 32.75, "completions/mean_terminated_length": 32.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23011840134859085, "epoch": 0.00326, "frac_reward_zero_std": 0.0, "grad_norm": 0.09467589855194092, "kl": 1.0343072935938835, "learning_rate": 9.608111272058044e-06, "loss": -0.0109, "num_tokens": 7361343.0, "reward": -0.11625000834465027, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0897759199142456, "sampling/importance_sampling_ratio/mean": 0.9940451383590698, "sampling/importance_sampling_ratio/min": 0.483467161655426, "sampling/sampling_logp_difference/max": 0.6849894523620605, "sampling/sampling_logp_difference/mean": 0.05157332122325897, "step": 163, "step_time": 10.889539830015565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 56.0, "completions/mean_terminated_length": 56.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2703712470829487, "epoch": 0.00328, "frac_reward_zero_std": 0.0, "grad_norm": 0.145019069314003, "kl": 1.1335981599986553, "learning_rate": 9.602027467916626e-06, "loss": -0.0083, "num_tokens": 7407228.0, "reward": -0.3056250214576721, "reward_std": 0.9521816968917847, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.3762426376342773, "sampling/importance_sampling_ratio/mean": 0.9380123615264893, "sampling/importance_sampling_ratio/min": 0.23050075769424438, "sampling/sampling_logp_difference/max": 1.5291528701782227, "sampling/sampling_logp_difference/mean": 0.0996682196855545, "step": 164, "step_time": 11.499591143001453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 45.8125, "completions/mean_terminated_length": 45.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19443966378457844, "epoch": 0.0033, "frac_reward_zero_std": 0.0, "grad_norm": 0.11093419790267944, "kl": 2.8064672499895096, "learning_rate": 9.595899448266928e-06, "loss": -0.013, "num_tokens": 7451447.0, "reward": -0.05437500774860382, "reward_std": 1.0698374509811401, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228137969970703, "sampling/importance_sampling_ratio/max": 1.0978337526321411, "sampling/importance_sampling_ratio/mean": 0.9553065896034241, "sampling/importance_sampling_ratio/min": 0.41429468989372253, "sampling/sampling_logp_difference/max": 0.8571739196777344, "sampling/sampling_logp_difference/mean": 0.07189778238534927, "step": 165, "step_time": 12.049461559006886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 638.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 82.0625, "completions/mean_terminated_length": 82.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.223911291686818, "epoch": 0.00332, "frac_reward_zero_std": 0.0, "grad_norm": 0.20398372411727905, "kl": 1.5396906286478043, "learning_rate": 9.589727293941669e-06, "loss": 0.0032, "num_tokens": 7497323.0, "reward": -0.24312500655651093, "reward_std": 0.962119460105896, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.0880249738693237, "sampling/importance_sampling_ratio/mean": 0.9414925575256348, "sampling/importance_sampling_ratio/min": 0.15418414771556854, "sampling/sampling_logp_difference/max": 1.9471772909164429, "sampling/sampling_logp_difference/mean": 0.10329846292734146, "step": 166, "step_time": 11.894078454992268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 97.875, "completions/mean_terminated_length": 97.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23884312971495092, "epoch": 0.00334, "frac_reward_zero_std": 0.0, "grad_norm": 0.1466054618358612, "kl": 2.1778254453092813, "learning_rate": 9.583511086355738e-06, "loss": -0.028, "num_tokens": 7545484.0, "reward": -0.11687500774860382, "reward_std": 0.9792624711990356, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.1042231321334839, "sampling/importance_sampling_ratio/mean": 0.9563034772872925, "sampling/importance_sampling_ratio/min": 0.32431560754776, "sampling/sampling_logp_difference/max": 0.8616485595703125, "sampling/sampling_logp_difference/mean": 0.07701686024665833, "step": 167, "step_time": 11.648299982000026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 89.3125, "completions/mean_terminated_length": 89.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2438822011463344, "epoch": 0.00336, "frac_reward_zero_std": 0.25, "grad_norm": 0.1945052593946457, "kl": 0.8177800681442022, "learning_rate": 9.577250907505117e-06, "loss": 0.0059, "num_tokens": 7592440.0, "reward": -0.43187499046325684, "reward_std": 0.7650657892227173, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.1036708354949951, "sampling/importance_sampling_ratio/mean": 1.0188193321228027, "sampling/importance_sampling_ratio/min": 0.6341941356658936, "sampling/sampling_logp_difference/max": 0.5537281036376953, "sampling/sampling_logp_difference/mean": 0.04294922202825546, "step": 168, "step_time": 11.991455132992996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 77.96875, "completions/mean_terminated_length": 77.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17950818012468517, "epoch": 0.00338, "frac_reward_zero_std": 0.0, "grad_norm": 0.10232657194137573, "kl": 0.7292975429445505, "learning_rate": 9.570946839965795e-06, "loss": -0.008, "num_tokens": 7637491.0, "reward": 0.13624998927116394, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.3473666906356812, "sampling/importance_sampling_ratio/mean": 1.0443449020385742, "sampling/importance_sampling_ratio/min": 1.007833480834961, "sampling/sampling_logp_difference/max": 0.24698442220687866, "sampling/sampling_logp_difference/mean": 0.028370734304189682, "step": 169, "step_time": 11.871604741994815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 45.28125, "completions/mean_terminated_length": 45.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23735572514124215, "epoch": 0.0034, "frac_reward_zero_std": 0.0, "grad_norm": 0.3342973291873932, "kl": 1.3066176120191813, "learning_rate": 9.564598966892683e-06, "loss": 0.0081, "num_tokens": 7683332.0, "reward": -0.24312500655651093, "reward_std": 0.9791113138198853, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.7195813655853271, "sampling/importance_sampling_ratio/mean": 0.9974862337112427, "sampling/importance_sampling_ratio/min": 0.3579532504081726, "sampling/sampling_logp_difference/max": 0.8624615669250488, "sampling/sampling_logp_difference/mean": 0.07483388483524323, "step": 170, "step_time": 11.353526392005733 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 47.375, "completions/mean_terminated_length": 34.32257843017578, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32092579244636, "epoch": 0.00342, "frac_reward_zero_std": 0.0, "grad_norm": 0.6635550856590271, "kl": 1.1108260285109282, "learning_rate": 9.558207372018519e-06, "loss": -0.0493, "num_tokens": 7729449.0, "reward": -0.3056250214576721, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.1617138385772705, "sampling/importance_sampling_ratio/mean": 0.9522285461425781, "sampling/importance_sampling_ratio/min": 0.40248239040374756, "sampling/sampling_logp_difference/max": 0.8058264255523682, "sampling/sampling_logp_difference/mean": 0.06331977248191833, "step": 171, "step_time": 11.549744052004826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 41.75, "completions/mean_terminated_length": 41.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25001781713217497, "epoch": 0.00344, "frac_reward_zero_std": 0.0, "grad_norm": 0.2492365539073944, "kl": 1.4092204980552197, "learning_rate": 9.551772139652762e-06, "loss": -0.0088, "num_tokens": 7773994.0, "reward": -0.11625000834465027, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0583306550979614, "sampling/importance_sampling_ratio/mean": 0.9819716811180115, "sampling/importance_sampling_ratio/min": 0.40036049485206604, "sampling/sampling_logp_difference/max": 0.4747048616409302, "sampling/sampling_logp_difference/mean": 0.05422298610210419, "step": 172, "step_time": 11.392058987992641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 32.0625, "completions/mean_terminated_length": 32.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2945086150430143, "epoch": 0.00346, "frac_reward_zero_std": 0.0, "grad_norm": 0.18762686848640442, "kl": 0.6693445947021246, "learning_rate": 9.545293354680477e-06, "loss": -0.0117, "num_tokens": 7819463.0, "reward": -0.43187499046325684, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.0541151762008667, "sampling/importance_sampling_ratio/mean": 0.998418927192688, "sampling/importance_sampling_ratio/min": 0.5164200663566589, "sampling/sampling_logp_difference/max": 0.35945796966552734, "sampling/sampling_logp_difference/mean": 0.04705168306827545, "step": 173, "step_time": 10.990765487018507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.0, "completions/max_terminated_length": 303.0, "completions/mean_length": 20.28125, "completions/mean_terminated_length": 20.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19725448358803988, "epoch": 0.00348, "frac_reward_zero_std": 0.0, "grad_norm": 0.3156987428665161, "kl": 1.5826831925660372, "learning_rate": 9.538771102561212e-06, "loss": 0.0039, "num_tokens": 7862410.0, "reward": 0.009374991059303284, "reward_std": 0.9418424367904663, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 2.244860887527466, "sampling/importance_sampling_ratio/mean": 1.018629789352417, "sampling/importance_sampling_ratio/min": 0.4785510003566742, "sampling/sampling_logp_difference/max": 0.8968733549118042, "sampling/sampling_logp_difference/mean": 0.07054291665554047, "step": 174, "step_time": 11.255730594013585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 88.84375, "completions/mean_terminated_length": 88.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26764701306819916, "epoch": 0.0035, "frac_reward_zero_std": 0.0, "grad_norm": 0.6680815815925598, "kl": 0.9965672474354506, "learning_rate": 9.53220546932789e-06, "loss": 0.0493, "num_tokens": 7908112.0, "reward": 0.009374991059303284, "reward_std": 1.0449373722076416, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 2.603682518005371, "sampling/importance_sampling_ratio/mean": 0.9792587757110596, "sampling/importance_sampling_ratio/min": 8.627753511625258e-10, "sampling/sampling_logp_difference/max": 20.93366050720215, "sampling/sampling_logp_difference/mean": 0.26690974831581116, "step": 175, "step_time": 13.458277518009709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 438.0, "completions/max_terminated_length": 438.0, "completions/mean_length": 37.09375, "completions/mean_terminated_length": 37.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2983807139098644, "epoch": 0.00352, "frac_reward_zero_std": 0.0, "grad_norm": 0.28840360045433044, "kl": 1.8175300657749176, "learning_rate": 9.525596541585646e-06, "loss": 0.0005, "num_tokens": 7953039.0, "reward": -0.3056250214576721, "reward_std": 0.9521816968917847, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.8844797611236572, "sampling/importance_sampling_ratio/mean": 0.956892728805542, "sampling/importance_sampling_ratio/min": 0.17221273481845856, "sampling/sampling_logp_difference/max": 1.7047791481018066, "sampling/sampling_logp_difference/mean": 0.09405039250850677, "step": 176, "step_time": 11.039028686012898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 45.15625, "completions/mean_terminated_length": 45.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3089366315398365, "epoch": 0.00354, "frac_reward_zero_std": 0.0, "grad_norm": 0.24420511722564697, "kl": 1.1494507659226656, "learning_rate": 9.518944406510707e-06, "loss": 0.0111, "num_tokens": 7998283.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 2.9153313636779785, "sampling/importance_sampling_ratio/mean": 1.0146385431289673, "sampling/importance_sampling_ratio/min": 0.33350440859794617, "sampling/sampling_logp_difference/max": 1.1806516647338867, "sampling/sampling_logp_difference/mean": 0.07866518944501877, "step": 177, "step_time": 11.941839513994637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 135.4375, "completions/mean_terminated_length": 135.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2620401280000806, "epoch": 0.00356, "frac_reward_zero_std": 0.0, "grad_norm": 0.5624740123748779, "kl": 0.9196281135082245, "learning_rate": 9.512249151849229e-06, "loss": -0.0116, "num_tokens": 8046777.0, "reward": 0.07312498986721039, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0300672054290771, "sampling/importance_sampling_ratio/mean": 0.9157457947731018, "sampling/importance_sampling_ratio/min": 0.3617764115333557, "sampling/sampling_logp_difference/max": 0.9211456775665283, "sampling/sampling_logp_difference/mean": 0.07541745901107788, "step": 178, "step_time": 13.42101712598378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 70.15625, "completions/mean_terminated_length": 70.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27386307809501886, "epoch": 0.00358, "frac_reward_zero_std": 0.0, "grad_norm": 0.24304714798927307, "kl": 0.8122001495212317, "learning_rate": 9.50551086591615e-06, "loss": 0.006, "num_tokens": 8092299.0, "reward": -0.24375000596046448, "reward_std": 1.024906873703003, "rewards/rollout_reward_func/mean": -0.24375000596046448, "rewards/rollout_reward_func/std": 0.9919474124908447, "sampling/importance_sampling_ratio/max": 1.8003535270690918, "sampling/importance_sampling_ratio/mean": 0.9702670574188232, "sampling/importance_sampling_ratio/min": 0.38565558195114136, "sampling/sampling_logp_difference/max": 0.9408102035522461, "sampling/sampling_logp_difference/mean": 0.07120586931705475, "step": 179, "step_time": 14.194908227014821 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1004.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 129.40625, "completions/mean_terminated_length": 129.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28861160995438695, "epoch": 0.0036, "frac_reward_zero_std": 0.0, "grad_norm": 0.9735471606254578, "kl": 1.847661817446351, "learning_rate": 9.498729637594016e-06, "loss": -0.0771, "num_tokens": 8140790.0, "reward": -0.17937500774860382, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.2868213653564453, "sampling/importance_sampling_ratio/mean": 0.994769275188446, "sampling/importance_sampling_ratio/min": 0.29706984758377075, "sampling/sampling_logp_difference/max": 0.830981969833374, "sampling/sampling_logp_difference/mean": 0.07880377769470215, "step": 180, "step_time": 13.650645513007476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 80.96875, "completions/mean_terminated_length": 80.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3083002893254161, "epoch": 0.00362, "frac_reward_zero_std": 0.0, "grad_norm": 0.3855002522468567, "kl": 1.3681310173124075, "learning_rate": 9.491905556331809e-06, "loss": 0.0236, "num_tokens": 8187554.0, "reward": 0.009374991059303284, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 2.5447607040405273, "sampling/importance_sampling_ratio/mean": 1.023846983909607, "sampling/importance_sampling_ratio/min": 0.3854679763317108, "sampling/sampling_logp_difference/max": 0.8852934837341309, "sampling/sampling_logp_difference/mean": 0.08214037120342255, "step": 181, "step_time": 11.621937715011882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 90.09375, "completions/mean_terminated_length": 90.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36625338229350746, "epoch": 0.00364, "frac_reward_zero_std": 0.25, "grad_norm": 0.261110782623291, "kl": 1.602514447644353, "learning_rate": 9.485038712143778e-06, "loss": -0.0174, "num_tokens": 8235029.0, "reward": -0.30625003576278687, "reward_std": 0.8005667924880981, "rewards/rollout_reward_func/mean": -0.30625003576278687, "rewards/rollout_reward_func/std": 0.9738972187042236, "sampling/importance_sampling_ratio/max": 1.21189284324646, "sampling/importance_sampling_ratio/mean": 0.8738062977790833, "sampling/importance_sampling_ratio/min": 0.21634526550769806, "sampling/sampling_logp_difference/max": 1.3362493515014648, "sampling/sampling_logp_difference/mean": 0.1320694386959076, "step": 182, "step_time": 11.933212326010107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 114.21875, "completions/mean_terminated_length": 114.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3216407923027873, "epoch": 0.00366, "frac_reward_zero_std": 0.0, "grad_norm": 0.27169638872146606, "kl": 0.7745823673903942, "learning_rate": 9.478129195608238e-06, "loss": -0.0365, "num_tokens": 8283632.0, "reward": -0.05500000715255737, "reward_std": 1.0331578254699707, "rewards/rollout_reward_func/mean": -0.05500000715255737, "rewards/rollout_reward_func/std": 1.022141933441162, "sampling/importance_sampling_ratio/max": 2.5129554271698, "sampling/importance_sampling_ratio/mean": 0.9978205561637878, "sampling/importance_sampling_ratio/min": 0.23220127820968628, "sampling/sampling_logp_difference/max": 0.886049747467041, "sampling/sampling_logp_difference/mean": 0.07007170468568802, "step": 183, "step_time": 11.943092412009719 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 26.9375, "completions/mean_terminated_length": 26.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2729319336358458, "epoch": 0.00368, "frac_reward_zero_std": 0.0, "grad_norm": 0.6581258773803711, "kl": 0.8016568124294281, "learning_rate": 9.471177097866384e-06, "loss": 0.0659, "num_tokens": 8327753.0, "reward": -0.24312500655651093, "reward_std": 0.9891104102134705, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 2.5140504837036133, "sampling/importance_sampling_ratio/mean": 1.119014024734497, "sampling/importance_sampling_ratio/min": 0.8998807072639465, "sampling/sampling_logp_difference/max": 0.8807318210601807, "sampling/sampling_logp_difference/mean": 0.06518411636352539, "step": 184, "step_time": 11.4922719939932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 554.0, "completions/max_terminated_length": 554.0, "completions/mean_length": 48.46875, "completions/mean_terminated_length": 48.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3048491175286472, "epoch": 0.0037, "frac_reward_zero_std": 0.0, "grad_norm": 0.6642479300498962, "kl": 0.7337502148002386, "learning_rate": 9.464182510621085e-06, "loss": 0.0788, "num_tokens": 8371945.0, "reward": -0.11625000834465027, "reward_std": 0.9712425470352173, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.5748202800750732, "sampling/importance_sampling_ratio/mean": 1.000204086303711, "sampling/importance_sampling_ratio/min": 0.2445310354232788, "sampling/sampling_logp_difference/max": 1.2097196578979492, "sampling/sampling_logp_difference/mean": 0.09121483564376831, "step": 185, "step_time": 11.507008578999375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 88.78125, "completions/mean_terminated_length": 88.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32490674615837634, "epoch": 0.00372, "frac_reward_zero_std": 0.0, "grad_norm": 0.6013665199279785, "kl": 0.924439612776041, "learning_rate": 9.457145526135673e-06, "loss": -0.0482, "num_tokens": 8419623.0, "reward": -0.36937499046325684, "reward_std": 0.9617735147476196, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 2.0703980922698975, "sampling/importance_sampling_ratio/mean": 0.9638165831565857, "sampling/importance_sampling_ratio/min": 0.1840457320213318, "sampling/sampling_logp_difference/max": 1.6665434837341309, "sampling/sampling_logp_difference/mean": 0.09693266451358795, "step": 186, "step_time": 12.0608631300056 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 47.625, "completions/mean_terminated_length": 47.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24705607909709215, "epoch": 0.00374, "frac_reward_zero_std": 0.0, "grad_norm": 0.5038254261016846, "kl": 0.6252033151686192, "learning_rate": 9.450066237232736e-06, "loss": -0.0143, "num_tokens": 8461878.0, "reward": 0.3243749737739563, "reward_std": 0.9977927207946777, "rewards/rollout_reward_func/mean": 0.3243749737739563, "rewards/rollout_reward_func/std": 0.9738613367080688, "sampling/importance_sampling_ratio/max": 1.060685396194458, "sampling/importance_sampling_ratio/mean": 0.9873270988464355, "sampling/importance_sampling_ratio/min": 0.5816887617111206, "sampling/sampling_logp_difference/max": 0.564358651638031, "sampling/sampling_logp_difference/mean": 0.04669709503650665, "step": 187, "step_time": 10.896587846007606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 91.90625, "completions/mean_terminated_length": 91.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29398789815604687, "epoch": 0.00376, "frac_reward_zero_std": 0.0, "grad_norm": 0.614689290523529, "kl": 0.7093193754553795, "learning_rate": 9.442944737292872e-06, "loss": 0.0069, "num_tokens": 8507542.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.3119361400604248, "sampling/importance_sampling_ratio/mean": 0.9833283424377441, "sampling/importance_sampling_ratio/min": 0.27320772409439087, "sampling/sampling_logp_difference/max": 1.208289623260498, "sampling/sampling_logp_difference/mean": 0.06289228051900864, "step": 188, "step_time": 14.523918174003484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 784.0, "completions/max_terminated_length": 784.0, "completions/mean_length": 79.0, "completions/mean_terminated_length": 79.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28180185053497553, "epoch": 0.00378, "frac_reward_zero_std": 0.0, "grad_norm": 0.255393385887146, "kl": 1.2472182903438807, "learning_rate": 9.435781120253486e-06, "loss": -0.0132, "num_tokens": 8553519.0, "reward": -0.053125008940696716, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0512865781784058, "sampling/importance_sampling_ratio/mean": 0.8582967519760132, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.865206003189087, "sampling/sampling_logp_difference/mean": 0.12665659189224243, "step": 189, "step_time": 12.474575376989378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 51.625, "completions/mean_terminated_length": 51.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3187544564716518, "epoch": 0.0038, "frac_reward_zero_std": 0.0, "grad_norm": 0.32220274209976196, "kl": 0.9109494853764772, "learning_rate": 9.428575480607526e-06, "loss": -0.001, "num_tokens": 8598290.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.485763669013977, "sampling/importance_sampling_ratio/mean": 0.9423636198043823, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3783820867538452, "sampling/sampling_logp_difference/mean": 0.09325695037841797, "step": 190, "step_time": 11.882345589983743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 64.03125, "completions/mean_terminated_length": 65.58064270019531, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4272070995066315, "epoch": 0.00382, "frac_reward_zero_std": 0.0, "grad_norm": 0.4181841015815735, "kl": 1.1475247014313936, "learning_rate": 9.421327913402252e-06, "loss": 0.0299, "num_tokens": 8643439.0, "reward": -0.17937500774860382, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.6218291521072388, "sampling/importance_sampling_ratio/mean": 0.9159607887268066, "sampling/importance_sampling_ratio/min": 0.12028013169765472, "sampling/sampling_logp_difference/max": 1.514897108078003, "sampling/sampling_logp_difference/mean": 0.14855685830116272, "step": 191, "step_time": 12.066646562991082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 85.0, "completions/mean_terminated_length": 85.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.34774377662688494, "epoch": 0.00384, "frac_reward_zero_std": 0.0, "grad_norm": 0.3366803526878357, "kl": 1.077026940882206, "learning_rate": 9.41403851423797e-06, "loss": 0.0093, "num_tokens": 8690818.0, "reward": -0.24250000715255737, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.4345630407333374, "sampling/importance_sampling_ratio/mean": 0.9182736873626709, "sampling/importance_sampling_ratio/min": 0.24116358160972595, "sampling/sampling_logp_difference/max": 1.4485177993774414, "sampling/sampling_logp_difference/mean": 0.1297055184841156, "step": 192, "step_time": 11.052587940990634 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 766.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 57.4375, "completions/mean_terminated_length": 57.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2588116195984185, "epoch": 0.00386, "frac_reward_zero_std": 0.0, "grad_norm": 0.4973640441894531, "kl": 0.9472073912620544, "learning_rate": 9.406707379266792e-06, "loss": 0.0458, "num_tokens": 8735916.0, "reward": -0.17937500774860382, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.731374740600586, "sampling/importance_sampling_ratio/mean": 1.0275171995162964, "sampling/importance_sampling_ratio/min": 0.31722351908683777, "sampling/sampling_logp_difference/max": 1.1834180355072021, "sampling/sampling_logp_difference/mean": 0.09088467061519623, "step": 193, "step_time": 12.423061569017591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 62.84375, "completions/mean_terminated_length": 62.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4133409531787038, "epoch": 0.00388, "frac_reward_zero_std": 0.25, "grad_norm": 0.3195493817329407, "kl": 2.996645711362362, "learning_rate": 9.399334605191332e-06, "loss": -0.0071, "num_tokens": 8782734.0, "reward": -0.4325000047683716, "reward_std": 0.7642077207565308, "rewards/rollout_reward_func/mean": -0.4325000047683716, "rewards/rollout_reward_func/std": 0.9217339158058167, "sampling/importance_sampling_ratio/max": 1.492735505104065, "sampling/importance_sampling_ratio/mean": 0.7670446038246155, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2676544189453125, "sampling/sampling_logp_difference/mean": 0.22311055660247803, "step": 194, "step_time": 12.015742426992801 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 119.65625, "completions/mean_terminated_length": 119.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29330159816890955, "epoch": 0.0039, "frac_reward_zero_std": 0.0, "grad_norm": 0.41547203063964844, "kl": 1.104164008051157, "learning_rate": 9.391920289263475e-06, "loss": 0.028, "num_tokens": 8830466.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.8563823699951172, "sampling/importance_sampling_ratio/mean": 0.9130421876907349, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3608531951904297, "sampling/sampling_logp_difference/mean": 0.12405179440975189, "step": 195, "step_time": 14.263306961984199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 56.875, "completions/mean_terminated_length": 56.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29605005122721195, "epoch": 0.00392, "frac_reward_zero_std": 0.25, "grad_norm": 0.17147061228752136, "kl": 0.8689693510532379, "learning_rate": 9.384464529283055e-06, "loss": -0.0178, "num_tokens": 8875784.0, "reward": -0.3056250214576721, "reward_std": 0.7289015054702759, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.5653986930847168, "sampling/importance_sampling_ratio/mean": 0.9577965140342712, "sampling/importance_sampling_ratio/min": 0.22284536063671112, "sampling/sampling_logp_difference/max": 1.196640968322754, "sampling/sampling_logp_difference/mean": 0.08028355240821838, "step": 196, "step_time": 11.918277502001729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 563.0, "completions/max_terminated_length": 563.0, "completions/mean_length": 80.21875, "completions/mean_terminated_length": 80.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2579923067241907, "epoch": 0.00394, "frac_reward_zero_std": 0.0, "grad_norm": 0.6975690722465515, "kl": 1.1814959328621626, "learning_rate": 9.376967423596588e-06, "loss": -0.0749, "num_tokens": 8921484.0, "reward": 0.009374991059303284, "reward_std": 1.0067436695098877, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 2.4202394485473633, "sampling/importance_sampling_ratio/mean": 0.9309335947036743, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.985405445098877, "sampling/sampling_logp_difference/mean": 0.12378285825252533, "step": 197, "step_time": 11.705866780008364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 89.5, "completions/mean_terminated_length": 89.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3333709603175521, "epoch": 0.00396, "frac_reward_zero_std": 0.0, "grad_norm": 0.4815191626548767, "kl": 0.8910646308213472, "learning_rate": 9.369429071095963e-06, "loss": -0.0347, "num_tokens": 8968774.0, "reward": -0.3056250214576721, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.052021861076355, "sampling/importance_sampling_ratio/mean": 0.8612916469573975, "sampling/importance_sampling_ratio/min": 0.20208366215229034, "sampling/sampling_logp_difference/max": 1.6421806812286377, "sampling/sampling_logp_difference/mean": 0.12814660370349884, "step": 198, "step_time": 11.012530655985756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 82.40625, "completions/mean_terminated_length": 82.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4051783671602607, "epoch": 0.00398, "frac_reward_zero_std": 0.0, "grad_norm": 0.3189831078052521, "kl": 2.424966225400567, "learning_rate": 9.361849571217149e-06, "loss": -0.0003, "num_tokens": 9016029.0, "reward": -0.49562501907348633, "reward_std": 0.860129177570343, "rewards/rollout_reward_func/mean": -0.49562501907348633, "rewards/rollout_reward_func/std": 0.8875880241394043, "sampling/importance_sampling_ratio/max": 1.3227403163909912, "sampling/importance_sampling_ratio/mean": 0.8373070359230042, "sampling/importance_sampling_ratio/min": 0.14387433230876923, "sampling/sampling_logp_difference/max": 1.8582992553710938, "sampling/sampling_logp_difference/mean": 0.1564883440732956, "step": 199, "step_time": 12.120312979008304 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 60.03125, "completions/mean_terminated_length": 60.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3176003433763981, "epoch": 0.004, "frac_reward_zero_std": 0.0, "grad_norm": 0.3403758108615875, "kl": 1.9963406808674335, "learning_rate": 9.35422902393887e-06, "loss": 0.0257, "num_tokens": 9062511.0, "reward": -0.36937499046325684, "reward_std": 0.9429470300674438, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.7510243654251099, "sampling/importance_sampling_ratio/mean": 0.9062750935554504, "sampling/importance_sampling_ratio/min": 0.18649423122406006, "sampling/sampling_logp_difference/max": 1.7082197666168213, "sampling/sampling_logp_difference/mean": 0.12808819115161896, "step": 200, "step_time": 11.938180326978909 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 429.0, "completions/max_terminated_length": 429.0, "completions/mean_length": 41.65625, "completions/mean_terminated_length": 41.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3363400474190712, "epoch": 0.00402, "frac_reward_zero_std": 0.0, "grad_norm": 0.3762238621711731, "kl": 1.6488744132220745, "learning_rate": 9.346567529781298e-06, "loss": -0.0209, "num_tokens": 9106257.0, "reward": -0.053125008940696716, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 2.449349880218506, "sampling/importance_sampling_ratio/mean": 0.9167366623878479, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0335171222686768, "sampling/sampling_logp_difference/mean": 0.16159087419509888, "step": 201, "step_time": 10.658684992995404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 429.0, "completions/max_terminated_length": 429.0, "completions/mean_length": 41.71875, "completions/mean_terminated_length": 41.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2861537607386708, "epoch": 0.00404, "frac_reward_zero_std": 0.0, "grad_norm": 0.4319276809692383, "kl": 2.9097008891403675, "learning_rate": 9.338865189804722e-06, "loss": -0.0435, "num_tokens": 9150874.0, "reward": -0.11625000834465027, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.5922677516937256, "sampling/importance_sampling_ratio/mean": 1.0081738233566284, "sampling/importance_sampling_ratio/min": 0.07480518519878387, "sampling/sampling_logp_difference/max": 2.447037696838379, "sampling/sampling_logp_difference/mean": 0.13617870211601257, "step": 202, "step_time": 11.38971155899344 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 757.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 153.5, "completions/mean_terminated_length": 153.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.300450477283448, "epoch": 0.00406, "frac_reward_zero_std": 0.0, "grad_norm": 0.6449027061462402, "kl": 0.9620937667787075, "learning_rate": 9.33112210560821e-06, "loss": 0.0873, "num_tokens": 9199735.0, "reward": -0.24250002205371857, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.24250002205371857, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 2.3970448970794678, "sampling/importance_sampling_ratio/mean": 1.1297340393066406, "sampling/importance_sampling_ratio/min": 0.42878854274749756, "sampling/sampling_logp_difference/max": 0.9321749210357666, "sampling/sampling_logp_difference/mean": 0.08287754654884338, "step": 203, "step_time": 12.414056638990587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 35.5625, "completions/mean_terminated_length": 35.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3513628374785185, "epoch": 0.00408, "frac_reward_zero_std": 0.0, "grad_norm": 0.6087873578071594, "kl": 1.8846221268177032, "learning_rate": 9.323338379328278e-06, "loss": 0.0477, "num_tokens": 9245305.0, "reward": -0.3056250214576721, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 2.4360251426696777, "sampling/importance_sampling_ratio/mean": 0.8647458553314209, "sampling/importance_sampling_ratio/min": 0.1346217840909958, "sampling/sampling_logp_difference/max": 1.9832756519317627, "sampling/sampling_logp_difference/mean": 0.21446630358695984, "step": 204, "step_time": 11.658364514987625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 112.71875, "completions/mean_terminated_length": 112.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31902090879157186, "epoch": 0.0041, "frac_reward_zero_std": 0.0, "grad_norm": 0.38766875863075256, "kl": 0.9970524441450834, "learning_rate": 9.315514113637534e-06, "loss": 0.0263, "num_tokens": 9292923.0, "reward": -0.11625000834465027, "reward_std": 0.9712425470352173, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.238213300704956, "sampling/importance_sampling_ratio/mean": 0.9849039316177368, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0660650730133057, "sampling/sampling_logp_difference/mean": 0.09850640594959259, "step": 205, "step_time": 12.03959601400129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 44.5, "completions/mean_terminated_length": 44.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.294849019497633, "epoch": 0.00412, "frac_reward_zero_std": 0.0, "grad_norm": 0.4925552010536194, "kl": 3.8262400925159454, "learning_rate": 9.30764941174334e-06, "loss": 0.0054, "num_tokens": 9336783.0, "reward": -0.17937500774860382, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.0079617500305176, "sampling/importance_sampling_ratio/max": 1.5031527280807495, "sampling/importance_sampling_ratio/mean": 0.9073201417922974, "sampling/importance_sampling_ratio/min": 0.2924247086048126, "sampling/sampling_logp_difference/max": 1.2286443710327148, "sampling/sampling_logp_difference/mean": 0.10241897404193878, "step": 206, "step_time": 10.883336068000062 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 76.4375, "completions/mean_terminated_length": 76.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.38016439648345113, "epoch": 0.00414, "frac_reward_zero_std": 0.0, "grad_norm": 0.3708244860172272, "kl": 1.6829995568841696, "learning_rate": 9.299744377386424e-06, "loss": -0.0239, "num_tokens": 9382731.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.032723307609558, "sampling/importance_sampling_ratio/mean": 0.8584965467453003, "sampling/importance_sampling_ratio/min": 0.17341992259025574, "sampling/sampling_logp_difference/max": 1.6852929592132568, "sampling/sampling_logp_difference/mean": 0.12424857914447784, "step": 207, "step_time": 11.965389048978977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 82.96875, "completions/mean_terminated_length": 82.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3791345562785864, "epoch": 0.00416, "frac_reward_zero_std": 0.0, "grad_norm": 0.34027737379074097, "kl": 0.9985489789396524, "learning_rate": 9.291799114839533e-06, "loss": -0.0171, "num_tokens": 9428920.0, "reward": -0.17937500774860382, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.589394450187683, "sampling/importance_sampling_ratio/mean": 0.9197549819946289, "sampling/importance_sampling_ratio/min": 0.23687578737735748, "sampling/sampling_logp_difference/max": 1.4525827169418335, "sampling/sampling_logp_difference/mean": 0.10333162546157837, "step": 208, "step_time": 13.536570250988007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 36.34375, "completions/mean_terminated_length": 36.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4103881875053048, "epoch": 0.00418, "frac_reward_zero_std": 0.0, "grad_norm": 0.3445111811161041, "kl": 1.8769060745835304, "learning_rate": 9.283813728906054e-06, "loss": 0.0308, "num_tokens": 9475085.0, "reward": -0.6212500333786011, "reward_std": 0.8246280550956726, "rewards/rollout_reward_func/mean": -0.6212500333786011, "rewards/rollout_reward_func/std": 0.8010466694831848, "sampling/importance_sampling_ratio/max": 2.279719114303589, "sampling/importance_sampling_ratio/mean": 0.9345857501029968, "sampling/importance_sampling_ratio/min": 0.19920308887958527, "sampling/sampling_logp_difference/max": 1.5090792179107666, "sampling/sampling_logp_difference/mean": 0.1587899625301361, "step": 209, "step_time": 10.820821216017066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 88.625, "completions/mean_terminated_length": 88.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3944318522699177, "epoch": 0.0042, "frac_reward_zero_std": 0.0, "grad_norm": 0.45451653003692627, "kl": 0.9131673276424408, "learning_rate": 9.275788324918624e-06, "loss": -0.0377, "num_tokens": 9522081.0, "reward": -0.17937500774860382, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.9526124000549316, "sampling/importance_sampling_ratio/mean": 0.9411023855209351, "sampling/importance_sampling_ratio/min": 0.2963758111000061, "sampling/sampling_logp_difference/max": 0.9506263732910156, "sampling/sampling_logp_difference/mean": 0.0894070714712143, "step": 210, "step_time": 12.001243265985977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 95.875, "completions/mean_terminated_length": 95.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.337998331990093, "epoch": 0.00422, "frac_reward_zero_std": 0.0, "grad_norm": 0.7013720273971558, "kl": 0.8304146453738213, "learning_rate": 9.267723008737745e-06, "loss": 0.044, "num_tokens": 9568800.0, "reward": -0.05437500774860382, "reward_std": 1.0698342323303223, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228137969970703, "sampling/importance_sampling_ratio/max": 1.841700792312622, "sampling/importance_sampling_ratio/mean": 1.0154889822006226, "sampling/importance_sampling_ratio/min": 0.45571085810661316, "sampling/sampling_logp_difference/max": 0.9716978669166565, "sampling/sampling_logp_difference/mean": 0.0801381766796112, "step": 211, "step_time": 14.003566774990759 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 75.15625, "completions/mean_terminated_length": 75.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3961804467253387, "epoch": 0.00424, "frac_reward_zero_std": 0.0, "grad_norm": 0.5955373048782349, "kl": 1.2072983514517546, "learning_rate": 9.259617886750395e-06, "loss": -0.037, "num_tokens": 9613779.0, "reward": 0.009999990463256836, "reward_std": 1.0074068307876587, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 2.2196786403656006, "sampling/importance_sampling_ratio/mean": 0.9613476991653442, "sampling/importance_sampling_ratio/min": 0.1638024002313614, "sampling/sampling_logp_difference/max": 1.5940892696380615, "sampling/sampling_logp_difference/mean": 0.10129082947969437, "step": 212, "step_time": 11.72113357300259 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 847.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 95.6875, "completions/mean_terminated_length": 95.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32245094748213887, "epoch": 0.00426, "frac_reward_zero_std": 0.0, "grad_norm": 0.713019609451294, "kl": 1.5309608820825815, "learning_rate": 9.251473065868606e-06, "loss": 0.0483, "num_tokens": 9659126.0, "reward": 0.07249999046325684, "reward_std": 0.9345266222953796, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.6468828916549683, "sampling/importance_sampling_ratio/mean": 1.0042517185211182, "sampling/importance_sampling_ratio/min": 0.21480469405651093, "sampling/sampling_logp_difference/max": 1.4842222929000854, "sampling/sampling_logp_difference/mean": 0.08188223093748093, "step": 213, "step_time": 13.432082317020104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 59.46875, "completions/mean_terminated_length": 59.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3775149662978947, "epoch": 0.00428, "frac_reward_zero_std": 0.0, "grad_norm": 0.48240652680397034, "kl": 1.445691004395485, "learning_rate": 9.243288653528075e-06, "loss": 0.0111, "num_tokens": 9704891.0, "reward": -0.17937500774860382, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.5264067649841309, "sampling/importance_sampling_ratio/mean": 0.9440103769302368, "sampling/importance_sampling_ratio/min": 0.1896393746137619, "sampling/sampling_logp_difference/max": 1.3769363164901733, "sampling/sampling_logp_difference/mean": 0.07775529474020004, "step": 214, "step_time": 13.676245289003418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 89.96875, "completions/mean_terminated_length": 89.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31685685412958264, "epoch": 0.0043, "frac_reward_zero_std": 0.0, "grad_norm": 0.372469425201416, "kl": 0.8759920299053192, "learning_rate": 9.235064757686733e-06, "loss": 0.0298, "num_tokens": 9749737.0, "reward": -0.24250002205371857, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.24250002205371857, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 1.6337231397628784, "sampling/importance_sampling_ratio/mean": 0.9886214137077332, "sampling/importance_sampling_ratio/min": 0.4553702175617218, "sampling/sampling_logp_difference/max": 0.589744508266449, "sampling/sampling_logp_difference/mean": 0.05567387491464615, "step": 215, "step_time": 13.81216969898378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 68.03125, "completions/mean_terminated_length": 68.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.41012114379554987, "epoch": 0.00432, "frac_reward_zero_std": 0.0, "grad_norm": 0.3501700758934021, "kl": 0.7557620350271463, "learning_rate": 9.226801486823325e-06, "loss": -0.0118, "num_tokens": 9796197.0, "reward": -0.3075000047683716, "reward_std": 1.015291690826416, "rewards/rollout_reward_func/mean": -0.3075000047683716, "rewards/rollout_reward_func/std": 0.9721641540527344, "sampling/importance_sampling_ratio/max": 1.3962229490280151, "sampling/importance_sampling_ratio/mean": 0.9815369844436646, "sampling/importance_sampling_ratio/min": 0.5472398996353149, "sampling/sampling_logp_difference/max": 0.3546643853187561, "sampling/sampling_logp_difference/mean": 0.05638913810253143, "step": 216, "step_time": 14.493039155000588 }, { "clip_ratio/high_max": 0.0052083334885537624, "clip_ratio/high_mean": 0.0026041667442768812, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026041667442768812, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 85.46875, "completions/mean_terminated_length": 85.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37030771700665355, "epoch": 0.00434, "frac_reward_zero_std": 0.0, "grad_norm": 0.7074593305587769, "kl": 0.8717960473150015, "learning_rate": 9.21849894993598e-06, "loss": -0.0318, "num_tokens": 9842215.0, "reward": -0.18000000715255737, "reward_std": 0.9618129730224609, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.6707559823989868, "sampling/importance_sampling_ratio/mean": 0.9289608001708984, "sampling/importance_sampling_ratio/min": 0.43482208251953125, "sampling/sampling_logp_difference/max": 0.8322668075561523, "sampling/sampling_logp_difference/mean": 0.08122918754816055, "step": 217, "step_time": 11.634905661041557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 68.65625, "completions/mean_terminated_length": 68.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.46133957896381617, "epoch": 0.00436, "frac_reward_zero_std": 0.0, "grad_norm": 0.31059616804122925, "kl": 1.0629548113793135, "learning_rate": 9.21015725654077e-06, "loss": -0.0035, "num_tokens": 9888906.0, "reward": -0.37062501907348633, "reward_std": 0.9600666761398315, "rewards/rollout_reward_func/mean": -0.37062501907348633, "rewards/rollout_reward_func/std": 0.9484790563583374, "sampling/importance_sampling_ratio/max": 1.7799617052078247, "sampling/importance_sampling_ratio/mean": 0.9301025867462158, "sampling/importance_sampling_ratio/min": 0.23132354021072388, "sampling/sampling_logp_difference/max": 1.3712522983551025, "sampling/sampling_logp_difference/mean": 0.0988718718290329, "step": 218, "step_time": 11.463260538002942 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 65.25, "completions/mean_terminated_length": 65.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4917349861934781, "epoch": 0.00438, "frac_reward_zero_std": 0.0, "grad_norm": 0.9774779081344604, "kl": 1.1805196087807417, "learning_rate": 9.201776516670274e-06, "loss": -0.1169, "num_tokens": 9935604.0, "reward": -0.4950000047683716, "reward_std": 0.860792338848114, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 2.6438283920288086, "sampling/importance_sampling_ratio/mean": 1.035828948020935, "sampling/importance_sampling_ratio/min": 0.36295080184936523, "sampling/sampling_logp_difference/max": 1.248530626296997, "sampling/sampling_logp_difference/mean": 0.12097081542015076, "step": 219, "step_time": 12.442716355995799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 592.0, "completions/max_terminated_length": 592.0, "completions/mean_length": 81.375, "completions/mean_terminated_length": 81.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.39231447922065854, "epoch": 0.0044, "frac_reward_zero_std": 0.0, "grad_norm": 0.9204134941101074, "kl": 2.2053935639560223, "learning_rate": 9.193356840872111e-06, "loss": 0.0347, "num_tokens": 9981665.0, "reward": -0.24250000715255737, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 2.1873533725738525, "sampling/importance_sampling_ratio/mean": 0.9683184027671814, "sampling/importance_sampling_ratio/min": 0.1488906443119049, "sampling/sampling_logp_difference/max": 1.5507255792617798, "sampling/sampling_logp_difference/mean": 0.12666815519332886, "step": 220, "step_time": 11.59723274197313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 102.78125, "completions/mean_terminated_length": 102.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5081737823784351, "epoch": 0.00442, "frac_reward_zero_std": 0.5, "grad_norm": 1.0332818031311035, "kl": 2.0556557942181826, "learning_rate": 9.184898340207502e-06, "loss": 0.0136, "num_tokens": 10030712.0, "reward": -0.684374988079071, "reward_std": 0.4484783113002777, "rewards/rollout_reward_func/mean": -0.684374988079071, "rewards/rollout_reward_func/std": 0.7451820969581604, "sampling/importance_sampling_ratio/max": 2.709672689437866, "sampling/importance_sampling_ratio/mean": 0.9817202091217041, "sampling/importance_sampling_ratio/min": 0.14734302461147308, "sampling/sampling_logp_difference/max": 1.5545777082443237, "sampling/sampling_logp_difference/mean": 0.1362125277519226, "step": 221, "step_time": 13.252212062987383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 36.96875, "completions/mean_terminated_length": 36.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37171109626069665, "epoch": 0.00444, "frac_reward_zero_std": 0.0, "grad_norm": 0.5969931483268738, "kl": 1.113691285252571, "learning_rate": 9.176401126249785e-06, "loss": 0.0151, "num_tokens": 10074532.0, "reward": 0.008749991655349731, "reward_std": 1.0615792274475098, "rewards/rollout_reward_func/mean": 0.008749991655349731, "rewards/rollout_reward_func/std": 1.0249146223068237, "sampling/importance_sampling_ratio/max": 2.166841506958008, "sampling/importance_sampling_ratio/mean": 0.9289106130599976, "sampling/importance_sampling_ratio/min": 0.1352982223033905, "sampling/sampling_logp_difference/max": 1.5077836513519287, "sampling/sampling_logp_difference/mean": 0.1258935034275055, "step": 222, "step_time": 11.660448807982903 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 547.0, "completions/max_terminated_length": 547.0, "completions/mean_length": 50.1875, "completions/mean_terminated_length": 50.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37792540481314063, "epoch": 0.00446, "frac_reward_zero_std": 0.0, "grad_norm": 0.9646852612495422, "kl": 1.0922446344047785, "learning_rate": 9.167865311082957e-06, "loss": 0.0659, "num_tokens": 10120020.0, "reward": -0.3056250214576721, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 2.2966806888580322, "sampling/importance_sampling_ratio/mean": 1.006945013999939, "sampling/importance_sampling_ratio/min": 0.23445504903793335, "sampling/sampling_logp_difference/max": 1.4301995038986206, "sampling/sampling_logp_difference/mean": 0.08753286302089691, "step": 223, "step_time": 11.05012073200487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 411.0, "completions/max_terminated_length": 411.0, "completions/mean_length": 41.28125, "completions/mean_terminated_length": 41.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3915408810134977, "epoch": 0.00448, "frac_reward_zero_std": 0.0, "grad_norm": 0.4921724498271942, "kl": 1.225566927343607, "learning_rate": 9.159291007300193e-06, "loss": -0.0301, "num_tokens": 10164607.0, "reward": -0.05375000834465027, "reward_std": 1.0535087585449219, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.9324184656143188, "sampling/importance_sampling_ratio/mean": 0.9399092793464661, "sampling/importance_sampling_ratio/min": 0.10921407490968704, "sampling/sampling_logp_difference/max": 1.652251958847046, "sampling/sampling_logp_difference/mean": 0.1176428496837616, "step": 224, "step_time": 11.269867344977683 }, { "clip_ratio/high_max": 0.0052083334885537624, "clip_ratio/high_mean": 0.0026041667442768812, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666744276881, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 80.78125, "completions/mean_terminated_length": 80.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.278193564619869, "epoch": 0.0045, "frac_reward_zero_std": 0.0, "grad_norm": 0.580879271030426, "kl": 0.803458247333765, "learning_rate": 9.150678328002352e-06, "loss": -0.0347, "num_tokens": 10209015.0, "reward": 0.38874998688697815, "reward_std": 0.8798136115074158, "rewards/rollout_reward_func/mean": 0.38874998688697815, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.8733834028244019, "sampling/importance_sampling_ratio/mean": 1.008380651473999, "sampling/importance_sampling_ratio/min": 0.5420088768005371, "sampling/sampling_logp_difference/max": 0.8403725624084473, "sampling/sampling_logp_difference/mean": 0.06348690390586853, "step": 225, "step_time": 11.498642857004597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.0, "completions/max_length": 599.0, "completions/max_terminated_length": 599.0, "completions/mean_length": 66.15625, "completions/mean_terminated_length": 66.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3639465607702732, "epoch": 0.00452, "frac_reward_zero_std": 0.0, "grad_norm": 0.7181900143623352, "kl": 1.2218828536570072, "learning_rate": 9.1420273867965e-06, "loss": 0.0396, "num_tokens": 10255164.0, "reward": -0.17937500774860382, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.208481788635254, "sampling/importance_sampling_ratio/mean": 1.0177865028381348, "sampling/importance_sampling_ratio/min": 0.1420992910861969, "sampling/sampling_logp_difference/max": 1.580946683883667, "sampling/sampling_logp_difference/mean": 0.09218201041221619, "step": 226, "step_time": 11.680400279983587 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 88.625, "completions/mean_terminated_length": 88.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2725011706352234, "epoch": 0.00454, "frac_reward_zero_std": 0.0, "grad_norm": 0.36935219168663025, "kl": 1.2952875941991806, "learning_rate": 9.133338297794396e-06, "loss": 0.0319, "num_tokens": 10301800.0, "reward": -0.11812500655651093, "reward_std": 1.024240493774414, "rewards/rollout_reward_func/mean": -0.11812500655651093, "rewards/rollout_reward_func/std": 1.0159674882888794, "sampling/importance_sampling_ratio/max": 1.9855998754501343, "sampling/importance_sampling_ratio/mean": 0.9688186645507812, "sampling/importance_sampling_ratio/min": 0.14441213011741638, "sampling/sampling_logp_difference/max": 1.8081855773925781, "sampling/sampling_logp_difference/mean": 0.0890895202755928, "step": 227, "step_time": 13.512238289993547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 79.25, "completions/mean_terminated_length": 79.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2937129787169397, "epoch": 0.00456, "frac_reward_zero_std": 0.0, "grad_norm": 0.24579060077667236, "kl": 2.310995487496257, "learning_rate": 9.124611175610998e-06, "loss": 0.0196, "num_tokens": 10347410.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 2.106710195541382, "sampling/importance_sampling_ratio/mean": 0.9853649735450745, "sampling/importance_sampling_ratio/min": 0.13919249176979065, "sampling/sampling_logp_difference/max": 1.7838327884674072, "sampling/sampling_logp_difference/mean": 0.09631407260894775, "step": 228, "step_time": 12.124837718998606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 62.0, "completions/mean_terminated_length": 62.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2660359516739845, "epoch": 0.00458, "frac_reward_zero_std": 0.0, "grad_norm": 0.4181634485721588, "kl": 0.8251086100935936, "learning_rate": 9.11584613536295e-06, "loss": 0.0008, "num_tokens": 10392808.0, "reward": -0.05375000834465027, "reward_std": 1.0531628131866455, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.2930998802185059, "sampling/importance_sampling_ratio/mean": 0.945566713809967, "sampling/importance_sampling_ratio/min": 0.7045014500617981, "sampling/sampling_logp_difference/max": 0.3721406161785126, "sampling/sampling_logp_difference/mean": 0.04633035510778427, "step": 229, "step_time": 11.602795216997038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 599.0, "completions/max_terminated_length": 599.0, "completions/mean_length": 72.6875, "completions/mean_terminated_length": 72.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25761079881340265, "epoch": 0.0046, "frac_reward_zero_std": 0.0, "grad_norm": 0.8810039162635803, "kl": 2.329243987798691, "learning_rate": 9.107043292667056e-06, "loss": 0.1065, "num_tokens": 10438537.0, "reward": -0.11625000834465027, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.168696403503418, "sampling/importance_sampling_ratio/mean": 1.0315982103347778, "sampling/importance_sampling_ratio/min": 0.3628893196582794, "sampling/sampling_logp_difference/max": 0.7302494049072266, "sampling/sampling_logp_difference/mean": 0.06687852740287781, "step": 230, "step_time": 11.91918900301971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 33.78125, "completions/mean_terminated_length": 34.35483932495117, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26216028118506074, "epoch": 0.00462, "frac_reward_zero_std": 0.0, "grad_norm": 0.7878467440605164, "kl": 1.4373469967395067, "learning_rate": 9.098202763638763e-06, "loss": 0.0446, "num_tokens": 10481552.0, "reward": -0.053125008940696716, "reward_std": 1.0711638927459717, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.244333267211914, "sampling/importance_sampling_ratio/mean": 0.9551945328712463, "sampling/importance_sampling_ratio/min": 0.32470813393592834, "sampling/sampling_logp_difference/max": 0.7325539588928223, "sampling/sampling_logp_difference/mean": 0.05298146232962608, "step": 231, "step_time": 11.40153116799047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 88.21875, "completions/mean_terminated_length": 88.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26060851803049445, "epoch": 0.00464, "frac_reward_zero_std": 0.0, "grad_norm": 0.37211981415748596, "kl": 0.9489412847906351, "learning_rate": 9.089324664890625e-06, "loss": -0.0021, "num_tokens": 10528836.0, "reward": -0.3062500059604645, "reward_std": 0.9979772567749023, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 1.9506746530532837, "sampling/importance_sampling_ratio/mean": 1.094486951828003, "sampling/importance_sampling_ratio/min": 0.5794711112976074, "sampling/sampling_logp_difference/max": 0.9953680038452148, "sampling/sampling_logp_difference/mean": 0.0675823986530304, "step": 232, "step_time": 14.417582760986988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 574.0, "completions/max_terminated_length": 574.0, "completions/mean_length": 43.8125, "completions/mean_terminated_length": 43.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20979349687695503, "epoch": 0.00466, "frac_reward_zero_std": 0.0, "grad_norm": 0.31201931834220886, "kl": 2.3064692318439484, "learning_rate": 9.080409113530767e-06, "loss": 0.0018, "num_tokens": 10574036.0, "reward": -0.30812501907348633, "reward_std": 0.8849272727966309, "rewards/rollout_reward_func/mean": -0.30812501907348633, "rewards/rollout_reward_func/std": 0.9712759852409363, "sampling/importance_sampling_ratio/max": 1.7133548259735107, "sampling/importance_sampling_ratio/mean": 1.0344598293304443, "sampling/importance_sampling_ratio/min": 0.3759787082672119, "sampling/sampling_logp_difference/max": 0.9772524833679199, "sampling/sampling_logp_difference/mean": 0.05107966810464859, "step": 233, "step_time": 11.846351245985716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 89.09375, "completions/mean_terminated_length": 89.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26319184876047075, "epoch": 0.00468, "frac_reward_zero_std": 0.0, "grad_norm": 1.0867829322814941, "kl": 3.233048304915428, "learning_rate": 9.071456227161335e-06, "loss": -0.005, "num_tokens": 10620183.0, "reward": -0.24250000715255737, "reward_std": 1.0074068307876587, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 2.7344236373901367, "sampling/importance_sampling_ratio/mean": 1.0623884201049805, "sampling/importance_sampling_ratio/min": 0.2561732530593872, "sampling/sampling_logp_difference/max": 1.361032485961914, "sampling/sampling_logp_difference/mean": 0.08285607397556305, "step": 234, "step_time": 12.179064899020887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 80.375, "completions/mean_terminated_length": 80.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3350432859733701, "epoch": 0.0047, "frac_reward_zero_std": 0.0, "grad_norm": 0.40971875190734863, "kl": 1.2564030606299639, "learning_rate": 9.062466123876954e-06, "loss": -0.0546, "num_tokens": 10667211.0, "reward": -0.3687500059604645, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.7433247566223145, "sampling/importance_sampling_ratio/mean": 0.9468555450439453, "sampling/importance_sampling_ratio/min": 0.25957202911376953, "sampling/sampling_logp_difference/max": 1.5366182327270508, "sampling/sampling_logp_difference/mean": 0.09579829126596451, "step": 235, "step_time": 11.859861619996082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 52.8125, "completions/mean_terminated_length": 52.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3281263308599591, "epoch": 0.00472, "frac_reward_zero_std": 0.0, "grad_norm": 0.2821723222732544, "kl": 1.6687918845564127, "learning_rate": 9.053438922263165e-06, "loss": 0.0141, "num_tokens": 10713376.0, "reward": -0.4950000047683716, "reward_std": 0.7969957590103149, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 2.023179292678833, "sampling/importance_sampling_ratio/mean": 0.8809139728546143, "sampling/importance_sampling_ratio/min": 0.045162029564380646, "sampling/sampling_logp_difference/max": 1.9286136627197266, "sampling/sampling_logp_difference/mean": 0.13844287395477295, "step": 236, "step_time": 13.932450453990896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 98.375, "completions/mean_terminated_length": 98.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2906048290897161, "epoch": 0.00474, "frac_reward_zero_std": 0.0, "grad_norm": 0.3309686779975891, "kl": 1.7407237198203802, "learning_rate": 9.044374741394859e-06, "loss": -0.0122, "num_tokens": 10759806.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.5914009809494019, "sampling/importance_sampling_ratio/mean": 0.8816614151000977, "sampling/importance_sampling_ratio/min": 0.09863068908452988, "sampling/sampling_logp_difference/max": 1.8567988872528076, "sampling/sampling_logp_difference/mean": 0.12452438473701477, "step": 237, "step_time": 11.658190926005773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0022321429569274187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022321429569274187, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 112.75, "completions/mean_terminated_length": 112.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22420509159564972, "epoch": 0.00476, "frac_reward_zero_std": 0.0, "grad_norm": 0.8847730159759521, "kl": 1.6300090067088604, "learning_rate": 9.035273700834707e-06, "loss": -0.0029, "num_tokens": 10804960.0, "reward": 0.07249999046325684, "reward_std": 1.0535087585449219, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.5432884693145752, "sampling/importance_sampling_ratio/mean": 0.9213290810585022, "sampling/importance_sampling_ratio/min": 0.12823045253753662, "sampling/sampling_logp_difference/max": 2.35107159614563, "sampling/sampling_logp_difference/mean": 0.11580392718315125, "step": 238, "step_time": 11.680019646999426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.017578125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017578125, "completions/clipped_ratio": 0.03125, "completions/max_length": 682.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 83.5625, "completions/mean_terminated_length": 85.74193572998047, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3107682524714619, "epoch": 0.00478, "frac_reward_zero_std": 0.0, "grad_norm": 0.671769380569458, "kl": 1.4389739595353603, "learning_rate": 9.026135920631592e-06, "loss": 0.1234, "num_tokens": 10850783.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.842463970184326, "sampling/importance_sampling_ratio/mean": 0.8858070373535156, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.450885772705078, "sampling/sampling_logp_difference/mean": 0.1351800262928009, "step": 239, "step_time": 12.83354129701911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 98.5625, "completions/mean_terminated_length": 98.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1803421201184392, "epoch": 0.0048, "frac_reward_zero_std": 0.0, "grad_norm": 0.5469832420349121, "kl": 3.0126063376665115, "learning_rate": 9.016961521319006e-06, "loss": 0.0205, "num_tokens": 10896643.0, "reward": 0.07312498986721039, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.8463276624679565, "sampling/importance_sampling_ratio/mean": 0.9392499923706055, "sampling/importance_sampling_ratio/min": 0.1869966983795166, "sampling/sampling_logp_difference/max": 1.6425271034240723, "sampling/sampling_logp_difference/mean": 0.09512656927108765, "step": 240, "step_time": 11.686757642986777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 637.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 121.96875, "completions/mean_terminated_length": 121.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17474013334140182, "epoch": 0.00482, "frac_reward_zero_std": 0.0, "grad_norm": 1.8881489038467407, "kl": 3.7348453029990196, "learning_rate": 9.00775062391348e-06, "loss": 0.1948, "num_tokens": 10943335.0, "reward": 0.07312498986721039, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 2.525247097015381, "sampling/importance_sampling_ratio/mean": 1.0530624389648438, "sampling/importance_sampling_ratio/min": 0.06588254868984222, "sampling/sampling_logp_difference/max": 2.437371253967285, "sampling/sampling_logp_difference/mean": 0.06996364891529083, "step": 241, "step_time": 12.446822688987595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 565.0, "completions/max_terminated_length": 565.0, "completions/mean_length": 70.40625, "completions/mean_terminated_length": 70.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16751019470393658, "epoch": 0.00484, "frac_reward_zero_std": 0.0, "grad_norm": 0.27049657702445984, "kl": 1.820386229082942, "learning_rate": 8.998503349912977e-06, "loss": 0.0939, "num_tokens": 10989694.0, "reward": -0.3700000047683716, "reward_std": 0.8781067728996277, "rewards/rollout_reward_func/mean": -0.3700000047683716, "rewards/rollout_reward_func/std": 0.9494174122810364, "sampling/importance_sampling_ratio/max": 1.7343555688858032, "sampling/importance_sampling_ratio/mean": 0.9713786244392395, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1479859352111816, "sampling/sampling_logp_difference/mean": 0.08548849821090698, "step": 242, "step_time": 11.81218020999222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1213.0, "completions/max_terminated_length": 1213.0, "completions/mean_length": 133.6875, "completions/mean_terminated_length": 133.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1328369383700192, "epoch": 0.00486, "frac_reward_zero_std": 0.0, "grad_norm": 0.29142439365386963, "kl": 1.8387114759534597, "learning_rate": 8.989219821295294e-06, "loss": 0.0443, "num_tokens": 11038736.0, "reward": -0.30625003576278687, "reward_std": 0.9515185356140137, "rewards/rollout_reward_func/mean": -0.30625003576278687, "rewards/rollout_reward_func/std": 0.9738972187042236, "sampling/importance_sampling_ratio/max": 1.3430159091949463, "sampling/importance_sampling_ratio/mean": 1.003828763961792, "sampling/importance_sampling_ratio/min": 0.18795625865459442, "sampling/sampling_logp_difference/max": 1.6902832984924316, "sampling/sampling_logp_difference/mean": 0.052009519189596176, "step": 243, "step_time": 14.720332826029335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 538.0, "completions/max_terminated_length": 538.0, "completions/mean_length": 81.0625, "completions/mean_terminated_length": 81.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1302615983877331, "epoch": 0.00488, "frac_reward_zero_std": 0.0, "grad_norm": 0.4144216775894165, "kl": 3.4126515723764896, "learning_rate": 8.979900160516447e-06, "loss": 0.0192, "num_tokens": 11084539.0, "reward": -0.05375000834465027, "reward_std": 1.053357720375061, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.6966495513916016, "sampling/importance_sampling_ratio/mean": 1.0024367570877075, "sampling/importance_sampling_ratio/min": 0.131053164601326, "sampling/sampling_logp_difference/max": 1.3138904571533203, "sampling/sampling_logp_difference/mean": 0.06879197806119919, "step": 244, "step_time": 11.918087997000839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 637.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 68.8125, "completions/mean_terminated_length": 68.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14865460037253797, "epoch": 0.0049, "frac_reward_zero_std": 0.0, "grad_norm": 0.3909507393836975, "kl": 2.3512891177088022, "learning_rate": 8.970544490509064e-06, "loss": 0.0352, "num_tokens": 11130870.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.8505966663360596, "sampling/importance_sampling_ratio/mean": 0.8869497776031494, "sampling/importance_sampling_ratio/min": 0.19314080476760864, "sampling/sampling_logp_difference/max": 1.650320053100586, "sampling/sampling_logp_difference/mean": 0.0932864248752594, "step": 245, "step_time": 12.214539304004575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.027901785913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.027901785913854837, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 72.375, "completions/mean_terminated_length": 72.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2332285693846643, "epoch": 0.00492, "frac_reward_zero_std": 0.0, "grad_norm": 0.32802507281303406, "kl": 2.604243189096451, "learning_rate": 8.961152934680752e-06, "loss": -0.0837, "num_tokens": 11177348.0, "reward": -0.36937499046325684, "reward_std": 0.9429470300674438, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.6376525163650513, "sampling/importance_sampling_ratio/mean": 0.7958878874778748, "sampling/importance_sampling_ratio/min": 0.10347403585910797, "sampling/sampling_logp_difference/max": 2.2218337059020996, "sampling/sampling_logp_difference/mean": 0.141080841422081, "step": 246, "step_time": 11.789451703014493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 95.21875, "completions/mean_terminated_length": 95.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13534964341670275, "epoch": 0.00494, "frac_reward_zero_std": 0.0, "grad_norm": 0.2006608098745346, "kl": 1.4691822212189436, "learning_rate": 8.951725616912489e-06, "loss": 0.0813, "num_tokens": 11222744.0, "reward": 0.07249999046325684, "reward_std": 1.0535087585449219, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.2555208206176758, "sampling/importance_sampling_ratio/mean": 0.9245393872261047, "sampling/importance_sampling_ratio/min": 0.07788310199975967, "sampling/sampling_logp_difference/max": 2.025334358215332, "sampling/sampling_logp_difference/mean": 0.08631512522697449, "step": 247, "step_time": 12.514046635980776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 367.0, "completions/max_terminated_length": 367.0, "completions/mean_length": 59.3125, "completions/mean_terminated_length": 59.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15821737330406904, "epoch": 0.00496, "frac_reward_zero_std": 0.0, "grad_norm": 0.3658364415168762, "kl": 2.8647869639098644, "learning_rate": 8.942262661556962e-06, "loss": -0.0146, "num_tokens": 11268357.0, "reward": -0.11750000715255737, "reward_std": 0.989517867565155, "rewards/rollout_reward_func/mean": -0.11750000715255737, "rewards/rollout_reward_func/std": 1.0166960954666138, "sampling/importance_sampling_ratio/max": 1.1828162670135498, "sampling/importance_sampling_ratio/mean": 0.8473047018051147, "sampling/importance_sampling_ratio/min": 2.8453320255472647e-10, "sampling/sampling_logp_difference/max": 17.5999813079834, "sampling/sampling_logp_difference/mean": 0.19875508546829224, "step": 248, "step_time": 11.249730991999968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 833.0, "completions/max_terminated_length": 833.0, "completions/mean_length": 162.40625, "completions/mean_terminated_length": 162.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1767854643985629, "epoch": 0.00498, "frac_reward_zero_std": 0.0, "grad_norm": 1.2006113529205322, "kl": 2.7082099709659815, "learning_rate": 8.932764193436953e-06, "loss": 0.1223, "num_tokens": 11317397.0, "reward": -0.18000000715255737, "reward_std": 0.9876828193664551, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.574161171913147, "sampling/importance_sampling_ratio/mean": 0.9679964780807495, "sampling/importance_sampling_ratio/min": 0.22696496546268463, "sampling/sampling_logp_difference/max": 2.0233309268951416, "sampling/sampling_logp_difference/mean": 0.05718069523572922, "step": 249, "step_time": 13.364791238986072 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1285.0, "completions/max_terminated_length": 1285.0, "completions/mean_length": 117.34375, "completions/mean_terminated_length": 117.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19022899842821062, "epoch": 0.005, "frac_reward_zero_std": 0.0, "grad_norm": 0.47203516960144043, "kl": 2.1220393646508455, "learning_rate": 8.923230337843675e-06, "loss": 0.0838, "num_tokens": 11365884.0, "reward": -0.43187499046325684, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 2.2153372764587402, "sampling/importance_sampling_ratio/mean": 0.8962876200675964, "sampling/importance_sampling_ratio/min": 0.22875259816646576, "sampling/sampling_logp_difference/max": 1.5133107900619507, "sampling/sampling_logp_difference/mean": 0.0807034969329834, "step": 250, "step_time": 15.058379413989314 }, { "clip_ratio/high_max": 0.0052083334885537624, "clip_ratio/high_mean": 0.0026041667442768812, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026041667442768812, "completions/clipped_ratio": 0.0, "completions/max_length": 1238.0, "completions/max_terminated_length": 1238.0, "completions/mean_length": 115.71875, "completions/mean_terminated_length": 115.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11545472987927496, "epoch": 0.00502, "frac_reward_zero_std": 0.0, "grad_norm": 0.7233448624610901, "kl": 1.636860202997923, "learning_rate": 8.913661220535132e-06, "loss": 0.0749, "num_tokens": 11412806.0, "reward": -0.053125008940696716, "reward_std": 1.0711638927459717, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.9205763339996338, "sampling/importance_sampling_ratio/mean": 1.06536865234375, "sampling/importance_sampling_ratio/min": 0.34038299322128296, "sampling/sampling_logp_difference/max": 1.0871586799621582, "sampling/sampling_logp_difference/mean": 0.05039035528898239, "step": 251, "step_time": 15.119843828986632 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 73.0625, "completions/mean_terminated_length": 73.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12666726228781044, "epoch": 0.00504, "frac_reward_zero_std": 0.0, "grad_norm": 0.40115317702293396, "kl": 2.6119855623692274, "learning_rate": 8.904056967734447e-06, "loss": 0.0075, "num_tokens": 11459831.0, "reward": -0.36937499046325684, "reward_std": 0.9427522420883179, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.907837986946106, "sampling/importance_sampling_ratio/mean": 1.035057544708252, "sampling/importance_sampling_ratio/min": 0.19715355336666107, "sampling/sampling_logp_difference/max": 1.947399377822876, "sampling/sampling_logp_difference/mean": 0.04552160203456879, "step": 252, "step_time": 11.277833818989166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 554.0, "completions/max_terminated_length": 554.0, "completions/mean_length": 97.90625, "completions/mean_terminated_length": 97.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11871610302478075, "epoch": 0.00506, "frac_reward_zero_std": 0.0, "grad_norm": 0.24178141355514526, "kl": 1.3185951597988605, "learning_rate": 8.894417706128209e-06, "loss": 0.0229, "num_tokens": 11505358.0, "reward": 0.38874998688697815, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": 0.38874998688697815, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.3237954378128052, "sampling/importance_sampling_ratio/mean": 0.9965953826904297, "sampling/importance_sampling_ratio/min": 0.374569296836853, "sampling/sampling_logp_difference/max": 1.000170350074768, "sampling/sampling_logp_difference/mean": 0.036074474453926086, "step": 253, "step_time": 11.651951374980854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 682.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 82.8125, "completions/mean_terminated_length": 82.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1016889731399715, "epoch": 0.00508, "frac_reward_zero_std": 0.0, "grad_norm": 0.5410573482513428, "kl": 1.5230537503957748, "learning_rate": 8.884743562864792e-06, "loss": 0.1344, "num_tokens": 11550609.0, "reward": -0.05375000834465027, "reward_std": 1.0167031288146973, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.8713279962539673, "sampling/importance_sampling_ratio/mean": 1.0753636360168457, "sampling/importance_sampling_ratio/min": 0.47712379693984985, "sampling/sampling_logp_difference/max": 1.3516427278518677, "sampling/sampling_logp_difference/mean": 0.03756985813379288, "step": 254, "step_time": 12.413962791986705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.020089285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020089285913854837, "completions/clipped_ratio": 0.0, "completions/max_length": 1114.0, "completions/max_terminated_length": 1114.0, "completions/mean_length": 69.40625, "completions/mean_terminated_length": 69.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1498743963893503, "epoch": 0.0051, "frac_reward_zero_std": 0.0, "grad_norm": 0.7328459620475769, "kl": 2.1107621882110834, "learning_rate": 8.875034665552686e-06, "loss": 0.0448, "num_tokens": 11596696.0, "reward": -0.24375000596046448, "reward_std": 1.0055902004241943, "rewards/rollout_reward_func/mean": -0.24375000596046448, "rewards/rollout_reward_func/std": 0.9919474124908447, "sampling/importance_sampling_ratio/max": 1.6543537378311157, "sampling/importance_sampling_ratio/mean": 1.0411864519119263, "sampling/importance_sampling_ratio/min": 0.07630592584609985, "sampling/sampling_logp_difference/max": 2.1116161346435547, "sampling/sampling_logp_difference/mean": 0.05976799130439758, "step": 255, "step_time": 14.171211854016292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 509.0, "completions/max_terminated_length": 509.0, "completions/mean_length": 85.78125, "completions/mean_terminated_length": 85.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1518457611091435, "epoch": 0.00512, "frac_reward_zero_std": 0.0, "grad_norm": 0.30291110277175903, "kl": 2.4234049823135138, "learning_rate": 8.865291142258813e-06, "loss": 0.0494, "num_tokens": 11642556.0, "reward": 0.13624998927116394, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.8539557456970215, "sampling/importance_sampling_ratio/mean": 1.05775785446167, "sampling/importance_sampling_ratio/min": 0.23167262971401215, "sampling/sampling_logp_difference/max": 1.6787261962890625, "sampling/sampling_logp_difference/mean": 0.049068719148635864, "step": 256, "step_time": 12.220216695008276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 554.0, "completions/max_terminated_length": 554.0, "completions/mean_length": 82.625, "completions/mean_terminated_length": 82.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2512075281701982, "epoch": 0.00514, "frac_reward_zero_std": 0.0, "grad_norm": 0.48503434658050537, "kl": 1.6400213222950697, "learning_rate": 8.855513121506826e-06, "loss": 0.038, "num_tokens": 11690620.0, "reward": -0.3687500059604645, "reward_std": 0.9712425470352173, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 2.375662326812744, "sampling/importance_sampling_ratio/mean": 0.9356033205986023, "sampling/importance_sampling_ratio/min": 0.21448004245758057, "sampling/sampling_logp_difference/max": 0.854459285736084, "sampling/sampling_logp_difference/mean": 0.0859653428196907, "step": 257, "step_time": 11.61391564099904 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 84.21875, "completions/mean_terminated_length": 84.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16601469228044152, "epoch": 0.00516, "frac_reward_zero_std": 0.0, "grad_norm": 0.4697384834289551, "kl": 1.2090565841645002, "learning_rate": 8.845700732275435e-06, "loss": -0.0567, "num_tokens": 11737465.0, "reward": 0.009999990463256836, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.4062037467956543, "sampling/importance_sampling_ratio/mean": 0.9935895800590515, "sampling/importance_sampling_ratio/min": 0.5147256255149841, "sampling/sampling_logp_difference/max": 0.6791185140609741, "sampling/sampling_logp_difference/mean": 0.05209371820092201, "step": 258, "step_time": 11.689063434016134 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 545.0, "completions/max_terminated_length": 545.0, "completions/mean_length": 82.0, "completions/mean_terminated_length": 82.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10831702244468033, "epoch": 0.00518, "frac_reward_zero_std": 0.0, "grad_norm": 0.6068504452705383, "kl": 1.2792255990207195, "learning_rate": 8.835854103996683e-06, "loss": 0.1033, "num_tokens": 11784069.0, "reward": 0.07187499105930328, "reward_std": 1.0334783792495728, "rewards/rollout_reward_func/mean": 0.07187499105930328, "rewards/rollout_reward_func/std": 1.0229730606079102, "sampling/importance_sampling_ratio/max": 1.3275953531265259, "sampling/importance_sampling_ratio/mean": 1.054015874862671, "sampling/importance_sampling_ratio/min": 0.5195345282554626, "sampling/sampling_logp_difference/max": 0.6797178983688354, "sampling/sampling_logp_difference/mean": 0.03672632575035095, "step": 259, "step_time": 12.04691001799074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1247.0, "completions/max_terminated_length": 1247.0, "completions/mean_length": 124.75, "completions/mean_terminated_length": 124.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20564253185875714, "epoch": 0.0052, "frac_reward_zero_std": 0.0, "grad_norm": 0.47013214230537415, "kl": 1.225655822083354, "learning_rate": 8.825973366554254e-06, "loss": 0.0803, "num_tokens": 11833153.0, "reward": -0.24375000596046448, "reward_std": 1.024712085723877, "rewards/rollout_reward_func/mean": -0.24375000596046448, "rewards/rollout_reward_func/std": 0.9919474124908447, "sampling/importance_sampling_ratio/max": 1.6310900449752808, "sampling/importance_sampling_ratio/mean": 1.0458319187164307, "sampling/importance_sampling_ratio/min": 0.5014946460723877, "sampling/sampling_logp_difference/max": 0.7227718830108643, "sampling/sampling_logp_difference/mean": 0.051300521939992905, "step": 260, "step_time": 14.560179970001627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 68.71875, "completions/mean_terminated_length": 68.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2720786619465798, "epoch": 0.00522, "frac_reward_zero_std": 0.0, "grad_norm": 0.3666876256465912, "kl": 1.3488037623465061, "learning_rate": 8.816058650281758e-06, "loss": 0.0142, "num_tokens": 11879206.0, "reward": -0.3687500059604645, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.3476459980010986, "sampling/importance_sampling_ratio/mean": 0.852741003036499, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4854285717010498, "sampling/sampling_logp_difference/mean": 0.09837142378091812, "step": 261, "step_time": 12.258569875004468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 438.0, "completions/max_terminated_length": 438.0, "completions/mean_length": 29.625, "completions/mean_terminated_length": 29.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2278089066967368, "epoch": 0.00524, "frac_reward_zero_std": 0.0, "grad_norm": 0.4830424189567566, "kl": 0.9781239777803421, "learning_rate": 8.806110085960999e-06, "loss": 0.0438, "num_tokens": 11923302.0, "reward": -0.11625000834465027, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.3859620094299316, "sampling/importance_sampling_ratio/mean": 0.9318643808364868, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.450307846069336, "sampling/sampling_logp_difference/mean": 0.08035841584205627, "step": 262, "step_time": 11.224801349984773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 62.0625, "completions/mean_terminated_length": 62.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16259809560142457, "epoch": 0.00526, "frac_reward_zero_std": 0.0, "grad_norm": 0.48653459548950195, "kl": 1.226403208449483, "learning_rate": 8.79612780482027e-06, "loss": 0.0806, "num_tokens": 11968943.0, "reward": -0.17937500774860382, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.4645367860794067, "sampling/importance_sampling_ratio/mean": 1.0544689893722534, "sampling/importance_sampling_ratio/min": 0.5433586835861206, "sampling/sampling_logp_difference/max": 0.6049861907958984, "sampling/sampling_logp_difference/mean": 0.05459568277001381, "step": 263, "step_time": 11.093287209012487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 608.0, "completions/max_terminated_length": 608.0, "completions/mean_length": 48.71875, "completions/mean_terminated_length": 48.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.253641210263595, "epoch": 0.00528, "frac_reward_zero_std": 0.0, "grad_norm": 0.5159488320350647, "kl": 0.9370337873697281, "learning_rate": 8.786111938532606e-06, "loss": -0.0017, "num_tokens": 12012476.0, "reward": 0.009999990463256836, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.5488966703414917, "sampling/importance_sampling_ratio/mean": 0.9657322764396667, "sampling/importance_sampling_ratio/min": 0.5556706190109253, "sampling/sampling_logp_difference/max": 0.5907502174377441, "sampling/sampling_logp_difference/mean": 0.06445999443531036, "step": 264, "step_time": 12.033950013996218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 73.21875, "completions/mean_terminated_length": 73.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3120930517325178, "epoch": 0.0053, "frac_reward_zero_std": 0.0, "grad_norm": 0.6441860795021057, "kl": 2.162873538210988, "learning_rate": 8.776062619214053e-06, "loss": 0.0057, "num_tokens": 12058103.0, "reward": -0.17937500774860382, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.5538939237594604, "sampling/importance_sampling_ratio/mean": 0.9574371576309204, "sampling/importance_sampling_ratio/min": 0.4058208465576172, "sampling/sampling_logp_difference/max": 0.9027112722396851, "sampling/sampling_logp_difference/mean": 0.1077428013086319, "step": 265, "step_time": 12.239974773030553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 599.0, "completions/max_terminated_length": 599.0, "completions/mean_length": 41.78125, "completions/mean_terminated_length": 41.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3739831327693537, "epoch": 0.00532, "frac_reward_zero_std": 0.0, "grad_norm": 1.014735221862793, "kl": 7.117995025590062, "learning_rate": 8.765979979421928e-06, "loss": -0.0855, "num_tokens": 12103261.0, "reward": -0.24250000715255737, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 2.8152201175689697, "sampling/importance_sampling_ratio/mean": 0.880955696105957, "sampling/importance_sampling_ratio/min": 0.352245956659317, "sampling/sampling_logp_difference/max": 1.12457275390625, "sampling/sampling_logp_difference/mean": 0.13256672024726868, "step": 266, "step_time": 11.483550884993747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1121.0, "completions/max_terminated_length": 1121.0, "completions/mean_length": 144.0, "completions/mean_terminated_length": 148.1290283203125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.33205682726111263, "epoch": 0.00534, "frac_reward_zero_std": 0.0, "grad_norm": 0.4483926594257355, "kl": 1.517567852512002, "learning_rate": 8.755864152153064e-06, "loss": 0.0249, "num_tokens": 12152016.0, "reward": 0.07124999165534973, "reward_std": 1.015291690826416, "rewards/rollout_reward_func/mean": 0.07124999165534973, "rewards/rollout_reward_func/std": 1.0223935842514038, "sampling/importance_sampling_ratio/max": 2.205949544906616, "sampling/importance_sampling_ratio/mean": 0.8667049407958984, "sampling/importance_sampling_ratio/min": 0.33239850401878357, "sampling/sampling_logp_difference/max": 0.8536747694015503, "sampling/sampling_logp_difference/mean": 0.0943973958492279, "step": 267, "step_time": 14.40349427799083 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 652.0, "completions/max_terminated_length": 652.0, "completions/mean_length": 66.0625, "completions/mean_terminated_length": 66.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25180500582791865, "epoch": 0.00536, "frac_reward_zero_std": 0.0, "grad_norm": 0.32170554995536804, "kl": 2.350392611697316, "learning_rate": 8.745715270842064e-06, "loss": 0.0406, "num_tokens": 12197754.0, "reward": -0.17937500774860382, "reward_std": 0.9883459806442261, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.8498414754867554, "sampling/importance_sampling_ratio/mean": 1.0057932138442993, "sampling/importance_sampling_ratio/min": 0.4090912342071533, "sampling/sampling_logp_difference/max": 0.9455058574676514, "sampling/sampling_logp_difference/mean": 0.08190833777189255, "step": 268, "step_time": 12.31659497700457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 419.0, "completions/max_terminated_length": 419.0, "completions/mean_length": 25.9375, "completions/mean_terminated_length": 26.25806427001953, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28178472339641303, "epoch": 0.00538, "frac_reward_zero_std": 0.0, "grad_norm": 0.1695215255022049, "kl": 1.583856901153922, "learning_rate": 8.73553346935953e-06, "loss": 0.0376, "num_tokens": 12242690.0, "reward": -0.36875003576278687, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 2.87719464302063, "sampling/importance_sampling_ratio/mean": 0.9957253932952881, "sampling/importance_sampling_ratio/min": 0.2941019535064697, "sampling/sampling_logp_difference/max": 1.1465086936950684, "sampling/sampling_logp_difference/mean": 0.09289535135030746, "step": 269, "step_time": 10.801463211006194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 102.90625, "completions/mean_terminated_length": 102.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24997285124845803, "epoch": 0.0054, "frac_reward_zero_std": 0.0, "grad_norm": 0.3373571038246155, "kl": 1.4692413304001093, "learning_rate": 8.725318882010307e-06, "loss": 0.0421, "num_tokens": 12289732.0, "reward": -0.36875003576278687, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.7761656045913696, "sampling/importance_sampling_ratio/mean": 1.0466651916503906, "sampling/importance_sampling_ratio/min": 0.33315786719322205, "sampling/sampling_logp_difference/max": 0.7981306314468384, "sampling/sampling_logp_difference/mean": 0.07437478750944138, "step": 270, "step_time": 15.439849391004827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 590.0, "completions/max_terminated_length": 590.0, "completions/mean_length": 71.5, "completions/mean_terminated_length": 71.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20986357983201742, "epoch": 0.00542, "frac_reward_zero_std": 0.0, "grad_norm": 0.22201591730117798, "kl": 1.517797777429223, "learning_rate": 8.715071643531708e-06, "loss": 0.0553, "num_tokens": 12336432.0, "reward": -0.18000000715255737, "reward_std": 0.9981721639633179, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.5389440059661865, "sampling/importance_sampling_ratio/mean": 1.0414639711380005, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.038465976715088, "sampling/sampling_logp_difference/mean": 0.06172827631235123, "step": 271, "step_time": 12.165501288007363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0028409091755747795, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028409091755747795, "completions/clipped_ratio": 0.03125, "completions/max_length": 653.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 142.0625, "completions/mean_terminated_length": 146.1290283203125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24345131823793054, "epoch": 0.00544, "frac_reward_zero_std": 0.0, "grad_norm": 0.52916020154953, "kl": 3.63723685964942, "learning_rate": 8.704791889091737e-06, "loss": -0.0219, "num_tokens": 12385358.0, "reward": -0.3687500059604645, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.4893014430999756, "sampling/importance_sampling_ratio/mean": 0.9702180624008179, "sampling/importance_sampling_ratio/min": 0.188723623752594, "sampling/sampling_logp_difference/max": 1.2349424362182617, "sampling/sampling_logp_difference/mean": 0.06525556743144989, "step": 272, "step_time": 12.19016081499285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 617.0, "completions/max_terminated_length": 617.0, "completions/mean_length": 94.1875, "completions/mean_terminated_length": 94.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2378421612083912, "epoch": 0.00546, "frac_reward_zero_std": 0.0, "grad_norm": 0.3787650167942047, "kl": 4.238742724061012, "learning_rate": 8.6944797542873e-06, "loss": -0.006, "num_tokens": 12432303.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.4172455072402954, "sampling/importance_sampling_ratio/mean": 0.8925941586494446, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.036755323410034, "sampling/sampling_logp_difference/mean": 0.11011795699596405, "step": 273, "step_time": 12.374660530993424 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 662.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 62.28125, "completions/mean_terminated_length": 62.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22515266854315996, "epoch": 0.00548, "frac_reward_zero_std": 0.0, "grad_norm": 0.27687206864356995, "kl": 1.7011241558939219, "learning_rate": 8.68413537514243e-06, "loss": 0.1099, "num_tokens": 12477416.0, "reward": -0.17937500774860382, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.5660696029663086, "sampling/importance_sampling_ratio/mean": 0.9843167066574097, "sampling/importance_sampling_ratio/min": 0.2461080253124237, "sampling/sampling_logp_difference/max": 1.4341768026351929, "sampling/sampling_logp_difference/mean": 0.061435192823410034, "step": 274, "step_time": 12.169888075004565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 637.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 63.25, "completions/mean_terminated_length": 63.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29419540194794536, "epoch": 0.0055, "frac_reward_zero_std": 0.0, "grad_norm": 0.34060850739479065, "kl": 1.8607396055012941, "learning_rate": 8.673758888106481e-06, "loss": 0.0922, "num_tokens": 12522685.0, "reward": -0.24312500655651093, "reward_std": 0.9791113138198853, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.8650884628295898, "sampling/importance_sampling_ratio/mean": 0.9800959825515747, "sampling/importance_sampling_ratio/min": 0.21875892579555511, "sampling/sampling_logp_difference/max": 1.5545157194137573, "sampling/sampling_logp_difference/mean": 0.09325528144836426, "step": 275, "step_time": 12.238628388004145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.0, "completions/max_length": 673.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 89.125, "completions/mean_terminated_length": 89.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2885579336434603, "epoch": 0.00552, "frac_reward_zero_std": 0.0, "grad_norm": 0.6129008531570435, "kl": 1.8571998551487923, "learning_rate": 8.663350430052328e-06, "loss": 0.0973, "num_tokens": 12569601.0, "reward": -0.11625000834465027, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.432018280029297, "sampling/importance_sampling_ratio/mean": 1.0127321481704712, "sampling/importance_sampling_ratio/min": 0.07426733523607254, "sampling/sampling_logp_difference/max": 1.1669983863830566, "sampling/sampling_logp_difference/mean": 0.08720683306455612, "step": 276, "step_time": 12.06151922900608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 455.0, "completions/max_terminated_length": 455.0, "completions/mean_length": 42.90625, "completions/mean_terminated_length": 42.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2866360442712903, "epoch": 0.00554, "frac_reward_zero_std": 0.0, "grad_norm": 0.2650032639503479, "kl": 1.3474781662225723, "learning_rate": 8.652910138274573e-06, "loss": 0.0396, "num_tokens": 12612922.0, "reward": 0.26249998807907104, "reward_std": 1.0074068307876587, "rewards/rollout_reward_func/mean": 0.26249998807907104, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.3518446683883667, "sampling/importance_sampling_ratio/mean": 0.8843410015106201, "sampling/importance_sampling_ratio/min": 0.23780101537704468, "sampling/sampling_logp_difference/max": 1.2066102027893066, "sampling/sampling_logp_difference/mean": 0.10268360376358032, "step": 277, "step_time": 11.177177764009684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 62.84375, "completions/mean_terminated_length": 62.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31966405687853694, "epoch": 0.00556, "frac_reward_zero_std": 0.0, "grad_norm": 0.4590837359428406, "kl": 1.7775117866694927, "learning_rate": 8.642438150487718e-06, "loss": -0.0245, "num_tokens": 12658090.0, "reward": -0.11687500774860382, "reward_std": 0.9791113138198853, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.3595048189163208, "sampling/importance_sampling_ratio/mean": 0.9544181823730469, "sampling/importance_sampling_ratio/min": 0.27158603072166443, "sampling/sampling_logp_difference/max": 0.7669291496276855, "sampling/sampling_logp_difference/mean": 0.08003604412078857, "step": 278, "step_time": 11.587250145006692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.0, "completions/max_terminated_length": 330.0, "completions/mean_length": 13.90625, "completions/mean_terminated_length": 13.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3564479434862733, "epoch": 0.00558, "frac_reward_zero_std": 0.0, "grad_norm": 0.20560313761234283, "kl": 1.4948947988450527, "learning_rate": 8.631934604824366e-06, "loss": 0.0701, "num_tokens": 12702045.0, "reward": -0.3687500059604645, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.4298968315124512, "sampling/importance_sampling_ratio/mean": 0.8956043720245361, "sampling/importance_sampling_ratio/min": 0.24468688666820526, "sampling/sampling_logp_difference/max": 1.3628246784210205, "sampling/sampling_logp_difference/mean": 0.11410669982433319, "step": 279, "step_time": 10.870718487989507 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 97.53125, "completions/mean_terminated_length": 97.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4125442886725068, "epoch": 0.0056, "frac_reward_zero_std": 0.0, "grad_norm": 0.3662418723106384, "kl": 1.922801285982132, "learning_rate": 8.62139963983338e-06, "loss": -0.0929, "num_tokens": 12750376.0, "reward": -0.3068750202655792, "reward_std": 0.8791072368621826, "rewards/rollout_reward_func/mean": -0.3068750202655792, "rewards/rollout_reward_func/std": 0.9730378985404968, "sampling/importance_sampling_ratio/max": 1.393816351890564, "sampling/importance_sampling_ratio/mean": 0.9217624664306641, "sampling/importance_sampling_ratio/min": 0.29069554805755615, "sampling/sampling_logp_difference/max": 1.1203539371490479, "sampling/sampling_logp_difference/mean": 0.08484220504760742, "step": 280, "step_time": 11.72061001900147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1294.0, "completions/max_terminated_length": 1294.0, "completions/mean_length": 84.125, "completions/mean_terminated_length": 84.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3434428097680211, "epoch": 0.00562, "frac_reward_zero_std": 0.0, "grad_norm": 0.5311670303344727, "kl": 1.439142368733883, "learning_rate": 8.610833394478073e-06, "loss": 0.1229, "num_tokens": 12797787.0, "reward": -0.3687500059604645, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 2.505481719970703, "sampling/importance_sampling_ratio/mean": 1.0627014636993408, "sampling/importance_sampling_ratio/min": 0.4756091833114624, "sampling/sampling_logp_difference/max": 0.6459312438964844, "sampling/sampling_logp_difference/mean": 0.07033371925354004, "step": 281, "step_time": 15.206139095993422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 608.0, "completions/max_terminated_length": 608.0, "completions/mean_length": 53.0625, "completions/mean_terminated_length": 53.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4433027277700603, "epoch": 0.00564, "frac_reward_zero_std": 0.0, "grad_norm": 0.43747058510780334, "kl": 1.78975828923285, "learning_rate": 8.60023600813436e-06, "loss": 0.078, "num_tokens": 12843301.0, "reward": -0.17937500774860382, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.4327373504638672, "sampling/importance_sampling_ratio/mean": 0.8810456991195679, "sampling/importance_sampling_ratio/min": 0.22415605187416077, "sampling/sampling_logp_difference/max": 1.2355568408966064, "sampling/sampling_logp_difference/mean": 0.1254313886165619, "step": 282, "step_time": 11.945918605008046 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 68.96875, "completions/mean_terminated_length": 68.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37398894876241684, "epoch": 0.00566, "frac_reward_zero_std": 0.0, "grad_norm": 0.34368669986724854, "kl": 1.2215432357043028, "learning_rate": 8.589607620588937e-06, "loss": 0.0475, "num_tokens": 12889803.0, "reward": -0.36875003576278687, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.7385473251342773, "sampling/importance_sampling_ratio/mean": 0.9598242044448853, "sampling/importance_sampling_ratio/min": 0.1837717741727829, "sampling/sampling_logp_difference/max": 1.2949614524841309, "sampling/sampling_logp_difference/mean": 0.0874956026673317, "step": 283, "step_time": 11.666394970998226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 583.0, "completions/max_terminated_length": 583.0, "completions/mean_length": 58.03125, "completions/mean_terminated_length": 58.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3157727983780205, "epoch": 0.00568, "frac_reward_zero_std": 0.0, "grad_norm": 0.3847329318523407, "kl": 1.2410346269607544, "learning_rate": 8.578948372037416e-06, "loss": 0.0545, "num_tokens": 12934833.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.4836070537567139, "sampling/importance_sampling_ratio/mean": 1.0368068218231201, "sampling/importance_sampling_ratio/min": 0.44333457946777344, "sampling/sampling_logp_difference/max": 0.4500417709350586, "sampling/sampling_logp_difference/mean": 0.053515177220106125, "step": 284, "step_time": 12.04990541299776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 865.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 75.9375, "completions/mean_terminated_length": 75.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.43714595260098577, "epoch": 0.0057, "frac_reward_zero_std": 0.0, "grad_norm": 0.590287446975708, "kl": 1.4424633793532848, "learning_rate": 8.568258403082492e-06, "loss": 0.0306, "num_tokens": 12981933.0, "reward": -0.36875003576278687, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767791748047, "sampling/importance_sampling_ratio/max": 1.8264695405960083, "sampling/importance_sampling_ratio/mean": 0.9901168942451477, "sampling/importance_sampling_ratio/min": 0.307711660861969, "sampling/sampling_logp_difference/max": 1.2223340272903442, "sampling/sampling_logp_difference/mean": 0.10441485047340393, "step": 285, "step_time": 13.414746734968503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 59.59375, "completions/mean_terminated_length": 59.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3760384516790509, "epoch": 0.00572, "frac_reward_zero_std": 0.0, "grad_norm": 0.31694701313972473, "kl": 1.0588195100426674, "learning_rate": 8.557537854732082e-06, "loss": 0.0629, "num_tokens": 13026244.0, "reward": -0.17937500774860382, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.486361026763916, "sampling/importance_sampling_ratio/mean": 0.9869402647018433, "sampling/importance_sampling_ratio/min": 0.3723195195198059, "sampling/sampling_logp_difference/max": 0.6992855072021484, "sampling/sampling_logp_difference/mean": 0.07934851944446564, "step": 286, "step_time": 13.971959876005712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 47.65625, "completions/mean_terminated_length": 47.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.35385291930288076, "epoch": 0.00574, "frac_reward_zero_std": 0.0, "grad_norm": 0.4025495946407318, "kl": 1.1272316239774227, "learning_rate": 8.546786868397465e-06, "loss": 0.0145, "num_tokens": 13069856.0, "reward": -0.05437500774860382, "reward_std": 1.0168324708938599, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228137969970703, "sampling/importance_sampling_ratio/max": 1.5559370517730713, "sampling/importance_sampling_ratio/mean": 0.9090474843978882, "sampling/importance_sampling_ratio/min": 0.2944495975971222, "sampling/sampling_logp_difference/max": 1.0645077228546143, "sampling/sampling_logp_difference/mean": 0.10219350457191467, "step": 287, "step_time": 11.216369532994577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.024553571827709675, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.024553571827709675, "completions/clipped_ratio": 0.0, "completions/max_length": 623.0, "completions/max_terminated_length": 623.0, "completions/mean_length": 61.3125, "completions/mean_terminated_length": 61.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4093446624465287, "epoch": 0.00576, "frac_reward_zero_std": 0.0, "grad_norm": 0.5453541278839111, "kl": 1.479773961007595, "learning_rate": 8.536005585891422e-06, "loss": 0.0316, "num_tokens": 13116032.0, "reward": -0.3056250214576721, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.7389837503433228, "sampling/importance_sampling_ratio/mean": 0.989113986492157, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.353276252746582, "sampling/sampling_logp_difference/mean": 0.09053964167833328, "step": 288, "step_time": 12.173902180984442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 78.21875, "completions/mean_terminated_length": 80.2258071899414, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.47671768302097917, "epoch": 0.00578, "frac_reward_zero_std": 0.25, "grad_norm": 0.40618616342544556, "kl": 2.02601027674973, "learning_rate": 8.525194149426354e-06, "loss": 0.042, "num_tokens": 13162497.0, "reward": -0.4950000047683716, "reward_std": 0.7564942836761475, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 2.1579630374908447, "sampling/importance_sampling_ratio/mean": 0.9983153343200684, "sampling/importance_sampling_ratio/min": 0.46153730154037476, "sampling/sampling_logp_difference/max": 0.9455811977386475, "sampling/sampling_logp_difference/mean": 0.1003868356347084, "step": 289, "step_time": 13.540566175004642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1724.0, "completions/max_terminated_length": 1724.0, "completions/mean_length": 78.46875, "completions/mean_terminated_length": 78.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4342928873375058, "epoch": 0.0058, "frac_reward_zero_std": 0.0, "grad_norm": 0.3057986795902252, "kl": 1.1843832023441792, "learning_rate": 8.514352701612418e-06, "loss": 0.0045, "num_tokens": 13208590.0, "reward": -0.24250000715255737, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 1.5962761640548706, "sampling/importance_sampling_ratio/mean": 0.9000616669654846, "sampling/importance_sampling_ratio/min": 0.28792041540145874, "sampling/sampling_logp_difference/max": 0.7994978427886963, "sampling/sampling_logp_difference/mean": 0.09948062896728516, "step": 290, "step_time": 15.856151450010657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 65.03125, "completions/mean_terminated_length": 65.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2808845331892371, "epoch": 0.00582, "frac_reward_zero_std": 0.0, "grad_norm": 0.4595259726047516, "kl": 0.7429244369268417, "learning_rate": 8.503481385455637e-06, "loss": 0.0732, "num_tokens": 13252965.0, "reward": 0.07249999046325684, "reward_std": 0.9984559416770935, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 2.086784601211548, "sampling/importance_sampling_ratio/mean": 1.043357014656067, "sampling/importance_sampling_ratio/min": 0.7620760798454285, "sampling/sampling_logp_difference/max": 0.877692699432373, "sampling/sampling_logp_difference/mean": 0.052820295095443726, "step": 291, "step_time": 11.96086203600862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 74.15625, "completions/mean_terminated_length": 74.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3901598888915032, "epoch": 0.00584, "frac_reward_zero_std": 0.0, "grad_norm": 0.3599090874195099, "kl": 1.157796585932374, "learning_rate": 8.492580344356023e-06, "loss": -0.0355, "num_tokens": 13299552.0, "reward": -0.30687499046325684, "reward_std": 0.9330272674560547, "rewards/rollout_reward_func/mean": -0.30687499046325684, "rewards/rollout_reward_func/std": 0.9730246663093567, "sampling/importance_sampling_ratio/max": 1.6673853397369385, "sampling/importance_sampling_ratio/mean": 1.0280876159667969, "sampling/importance_sampling_ratio/min": 0.49178528785705566, "sampling/sampling_logp_difference/max": 0.5408058166503906, "sampling/sampling_logp_difference/mean": 0.06323625147342682, "step": 292, "step_time": 12.079119491994788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 869.0, "completions/max_terminated_length": 869.0, "completions/mean_length": 123.3125, "completions/mean_terminated_length": 123.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3960358975455165, "epoch": 0.00586, "frac_reward_zero_std": 0.0, "grad_norm": 0.546221911907196, "kl": 1.1840096041560173, "learning_rate": 8.481649722105677e-06, "loss": -0.0926, "num_tokens": 13348753.0, "reward": -0.24312500655651093, "reward_std": 0.9424567818641663, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.6802222728729248, "sampling/importance_sampling_ratio/mean": 1.0354527235031128, "sampling/importance_sampling_ratio/min": 0.3646121025085449, "sampling/sampling_logp_difference/max": 0.7099819183349609, "sampling/sampling_logp_difference/mean": 0.07520860433578491, "step": 293, "step_time": 13.068040400990867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 646.0, "completions/max_terminated_length": 646.0, "completions/mean_length": 129.4375, "completions/mean_terminated_length": 129.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.38716966193169355, "epoch": 0.00588, "frac_reward_zero_std": 0.0, "grad_norm": 0.6961382627487183, "kl": 2.1677887327969074, "learning_rate": 8.470689662886895e-06, "loss": 0.0389, "num_tokens": 13397974.0, "reward": -0.3062500059604645, "reward_std": 1.016998529434204, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 1.4160826206207275, "sampling/importance_sampling_ratio/mean": 0.9494105577468872, "sampling/importance_sampling_ratio/min": 0.4492630362510681, "sampling/sampling_logp_difference/max": 0.8387579917907715, "sampling/sampling_logp_difference/mean": 0.08505851030349731, "step": 294, "step_time": 12.288006430004316 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 68.78125, "completions/mean_terminated_length": 68.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.30993928015232086, "epoch": 0.0059, "frac_reward_zero_std": 0.0, "grad_norm": 1.0048136711120605, "kl": 0.889720544219017, "learning_rate": 8.459700311270268e-06, "loss": 0.1431, "num_tokens": 13442572.0, "reward": 0.009374991059303284, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 2.6519124507904053, "sampling/importance_sampling_ratio/mean": 1.076092004776001, "sampling/importance_sampling_ratio/min": 0.449167400598526, "sampling/sampling_logp_difference/max": 0.5862261652946472, "sampling/sampling_logp_difference/mean": 0.05300695076584816, "step": 295, "step_time": 11.500245558992901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 76.375, "completions/mean_terminated_length": 76.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36250967951491475, "epoch": 0.00592, "frac_reward_zero_std": 0.0, "grad_norm": 0.5022215843200684, "kl": 0.9621922336518764, "learning_rate": 8.44868181221277e-06, "loss": -0.0122, "num_tokens": 13488546.0, "reward": -0.24250000715255737, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.8046271800994873, "sampling/importance_sampling_ratio/mean": 1.036481261253357, "sampling/importance_sampling_ratio/min": 0.48460185527801514, "sampling/sampling_logp_difference/max": 0.7002155780792236, "sampling/sampling_logp_difference/mean": 0.0713653415441513, "step": 296, "step_time": 11.750781191993156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 590.0, "completions/max_terminated_length": 590.0, "completions/mean_length": 45.3125, "completions/mean_terminated_length": 45.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4524043435230851, "epoch": 0.00594, "frac_reward_zero_std": 0.25, "grad_norm": 0.31932759284973145, "kl": 1.3975597191601992, "learning_rate": 8.437634311055855e-06, "loss": -0.0259, "num_tokens": 13534570.0, "reward": -0.43187499046325684, "reward_std": 0.7012691497802734, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.9783231019973755, "sampling/importance_sampling_ratio/mean": 0.9619506597518921, "sampling/importance_sampling_ratio/min": 0.41030311584472656, "sampling/sampling_logp_difference/max": 0.5259385108947754, "sampling/sampling_logp_difference/mean": 0.08243109285831451, "step": 297, "step_time": 11.56776772801095 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 81.8125, "completions/mean_terminated_length": 81.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3694874788634479, "epoch": 0.00596, "frac_reward_zero_std": 0.0, "grad_norm": 0.3312716782093048, "kl": 1.7146843392401934, "learning_rate": 8.42655795352353e-06, "loss": -0.0036, "num_tokens": 13579581.0, "reward": 0.07249997556209564, "reward_std": 0.9622916579246521, "rewards/rollout_reward_func/mean": 0.07249997556209564, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.639717698097229, "sampling/importance_sampling_ratio/mean": 0.9437705278396606, "sampling/importance_sampling_ratio/min": 0.43561360239982605, "sampling/sampling_logp_difference/max": 1.230438232421875, "sampling/sampling_logp_difference/mean": 0.09193848073482513, "step": 298, "step_time": 12.088776618002157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 383.0, "completions/max_terminated_length": 383.0, "completions/mean_length": 46.15625, "completions/mean_terminated_length": 46.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.35710514849051833, "epoch": 0.00598, "frac_reward_zero_std": 0.0, "grad_norm": 0.38370758295059204, "kl": 2.370790299028158, "learning_rate": 8.415452885720438e-06, "loss": -0.0246, "num_tokens": 13623857.0, "reward": -0.053125008940696716, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 2.553661346435547, "sampling/importance_sampling_ratio/mean": 0.9877766370773315, "sampling/importance_sampling_ratio/min": 0.21463853120803833, "sampling/sampling_logp_difference/max": 1.4896125793457031, "sampling/sampling_logp_difference/mean": 0.09937290847301483, "step": 299, "step_time": 11.014400295985979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 96.78125, "completions/mean_terminated_length": 96.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4135601769667119, "epoch": 0.006, "frac_reward_zero_std": 0.0, "grad_norm": 0.6443390250205994, "kl": 1.6024995520710945, "learning_rate": 8.40431925412993e-06, "loss": -0.0406, "num_tokens": 13670632.0, "reward": -0.3062500059604645, "reward_std": 0.9979773163795471, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 2.5307719707489014, "sampling/importance_sampling_ratio/mean": 1.078609824180603, "sampling/importance_sampling_ratio/min": 0.5208396315574646, "sampling/sampling_logp_difference/max": 1.2768340110778809, "sampling/sampling_logp_difference/mean": 0.08136668056249619, "step": 300, "step_time": 14.129331859003287 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 34.125, "completions/mean_terminated_length": 34.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.49589491728693247, "epoch": 0.00602, "frac_reward_zero_std": 0.0, "grad_norm": 0.3714132010936737, "kl": 1.100936571136117, "learning_rate": 8.393157205612137e-06, "loss": 0.0279, "num_tokens": 13716217.0, "reward": -0.36875003576278687, "reward_std": 0.9074459075927734, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767791748047, "sampling/importance_sampling_ratio/max": 1.9852209091186523, "sampling/importance_sampling_ratio/mean": 0.9642224907875061, "sampling/importance_sampling_ratio/min": 0.4841182231903076, "sampling/sampling_logp_difference/max": 0.7471112012863159, "sampling/sampling_logp_difference/mean": 0.09048432856798172, "step": 301, "step_time": 11.888648384985572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 122.0, "completions/mean_terminated_length": 122.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4287217278033495, "epoch": 0.00604, "frac_reward_zero_std": 0.0, "grad_norm": 0.3559570908546448, "kl": 1.085388908162713, "learning_rate": 8.38196688740202e-06, "loss": 0.0365, "num_tokens": 13764532.0, "reward": -0.11812500655651093, "reward_std": 0.9766770601272583, "rewards/rollout_reward_func/mean": -0.11812500655651093, "rewards/rollout_reward_func/std": 1.0159674882888794, "sampling/importance_sampling_ratio/max": 2.4644742012023926, "sampling/importance_sampling_ratio/mean": 1.0022393465042114, "sampling/importance_sampling_ratio/min": 0.4732849895954132, "sampling/sampling_logp_difference/max": 0.9193867444992065, "sampling/sampling_logp_difference/mean": 0.08248088508844376, "step": 302, "step_time": 13.531842463031353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 77.75, "completions/mean_terminated_length": 77.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3897771933116019, "epoch": 0.00606, "frac_reward_zero_std": 0.0, "grad_norm": 0.2658595144748688, "kl": 1.0599138606339693, "learning_rate": 8.370748447107445e-06, "loss": 0.0024, "num_tokens": 13811566.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.5739210844039917, "sampling/importance_sampling_ratio/mean": 0.9610825181007385, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1773428916931152, "sampling/sampling_logp_difference/mean": 0.08488079160451889, "step": 303, "step_time": 11.165675405005459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 111.1875, "completions/mean_terminated_length": 111.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.42494998779147863, "epoch": 0.00608, "frac_reward_zero_std": 0.0, "grad_norm": 0.30151358246803284, "kl": 1.5357411094009876, "learning_rate": 8.359502032707219e-06, "loss": 0.0239, "num_tokens": 13859967.0, "reward": -0.3056250214576721, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.510289192199707, "sampling/importance_sampling_ratio/mean": 0.9280977249145508, "sampling/importance_sampling_ratio/min": 0.2728807032108307, "sampling/sampling_logp_difference/max": 0.9023241996765137, "sampling/sampling_logp_difference/mean": 0.0943627804517746, "step": 304, "step_time": 11.994225078989984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 71.21875, "completions/mean_terminated_length": 71.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.38073471281677485, "epoch": 0.0061, "frac_reward_zero_std": 0.25, "grad_norm": 0.4707571268081665, "kl": 1.843195864930749, "learning_rate": 8.34822779254915e-06, "loss": 0.0685, "num_tokens": 13905441.0, "reward": -0.24250002205371857, "reward_std": 0.7926585674285889, "rewards/rollout_reward_func/mean": -0.24250002205371857, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 1.7962863445281982, "sampling/importance_sampling_ratio/mean": 1.0443248748779297, "sampling/importance_sampling_ratio/min": 0.5371094346046448, "sampling/sampling_logp_difference/max": 0.6431723833084106, "sampling/sampling_logp_difference/mean": 0.06555349379777908, "step": 305, "step_time": 13.884831222989305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 135.15625, "completions/mean_terminated_length": 135.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.33656962076202035, "epoch": 0.00612, "frac_reward_zero_std": 0.0, "grad_norm": 0.5652507543563843, "kl": 0.8127346429973841, "learning_rate": 8.336925875348093e-06, "loss": 0.0297, "num_tokens": 13952988.0, "reward": 0.07249999046325684, "reward_std": 1.0533576011657715, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.4178828001022339, "sampling/importance_sampling_ratio/mean": 0.9670929908752441, "sampling/importance_sampling_ratio/min": 0.49255865812301636, "sampling/sampling_logp_difference/max": 0.6462903022766113, "sampling/sampling_logp_difference/mean": 0.06580173969268799, "step": 306, "step_time": 14.607601336989319 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 78.46875, "completions/mean_terminated_length": 78.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3346112002618611, "epoch": 0.00614, "frac_reward_zero_std": 0.0, "grad_norm": 0.44079461693763733, "kl": 4.233878115192056, "learning_rate": 8.32559643018397e-06, "loss": 0.0745, "num_tokens": 13999113.0, "reward": -0.3075000047683716, "reward_std": 1.0152779817581177, "rewards/rollout_reward_func/mean": -0.3075000047683716, "rewards/rollout_reward_func/std": 0.9721508622169495, "sampling/importance_sampling_ratio/max": 2.1701905727386475, "sampling/importance_sampling_ratio/mean": 1.0190544128417969, "sampling/importance_sampling_ratio/min": 0.3403788208961487, "sampling/sampling_logp_difference/max": 1.1517682075500488, "sampling/sampling_logp_difference/mean": 0.08676497638225555, "step": 307, "step_time": 11.566633004993491 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 88.90625, "completions/mean_terminated_length": 88.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32038493221625686, "epoch": 0.00616, "frac_reward_zero_std": 0.0, "grad_norm": 0.4136032462120056, "kl": 1.4104246404021978, "learning_rate": 8.314239606499828e-06, "loss": 0.0391, "num_tokens": 14045758.0, "reward": -0.18000000715255737, "reward_std": 1.0533576011657715, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.4405304193496704, "sampling/importance_sampling_ratio/mean": 1.0532375574111938, "sampling/importance_sampling_ratio/min": 0.4284174144268036, "sampling/sampling_logp_difference/max": 0.8267576694488525, "sampling/sampling_logp_difference/mean": 0.06604554504156113, "step": 308, "step_time": 13.907249313007924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 67.03125, "completions/mean_terminated_length": 67.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31598875019699335, "epoch": 0.00618, "frac_reward_zero_std": 0.25, "grad_norm": 0.25037869811058044, "kl": 2.2072597853839397, "learning_rate": 8.302855554099842e-06, "loss": 0.0523, "num_tokens": 14091446.0, "reward": -0.36937499046325684, "reward_std": 0.7556362152099609, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.419077754020691, "sampling/importance_sampling_ratio/mean": 0.9780268669128418, "sampling/importance_sampling_ratio/min": 0.31588104367256165, "sampling/sampling_logp_difference/max": 1.009913444519043, "sampling/sampling_logp_difference/mean": 0.06839141994714737, "step": 309, "step_time": 14.7047868759837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 132.03125, "completions/mean_terminated_length": 132.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.478392930701375, "epoch": 0.0062, "frac_reward_zero_std": 0.0, "grad_norm": 0.7735646367073059, "kl": 1.821062596514821, "learning_rate": 8.29144442314736e-06, "loss": -0.1441, "num_tokens": 14140795.0, "reward": -0.5581250190734863, "reward_std": 0.8798531293869019, "rewards/rollout_reward_func/mean": -0.5581250190734863, "rewards/rollout_reward_func/std": 0.8484271168708801, "sampling/importance_sampling_ratio/max": 1.9790502786636353, "sampling/importance_sampling_ratio/mean": 0.9586645364761353, "sampling/importance_sampling_ratio/min": 0.40730178356170654, "sampling/sampling_logp_difference/max": 0.8510012030601501, "sampling/sampling_logp_difference/mean": 0.09479974210262299, "step": 310, "step_time": 14.799186345975613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 977.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 46.8125, "completions/mean_terminated_length": 46.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36779607366770506, "epoch": 0.00622, "frac_reward_zero_std": 0.0, "grad_norm": 0.21523083746433258, "kl": 1.4397620987147093, "learning_rate": 8.280006364162915e-06, "loss": -0.0262, "num_tokens": 14186601.0, "reward": -0.43312501907348633, "reward_std": 0.9135864973068237, "rewards/rollout_reward_func/mean": -0.43312501907348633, "rewards/rollout_reward_func/std": 0.9207234978675842, "sampling/importance_sampling_ratio/max": 1.390527367591858, "sampling/importance_sampling_ratio/mean": 0.9586175084114075, "sampling/importance_sampling_ratio/min": 0.33071887493133545, "sampling/sampling_logp_difference/max": 0.7730322480201721, "sampling/sampling_logp_difference/mean": 0.06798749417066574, "step": 311, "step_time": 13.377737199007242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 92.15625, "completions/mean_terminated_length": 92.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4288280475884676, "epoch": 0.00624, "frac_reward_zero_std": 0.25, "grad_norm": 0.5183484554290771, "kl": 2.412188747897744, "learning_rate": 8.268541528022238e-06, "loss": 0.0074, "num_tokens": 14233665.0, "reward": -0.18000002205371857, "reward_std": 0.7646863460540771, "rewards/rollout_reward_func/mean": -0.18000002205371857, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.4954015016555786, "sampling/importance_sampling_ratio/mean": 0.8719304800033569, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.550323486328125, "sampling/sampling_logp_difference/mean": 0.10414378345012665, "step": 312, "step_time": 12.005984367024212 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 339.0, "completions/max_terminated_length": 339.0, "completions/mean_length": 20.5, "completions/mean_terminated_length": 20.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.35848562326282263, "epoch": 0.00626, "frac_reward_zero_std": 0.0, "grad_norm": 0.3827018439769745, "kl": 1.4189422130584717, "learning_rate": 8.257050065954267e-06, "loss": 0.0941, "num_tokens": 14278755.0, "reward": -0.3062500059604645, "reward_std": 0.9981721043586731, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 1.7752277851104736, "sampling/importance_sampling_ratio/mean": 0.9910421371459961, "sampling/importance_sampling_ratio/min": 7.106356614272613e-10, "sampling/sampling_logp_difference/max": 20.528457641601562, "sampling/sampling_logp_difference/mean": 0.2527390718460083, "step": 313, "step_time": 11.047525219990348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 58.53125, "completions/mean_terminated_length": 58.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2679013190791011, "epoch": 0.00628, "frac_reward_zero_std": 0.0, "grad_norm": 0.5078871846199036, "kl": 1.7425630670040846, "learning_rate": 8.245532129539153e-06, "loss": 0.1107, "num_tokens": 14323485.0, "reward": -0.05437500774860382, "reward_std": 1.0339864492416382, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228264331817627, "sampling/importance_sampling_ratio/max": 1.3853355646133423, "sampling/importance_sampling_ratio/mean": 1.013358473777771, "sampling/importance_sampling_ratio/min": 0.3825870156288147, "sampling/sampling_logp_difference/max": 0.8673291206359863, "sampling/sampling_logp_difference/mean": 0.06052015721797943, "step": 314, "step_time": 11.865158115004306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 62.8125, "completions/mean_terminated_length": 62.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36093697138130665, "epoch": 0.0063, "frac_reward_zero_std": 0.0, "grad_norm": 0.3463307321071625, "kl": 1.6082743369042873, "learning_rate": 8.233987870706267e-06, "loss": 0.0599, "num_tokens": 14370024.0, "reward": -0.3056250214576721, "reward_std": 0.9521816968917847, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.439252495765686, "sampling/importance_sampling_ratio/mean": 1.0145134925842285, "sampling/importance_sampling_ratio/min": 0.5506206154823303, "sampling/sampling_logp_difference/max": 0.44033098220825195, "sampling/sampling_logp_difference/mean": 0.06013251841068268, "step": 315, "step_time": 11.561832765000872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1031.0, "completions/max_terminated_length": 1031.0, "completions/mean_length": 99.03125, "completions/mean_terminated_length": 99.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3401013342663646, "epoch": 0.00632, "frac_reward_zero_std": 0.0, "grad_norm": 0.5474528670310974, "kl": 1.1815725173801184, "learning_rate": 8.222417441732182e-06, "loss": 0.0537, "num_tokens": 14416840.0, "reward": -0.3687500059604645, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.5978741645812988, "sampling/importance_sampling_ratio/mean": 1.0650148391723633, "sampling/importance_sampling_ratio/min": 0.7245352864265442, "sampling/sampling_logp_difference/max": 0.6273505687713623, "sampling/sampling_logp_difference/mean": 0.052770815789699554, "step": 316, "step_time": 14.014811822009506 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.03125, "completions/max_length": 429.0, "completions/max_terminated_length": 429.0, "completions/mean_length": 25.65625, "completions/mean_terminated_length": 25.967741012573242, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3897483889013529, "epoch": 0.00634, "frac_reward_zero_std": 0.0, "grad_norm": 0.729887843132019, "kl": 2.817006044089794, "learning_rate": 8.210820995238682e-06, "loss": 0.0756, "num_tokens": 14461743.0, "reward": -0.17937500774860382, "reward_std": 0.9883459806442261, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.821393370628357, "sampling/importance_sampling_ratio/mean": 1.0028188228607178, "sampling/importance_sampling_ratio/min": 0.1618192046880722, "sampling/sampling_logp_difference/max": 1.1213245391845703, "sampling/sampling_logp_difference/mean": 0.07663894444704056, "step": 317, "step_time": 10.89409840600274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 37.28125, "completions/mean_terminated_length": 37.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32735067955218256, "epoch": 0.00636, "frac_reward_zero_std": 0.0, "grad_norm": 0.6836167573928833, "kl": 1.1858269032090902, "learning_rate": 8.199198684190737e-06, "loss": 0.1099, "num_tokens": 14506007.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.8676016330718994, "sampling/importance_sampling_ratio/mean": 1.0002789497375488, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 19.04401397705078, "sampling/sampling_logp_difference/mean": 0.21755573153495789, "step": 318, "step_time": 11.65308016299241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 180.0625, "completions/mean_terminated_length": 180.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.34294644510373473, "epoch": 0.00638, "frac_reward_zero_std": 0.0, "grad_norm": 0.6360172033309937, "kl": 1.4033957552164793, "learning_rate": 8.187550661894482e-06, "loss": -0.0703, "num_tokens": 14555538.0, "reward": 0.07312498986721039, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.5741926431655884, "sampling/importance_sampling_ratio/mean": 1.0333683490753174, "sampling/importance_sampling_ratio/min": 0.43814390897750854, "sampling/sampling_logp_difference/max": 0.7342720031738281, "sampling/sampling_logp_difference/mean": 0.05710350722074509, "step": 319, "step_time": 14.79878076400928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 628.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 29.3125, "completions/mean_terminated_length": 29.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.40356071572750807, "epoch": 0.0064, "frac_reward_zero_std": 0.25, "grad_norm": 0.3905542194843292, "kl": 0.9524715896695852, "learning_rate": 8.175877081995205e-06, "loss": 0.0851, "num_tokens": 14600433.0, "reward": -0.4325000047683716, "reward_std": 0.7644026279449463, "rewards/rollout_reward_func/mean": -0.4325000047683716, "rewards/rollout_reward_func/std": 0.9217339158058167, "sampling/importance_sampling_ratio/max": 1.4511032104492188, "sampling/importance_sampling_ratio/mean": 0.9761359691619873, "sampling/importance_sampling_ratio/min": 0.40956565737724304, "sampling/sampling_logp_difference/max": 0.7231721878051758, "sampling/sampling_logp_difference/mean": 0.07285532355308533, "step": 320, "step_time": 11.542198194030789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 62.5625, "completions/mean_terminated_length": 62.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.30726118572056293, "epoch": 0.00642, "frac_reward_zero_std": 0.0, "grad_norm": 0.4183712303638458, "kl": 1.614510802552104, "learning_rate": 8.16417809847532e-06, "loss": 0.032, "num_tokens": 14644950.0, "reward": 0.009999990463256836, "reward_std": 1.0797353982925415, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.5138885974884033, "sampling/importance_sampling_ratio/mean": 0.9960873126983643, "sampling/importance_sampling_ratio/min": 0.5375962853431702, "sampling/sampling_logp_difference/max": 0.6153182983398438, "sampling/sampling_logp_difference/mean": 0.06613914668560028, "step": 321, "step_time": 12.134353092995298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 637.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 86.25, "completions/mean_terminated_length": 86.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3081046477891505, "epoch": 0.00644, "frac_reward_zero_std": 0.0, "grad_norm": 0.5263993740081787, "kl": 1.2657500840723515, "learning_rate": 8.152453865652327e-06, "loss": -0.0027, "num_tokens": 14690058.0, "reward": -0.05375000834465027, "reward_std": 1.01747727394104, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.634607195854187, "sampling/importance_sampling_ratio/mean": 1.006521463394165, "sampling/importance_sampling_ratio/min": 0.24909710884094238, "sampling/sampling_logp_difference/max": 1.2868438959121704, "sampling/sampling_logp_difference/mean": 0.06932520121335983, "step": 322, "step_time": 12.29204839898739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 527.0, "completions/max_terminated_length": 527.0, "completions/mean_length": 99.3125, "completions/mean_terminated_length": 99.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3443588246591389, "epoch": 0.00646, "frac_reward_zero_std": 0.0, "grad_norm": 0.385916143655777, "kl": 1.1987852938473225, "learning_rate": 8.140704538176782e-06, "loss": 0.0426, "num_tokens": 14736332.0, "reward": 0.13624998927116394, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.799280047416687, "sampling/importance_sampling_ratio/mean": 1.0013372898101807, "sampling/importance_sampling_ratio/min": 0.46255093812942505, "sampling/sampling_logp_difference/max": 0.6911153793334961, "sampling/sampling_logp_difference/mean": 0.06663398444652557, "step": 323, "step_time": 11.193801797999186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 593.0, "completions/max_terminated_length": 593.0, "completions/mean_length": 95.34375, "completions/mean_terminated_length": 95.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3865455808117986, "epoch": 0.00648, "frac_reward_zero_std": 0.0, "grad_norm": 0.35169100761413574, "kl": 1.2213424034416676, "learning_rate": 8.128930271030256e-06, "loss": 0.0098, "num_tokens": 14782417.0, "reward": 0.07249999046325684, "reward_std": 1.0705007314682007, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.5573762655258179, "sampling/importance_sampling_ratio/mean": 0.9544377326965332, "sampling/importance_sampling_ratio/min": 0.39041516184806824, "sampling/sampling_logp_difference/max": 0.8604931831359863, "sampling/sampling_logp_difference/mean": 0.06583061814308167, "step": 324, "step_time": 11.979838525003288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 96.3125, "completions/mean_terminated_length": 96.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32105002854950726, "epoch": 0.0065, "frac_reward_zero_std": 0.0, "grad_norm": 0.549353837966919, "kl": 0.7014435082674026, "learning_rate": 8.117131219523293e-06, "loss": 0.0273, "num_tokens": 14829042.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.4412217140197754, "sampling/importance_sampling_ratio/mean": 1.033544659614563, "sampling/importance_sampling_ratio/min": 0.7420470118522644, "sampling/sampling_logp_difference/max": 0.32257914543151855, "sampling/sampling_logp_difference/mean": 0.040619440376758575, "step": 325, "step_time": 12.080492327993852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 30.6875, "completions/mean_terminated_length": 30.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4176249401643872, "epoch": 0.00652, "frac_reward_zero_std": 0.0, "grad_norm": 0.31926292181015015, "kl": 0.6175532285124063, "learning_rate": 8.10530753929336e-06, "loss": 0.0169, "num_tokens": 14873272.0, "reward": -0.17937500774860382, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.6503145694732666, "sampling/importance_sampling_ratio/mean": 0.9784739017486572, "sampling/importance_sampling_ratio/min": 0.4620957374572754, "sampling/sampling_logp_difference/max": 0.5684201717376709, "sampling/sampling_logp_difference/mean": 0.06892566382884979, "step": 326, "step_time": 13.294813231979788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 110.40625, "completions/mean_terminated_length": 113.45160675048828, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.45923478342592716, "epoch": 0.00654, "frac_reward_zero_std": 0.25, "grad_norm": 0.39982208609580994, "kl": 0.6576167438179255, "learning_rate": 8.093459386302788e-06, "loss": -0.056, "num_tokens": 14921705.0, "reward": -0.11750000715255737, "reward_std": 0.7728759050369263, "rewards/rollout_reward_func/mean": -0.11750000715255737, "rewards/rollout_reward_func/std": 1.0166834592819214, "sampling/importance_sampling_ratio/max": 1.6454050540924072, "sampling/importance_sampling_ratio/mean": 1.0376091003417969, "sampling/importance_sampling_ratio/min": 0.8763947486877441, "sampling/sampling_logp_difference/max": 0.4652705192565918, "sampling/sampling_logp_difference/mean": 0.03207266330718994, "step": 327, "step_time": 12.073301473013998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 34.46875, "completions/mean_terminated_length": 34.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3832801324315369, "epoch": 0.00656, "frac_reward_zero_std": 0.25, "grad_norm": 0.273401141166687, "kl": 1.113759795203805, "learning_rate": 8.081586916836728e-06, "loss": -0.0036, "num_tokens": 14966706.0, "reward": -0.3056250214576721, "reward_std": 0.7289015054702759, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.2966265678405762, "sampling/importance_sampling_ratio/mean": 0.9780166149139404, "sampling/importance_sampling_ratio/min": 0.5502302646636963, "sampling/sampling_logp_difference/max": 0.5759601593017578, "sampling/sampling_logp_difference/mean": 0.04674416035413742, "step": 328, "step_time": 11.53736291801033 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 50.1875, "completions/mean_terminated_length": 50.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29011610336601734, "epoch": 0.00658, "frac_reward_zero_std": 0.0, "grad_norm": 0.3032393455505371, "kl": 0.6323705278337002, "learning_rate": 8.069690287501078e-06, "loss": -0.017, "num_tokens": 15011202.0, "reward": 0.07249999046325684, "reward_std": 1.0173444747924805, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.2460423707962036, "sampling/importance_sampling_ratio/mean": 0.9788497686386108, "sampling/importance_sampling_ratio/min": 0.5733948945999146, "sampling/sampling_logp_difference/max": 0.5821561813354492, "sampling/sampling_logp_difference/mean": 0.03768813610076904, "step": 329, "step_time": 13.263609989000543 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 69.8125, "completions/mean_terminated_length": 69.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2892871778458357, "epoch": 0.0066, "frac_reward_zero_std": 0.0, "grad_norm": 0.40937069058418274, "kl": 1.6720833126455545, "learning_rate": 8.05776965522042e-06, "loss": 0.0099, "num_tokens": 15057095.0, "reward": -0.24250000715255737, "reward_std": 0.9626314640045166, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 1.4130674600601196, "sampling/importance_sampling_ratio/mean": 0.9857897758483887, "sampling/importance_sampling_ratio/min": 0.4234904646873474, "sampling/sampling_logp_difference/max": 1.0923347473144531, "sampling/sampling_logp_difference/mean": 0.07172787934541702, "step": 330, "step_time": 12.120899564986757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 81.09375, "completions/mean_terminated_length": 81.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3134558480232954, "epoch": 0.00662, "frac_reward_zero_std": 0.0, "grad_norm": 0.3374234437942505, "kl": 0.5853395331650972, "learning_rate": 8.045825177235952e-06, "loss": 0.0256, "num_tokens": 15103241.0, "reward": -0.11625000834465027, "reward_std": 0.9712425470352173, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.6162471771240234, "sampling/importance_sampling_ratio/mean": 1.057753562927246, "sampling/importance_sampling_ratio/min": 0.7943946123123169, "sampling/sampling_logp_difference/max": 0.4962620735168457, "sampling/sampling_logp_difference/mean": 0.04106259346008301, "step": 331, "step_time": 11.592228671004705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 56.65625, "completions/mean_terminated_length": 56.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3073793121147901, "epoch": 0.00664, "frac_reward_zero_std": 0.0, "grad_norm": 0.39768368005752563, "kl": 0.6181282456964254, "learning_rate": 8.033857011103411e-06, "loss": -0.0018, "num_tokens": 15147131.0, "reward": 0.008749991655349731, "reward_std": 1.0608855485916138, "rewards/rollout_reward_func/mean": 0.008749991655349731, "rewards/rollout_reward_func/std": 1.0249146223068237, "sampling/importance_sampling_ratio/max": 1.666068434715271, "sampling/importance_sampling_ratio/mean": 1.021540641784668, "sampling/importance_sampling_ratio/min": 0.7022718191146851, "sampling/sampling_logp_difference/max": 0.5143094062805176, "sampling/sampling_logp_difference/mean": 0.03904948756098747, "step": 332, "step_time": 11.985685423009272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 84.75, "completions/mean_terminated_length": 84.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28857528883963823, "epoch": 0.00666, "frac_reward_zero_std": 0.25, "grad_norm": 0.24314478039741516, "kl": 0.5640790872275829, "learning_rate": 8.021865314691006e-06, "loss": -0.0014, "num_tokens": 15193539.0, "reward": -0.24250000715255737, "reward_std": 0.7564942836761475, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 1.237607479095459, "sampling/importance_sampling_ratio/mean": 1.0262601375579834, "sampling/importance_sampling_ratio/min": 0.6383822560310364, "sampling/sampling_logp_difference/max": 0.45877861976623535, "sampling/sampling_logp_difference/mean": 0.03380618989467621, "step": 333, "step_time": 14.207204285987245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 83.625, "completions/mean_terminated_length": 83.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2832246934995055, "epoch": 0.00668, "frac_reward_zero_std": 0.0, "grad_norm": 0.3252204358577728, "kl": 0.5553797762840986, "learning_rate": 8.009850246177311e-06, "loss": -0.0109, "num_tokens": 15240015.0, "reward": -0.18000000715255737, "reward_std": 1.0341415405273438, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.0809348821640015, "sampling/importance_sampling_ratio/mean": 0.9787272214889526, "sampling/importance_sampling_ratio/min": 0.5189725756645203, "sampling/sampling_logp_difference/max": 0.46503257751464844, "sampling/sampling_logp_difference/mean": 0.03815162926912308, "step": 334, "step_time": 11.638517660016078 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 111.625, "completions/mean_terminated_length": 111.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25465743825770915, "epoch": 0.0067, "frac_reward_zero_std": 0.0, "grad_norm": 0.4326549768447876, "kl": 2.248137411661446, "learning_rate": 7.997811964049207e-06, "loss": -0.009, "num_tokens": 15285210.0, "reward": 0.1356249898672104, "reward_std": 1.0259160995483398, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175585746765137, "sampling/importance_sampling_ratio/max": 1.4533874988555908, "sampling/importance_sampling_ratio/mean": 0.9848540425300598, "sampling/importance_sampling_ratio/min": 0.48156100511550903, "sampling/sampling_logp_difference/max": 0.7302265167236328, "sampling/sampling_logp_difference/mean": 0.04748786240816116, "step": 335, "step_time": 11.82576831697952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0026041667442768812, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026041667442768812, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 62.65625, "completions/mean_terminated_length": 62.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36155885830521584, "epoch": 0.00672, "frac_reward_zero_std": 0.0, "grad_norm": 0.3141798675060272, "kl": 0.8517661187797785, "learning_rate": 7.98575062709977e-06, "loss": -0.0052, "num_tokens": 15331828.0, "reward": -0.4950000047683716, "reward_std": 0.860792338848114, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 1.857344150543213, "sampling/importance_sampling_ratio/mean": 0.9936472177505493, "sampling/importance_sampling_ratio/min": 0.4865509569644928, "sampling/sampling_logp_difference/max": 0.7202234268188477, "sampling/sampling_logp_difference/mean": 0.051786042749881744, "step": 336, "step_time": 12.055905910034198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 66.84375, "completions/mean_terminated_length": 68.48387145996094, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27952844463288784, "epoch": 0.00674, "frac_reward_zero_std": 0.0, "grad_norm": 0.3634644150733948, "kl": 1.0092331115156412, "learning_rate": 7.97366639442619e-06, "loss": -0.007, "num_tokens": 15377848.0, "reward": 0.008749991655349731, "reward_std": 1.0615792274475098, "rewards/rollout_reward_func/mean": 0.008749991655349731, "rewards/rollout_reward_func/std": 1.0249146223068237, "sampling/importance_sampling_ratio/max": 1.0618085861206055, "sampling/importance_sampling_ratio/mean": 0.947988748550415, "sampling/importance_sampling_ratio/min": 0.5284647941589355, "sampling/sampling_logp_difference/max": 0.5928144454956055, "sampling/sampling_logp_difference/mean": 0.04816784709692001, "step": 337, "step_time": 11.672149914993497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 44.5, "completions/mean_terminated_length": 44.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.35377247747965157, "epoch": 0.00676, "frac_reward_zero_std": 0.0, "grad_norm": 0.4130244255065918, "kl": 1.203724717721343, "learning_rate": 7.961559425427666e-06, "loss": -0.0001, "num_tokens": 15422160.0, "reward": 0.009374991059303284, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.1858381032943726, "sampling/importance_sampling_ratio/mean": 0.9777672290802002, "sampling/importance_sampling_ratio/min": 0.36857402324676514, "sampling/sampling_logp_difference/max": 0.9425263404846191, "sampling/sampling_logp_difference/mean": 0.07325349003076553, "step": 338, "step_time": 11.36364450801193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 53.1875, "completions/mean_terminated_length": 53.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27085520047694445, "epoch": 0.00678, "frac_reward_zero_std": 0.0, "grad_norm": 0.3755432069301605, "kl": 2.7165376096963882, "learning_rate": 7.949429879803298e-06, "loss": -0.0171, "num_tokens": 15467611.0, "reward": -0.18062500655651093, "reward_std": 1.0168434381484985, "rewards/rollout_reward_func/mean": -0.18062500655651093, "rewards/rollout_reward_func/std": 1.0064499378204346, "sampling/importance_sampling_ratio/max": 1.661360502243042, "sampling/importance_sampling_ratio/mean": 1.0272555351257324, "sampling/importance_sampling_ratio/min": 0.47606515884399414, "sampling/sampling_logp_difference/max": 0.7419805526733398, "sampling/sampling_logp_difference/mean": 0.03993844985961914, "step": 339, "step_time": 12.344319995012484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 874.0, "completions/max_terminated_length": 874.0, "completions/mean_length": 95.875, "completions/mean_terminated_length": 95.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2082426925189793, "epoch": 0.0068, "frac_reward_zero_std": 0.0, "grad_norm": 0.8482993245124817, "kl": 0.7074008025228977, "learning_rate": 7.937277917549998e-06, "loss": 0.0475, "num_tokens": 15513198.0, "reward": 0.13624998927116394, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.1379027366638184, "sampling/importance_sampling_ratio/mean": 1.0932804346084595, "sampling/importance_sampling_ratio/min": 0.7123074531555176, "sampling/sampling_logp_difference/max": 0.6472995281219482, "sampling/sampling_logp_difference/mean": 0.03540646657347679, "step": 340, "step_time": 13.058048783997947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 87.375, "completions/mean_terminated_length": 87.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27233211463317275, "epoch": 0.00682, "frac_reward_zero_std": 0.0, "grad_norm": 0.30496877431869507, "kl": 1.4458485133945942, "learning_rate": 7.925103698960361e-06, "loss": -0.0281, "num_tokens": 15560035.0, "reward": 0.009999990463256836, "reward_std": 1.0074068307876587, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.052704095840454, "sampling/importance_sampling_ratio/mean": 0.9144979119300842, "sampling/importance_sampling_ratio/min": 0.3158247172832489, "sampling/sampling_logp_difference/max": 1.235713005065918, "sampling/sampling_logp_difference/mean": 0.06989537179470062, "step": 341, "step_time": 11.411023553002451 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 131.25, "completions/mean_terminated_length": 131.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2021989985369146, "epoch": 0.00684, "frac_reward_zero_std": 0.0, "grad_norm": 0.3813616633415222, "kl": 1.352479599416256, "learning_rate": 7.91290738462056e-06, "loss": -0.0034, "num_tokens": 15607887.0, "reward": -0.11687500774860382, "reward_std": 1.0445914268493652, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.5981453657150269, "sampling/importance_sampling_ratio/mean": 0.9655764102935791, "sampling/importance_sampling_ratio/min": 0.3332819938659668, "sampling/sampling_logp_difference/max": 1.0901317596435547, "sampling/sampling_logp_difference/mean": 0.052520185708999634, "step": 342, "step_time": 13.812131300997862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 78.21875, "completions/mean_terminated_length": 78.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2573883533477783, "epoch": 0.00686, "frac_reward_zero_std": 0.25, "grad_norm": 0.48817238211631775, "kl": 0.6754311081022024, "learning_rate": 7.900689135408226e-06, "loss": -0.0034, "num_tokens": 15653954.0, "reward": -0.11625000834465027, "reward_std": 0.7564942836761475, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.866119623184204, "sampling/importance_sampling_ratio/mean": 0.9896302223205566, "sampling/importance_sampling_ratio/min": 0.4881737232208252, "sampling/sampling_logp_difference/max": 0.7213287353515625, "sampling/sampling_logp_difference/mean": 0.04906249791383743, "step": 343, "step_time": 13.38269592699362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 62.53125, "completions/mean_terminated_length": 62.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19838439184240997, "epoch": 0.00688, "frac_reward_zero_std": 0.0, "grad_norm": 0.2271137237548828, "kl": 0.6459877770394087, "learning_rate": 7.888449112490325e-06, "loss": 0.0019, "num_tokens": 15698516.0, "reward": 0.07312498986721039, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.4496477842330933, "sampling/importance_sampling_ratio/mean": 1.0250990390777588, "sampling/importance_sampling_ratio/min": 0.810938835144043, "sampling/sampling_logp_difference/max": 0.3764446973800659, "sampling/sampling_logp_difference/mean": 0.01765855774283409, "step": 344, "step_time": 11.3476022429968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 90.90625, "completions/mean_terminated_length": 90.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26630913466215134, "epoch": 0.0069, "frac_reward_zero_std": 0.0, "grad_norm": 0.31296291947364807, "kl": 0.5702593736350536, "learning_rate": 7.876187477321033e-06, "loss": -0.0201, "num_tokens": 15745260.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.2039995193481445, "sampling/importance_sampling_ratio/mean": 0.9875510931015015, "sampling/importance_sampling_ratio/min": 0.5571715831756592, "sampling/sampling_logp_difference/max": 0.5975306034088135, "sampling/sampling_logp_difference/mean": 0.031617164611816406, "step": 345, "step_time": 12.037222529004794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 47.78125, "completions/mean_terminated_length": 47.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23370578582398593, "epoch": 0.00692, "frac_reward_zero_std": 0.0, "grad_norm": 0.3342841863632202, "kl": 1.8700887002050877, "learning_rate": 7.86390439163961e-06, "loss": -0.0116, "num_tokens": 15789150.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.4844456911087036, "sampling/importance_sampling_ratio/mean": 1.0056626796722412, "sampling/importance_sampling_ratio/min": 0.33122336864471436, "sampling/sampling_logp_difference/max": 1.104386329650879, "sampling/sampling_logp_difference/mean": 0.04360474646091461, "step": 346, "step_time": 11.509140807007498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 82.90625, "completions/mean_terminated_length": 82.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18807253101840615, "epoch": 0.00694, "frac_reward_zero_std": 0.0, "grad_norm": 0.21268072724342346, "kl": 1.0578780192881823, "learning_rate": 7.851600017468257e-06, "loss": -0.003, "num_tokens": 15834980.0, "reward": 0.009374991059303284, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.075132966041565, "sampling/importance_sampling_ratio/mean": 0.9792506694793701, "sampling/importance_sampling_ratio/min": 0.4252690374851227, "sampling/sampling_logp_difference/max": 0.8536710739135742, "sampling/sampling_logp_difference/mean": 0.030391912907361984, "step": 347, "step_time": 11.566596249009308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 59.15625, "completions/mean_terminated_length": 59.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2265779951121658, "epoch": 0.00696, "frac_reward_zero_std": 0.0, "grad_norm": 0.20966576039791107, "kl": 1.0068186670541763, "learning_rate": 7.839274517109987e-06, "loss": -0.0173, "num_tokens": 15880812.0, "reward": -0.17937500774860382, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.0979009866714478, "sampling/importance_sampling_ratio/mean": 0.9928737878799438, "sampling/importance_sampling_ratio/min": 0.614767849445343, "sampling/sampling_logp_difference/max": 0.4796323776245117, "sampling/sampling_logp_difference/mean": 0.026975281536579132, "step": 348, "step_time": 11.642432460008422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 105.375, "completions/mean_terminated_length": 105.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2678698217496276, "epoch": 0.00698, "frac_reward_zero_std": 0.0, "grad_norm": 0.38044288754463196, "kl": 0.5009054029360414, "learning_rate": 7.826928053146482e-06, "loss": -0.0166, "num_tokens": 15927608.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.1503022909164429, "sampling/importance_sampling_ratio/mean": 0.9781397581100464, "sampling/importance_sampling_ratio/min": 0.4344131648540497, "sampling/sampling_logp_difference/max": 0.8511489629745483, "sampling/sampling_logp_difference/mean": 0.03492416441440582, "step": 349, "step_time": 12.207655185986368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 75.5, "completions/mean_terminated_length": 75.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18499936745502055, "epoch": 0.007, "frac_reward_zero_std": 0.0, "grad_norm": 0.5705176591873169, "kl": 0.6611536610871553, "learning_rate": 7.814560788435947e-06, "loss": -0.0485, "num_tokens": 15973689.0, "reward": -0.17937500774860382, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.366039752960205, "sampling/importance_sampling_ratio/mean": 0.9900723695755005, "sampling/importance_sampling_ratio/min": 0.41881850361824036, "sampling/sampling_logp_difference/max": 0.8568465709686279, "sampling/sampling_logp_difference/mean": 0.03686421364545822, "step": 350, "step_time": 12.083308423993003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 115.65625, "completions/mean_terminated_length": 115.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2038463680073619, "epoch": 0.00702, "frac_reward_zero_std": 0.0, "grad_norm": 0.5293048024177551, "kl": 0.670192513614893, "learning_rate": 7.802172886110968e-06, "loss": 0.0113, "num_tokens": 16021774.0, "reward": -0.11687500774860382, "reward_std": 1.042907953262329, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.3249030113220215, "sampling/importance_sampling_ratio/mean": 1.0326836109161377, "sampling/importance_sampling_ratio/min": 0.8963872194290161, "sampling/sampling_logp_difference/max": 0.48433828353881836, "sampling/sampling_logp_difference/mean": 0.027394339442253113, "step": 351, "step_time": 14.188922255998477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 64.40625, "completions/mean_terminated_length": 64.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18138200882822275, "epoch": 0.00704, "frac_reward_zero_std": 0.0, "grad_norm": 0.13621556758880615, "kl": 0.614533307030797, "learning_rate": 7.789764509576347e-06, "loss": -0.0133, "num_tokens": 16067152.0, "reward": -0.053125008940696716, "reward_std": 0.9521816968917847, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0620243549346924, "sampling/importance_sampling_ratio/mean": 0.9830700159072876, "sampling/importance_sampling_ratio/min": 0.4245852828025818, "sampling/sampling_logp_difference/max": 0.8548736572265625, "sampling/sampling_logp_difference/mean": 0.03089721128344536, "step": 352, "step_time": 11.481551811993995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 456.0, "completions/max_terminated_length": 456.0, "completions/mean_length": 33.3125, "completions/mean_terminated_length": 33.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20070523652248085, "epoch": 0.00706, "frac_reward_zero_std": 0.0, "grad_norm": 0.18504968285560608, "kl": 0.5907430090010166, "learning_rate": 7.77733582250696e-06, "loss": -0.0128, "num_tokens": 16110661.0, "reward": 0.009374991059303284, "reward_std": 1.0445914268493652, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.025668978691101, "sampling/importance_sampling_ratio/mean": 0.988985538482666, "sampling/importance_sampling_ratio/min": 0.6854473352432251, "sampling/sampling_logp_difference/max": 0.3753313422203064, "sampling/sampling_logp_difference/mean": 0.01768086664378643, "step": 353, "step_time": 11.87609670299571 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 134.21875, "completions/mean_terminated_length": 134.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13096372154541314, "epoch": 0.00708, "frac_reward_zero_std": 0.0, "grad_norm": 1.0821772813796997, "kl": 0.848408767953515, "learning_rate": 7.764886988845588e-06, "loss": 0.0328, "num_tokens": 16157554.0, "reward": 0.1993749886751175, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.2616263628005981, "sampling/importance_sampling_ratio/mean": 1.008792519569397, "sampling/importance_sampling_ratio/min": 0.46973592042922974, "sampling/sampling_logp_difference/max": 0.7761375904083252, "sampling/sampling_logp_difference/mean": 0.02622298151254654, "step": 354, "step_time": 11.703960024999105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 52.4375, "completions/mean_terminated_length": 52.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11357178189791739, "epoch": 0.0071, "frac_reward_zero_std": 0.0, "grad_norm": 0.46186748147010803, "kl": 0.658944284543395, "learning_rate": 7.752418172800769e-06, "loss": 0.0037, "num_tokens": 16200950.0, "reward": 0.2606250047683716, "reward_std": 1.0439131259918213, "rewards/rollout_reward_func/mean": 0.2606250047683716, "rewards/rollout_reward_func/std": 0.9921171069145203, "sampling/importance_sampling_ratio/max": 1.1709816455841064, "sampling/importance_sampling_ratio/mean": 1.0130367279052734, "sampling/importance_sampling_ratio/min": 0.7625492811203003, "sampling/sampling_logp_difference/max": 0.5142450332641602, "sampling/sampling_logp_difference/mean": 0.024173058569431305, "step": 355, "step_time": 11.28368110399606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 69.84375, "completions/mean_terminated_length": 69.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12112702918238938, "epoch": 0.00712, "frac_reward_zero_std": 0.25, "grad_norm": 0.31616273522377014, "kl": 1.8439569044858217, "learning_rate": 7.73992953884461e-06, "loss": 0.0021, "num_tokens": 16246876.0, "reward": -0.05375000834465027, "reward_std": 0.7646863460540771, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.6263753175735474, "sampling/importance_sampling_ratio/mean": 1.0265095233917236, "sampling/importance_sampling_ratio/min": 0.479398250579834, "sampling/sampling_logp_difference/max": 0.7351641654968262, "sampling/sampling_logp_difference/mean": 0.024886466562747955, "step": 356, "step_time": 12.193995025991171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.0, "completions/max_terminated_length": 339.0, "completions/mean_length": 12.53125, "completions/mean_terminated_length": 12.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14148568757809699, "epoch": 0.00714, "frac_reward_zero_std": 0.0, "grad_norm": 0.09852378070354462, "kl": 2.0956640783697367, "learning_rate": 7.727421251710637e-06, "loss": -0.0061, "num_tokens": 16290824.0, "reward": -0.3687500059604645, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.1321438550949097, "sampling/importance_sampling_ratio/mean": 0.9807384014129639, "sampling/importance_sampling_ratio/min": 0.5634896755218506, "sampling/sampling_logp_difference/max": 0.5616235733032227, "sampling/sampling_logp_difference/mean": 0.03503769636154175, "step": 357, "step_time": 10.747293036984047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1288.0, "completions/max_terminated_length": 1288.0, "completions/mean_length": 139.96875, "completions/mean_terminated_length": 139.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2011099117808044, "epoch": 0.00716, "frac_reward_zero_std": 0.0, "grad_norm": 0.60221928358078, "kl": 0.7146632624790072, "learning_rate": 7.714893476391613e-06, "loss": 0.0205, "num_tokens": 16338768.0, "reward": -0.11687500774860382, "reward_std": 1.0259159803390503, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.7109856605529785, "sampling/importance_sampling_ratio/mean": 1.0078500509262085, "sampling/importance_sampling_ratio/min": 0.5926328897476196, "sampling/sampling_logp_difference/max": 0.5465831756591797, "sampling/sampling_logp_difference/mean": 0.04346437007188797, "step": 358, "step_time": 14.417547320001177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 97.9375, "completions/mean_terminated_length": 97.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20228623668663204, "epoch": 0.00718, "frac_reward_zero_std": 0.0, "grad_norm": 0.17788751423358917, "kl": 2.1322043407708406, "learning_rate": 7.702346378137364e-06, "loss": -0.003, "num_tokens": 16386245.0, "reward": -0.3056250214576721, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.1164672374725342, "sampling/importance_sampling_ratio/mean": 1.0016456842422485, "sampling/importance_sampling_ratio/min": 0.5526162385940552, "sampling/sampling_logp_difference/max": 0.5931320190429688, "sampling/sampling_logp_difference/mean": 0.022496867924928665, "step": 359, "step_time": 12.26927337199595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 829.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 80.9375, "completions/mean_terminated_length": 80.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18839053716510534, "epoch": 0.0072, "frac_reward_zero_std": 0.0, "grad_norm": 0.5075739622116089, "kl": 3.5211931876838207, "learning_rate": 7.689780122452598e-06, "loss": -0.0176, "num_tokens": 16433498.0, "reward": -0.11750000715255737, "reward_std": 0.9887310266494751, "rewards/rollout_reward_func/mean": -0.11750000715255737, "rewards/rollout_reward_func/std": 1.0166834592819214, "sampling/importance_sampling_ratio/max": 1.14876389503479, "sampling/importance_sampling_ratio/mean": 0.9677733182907104, "sampling/importance_sampling_ratio/min": 0.42139557003974915, "sampling/sampling_logp_difference/max": 0.8582940101623535, "sampling/sampling_logp_difference/mean": 0.040431179106235504, "step": 360, "step_time": 13.297503115005384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 456.0, "completions/max_terminated_length": 456.0, "completions/mean_length": 65.0625, "completions/mean_terminated_length": 65.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13168125739321113, "epoch": 0.00722, "frac_reward_zero_std": 0.0, "grad_norm": 0.11316512525081635, "kl": 0.6807734929025173, "learning_rate": 7.67719487509472e-06, "loss": -0.0216, "num_tokens": 16479164.0, "reward": -0.05375000834465027, "reward_std": 1.0705007314682007, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.029054045677185, "sampling/importance_sampling_ratio/mean": 0.9912177324295044, "sampling/importance_sampling_ratio/min": 0.4656117260456085, "sampling/sampling_logp_difference/max": 0.7607598304748535, "sampling/sampling_logp_difference/mean": 0.01962273009121418, "step": 361, "step_time": 10.906776259012986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 73.78125, "completions/mean_terminated_length": 73.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09700034908019006, "epoch": 0.00724, "frac_reward_zero_std": 0.0, "grad_norm": 0.2941007614135742, "kl": 2.666662694886327, "learning_rate": 7.664590802071653e-06, "loss": 0.0004, "num_tokens": 16524970.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.0862760543823242, "sampling/importance_sampling_ratio/mean": 0.9571384191513062, "sampling/importance_sampling_ratio/min": 0.37461820244789124, "sampling/sampling_logp_difference/max": 0.9819064140319824, "sampling/sampling_logp_difference/mean": 0.04264073446393013, "step": 362, "step_time": 14.114839115994982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 64.21875, "completions/mean_terminated_length": 64.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13691839342936873, "epoch": 0.00726, "frac_reward_zero_std": 0.0, "grad_norm": 0.23241649568080902, "kl": 1.3552883248776197, "learning_rate": 7.651968069639637e-06, "loss": -0.0161, "num_tokens": 16569626.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.3516026735305786, "sampling/importance_sampling_ratio/mean": 0.9732315540313721, "sampling/importance_sampling_ratio/min": 0.33320891857147217, "sampling/sampling_logp_difference/max": 1.097060203552246, "sampling/sampling_logp_difference/mean": 0.05260109156370163, "step": 363, "step_time": 12.149256703996798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 112.8125, "completions/mean_terminated_length": 112.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1497728095855564, "epoch": 0.00728, "frac_reward_zero_std": 0.0, "grad_norm": 0.3424515128135681, "kl": 1.1250272272154689, "learning_rate": 7.63932684430105e-06, "loss": -0.018, "num_tokens": 16617369.0, "reward": -0.11625000834465027, "reward_std": 0.9626314640045166, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0946087837219238, "sampling/importance_sampling_ratio/mean": 0.9740407466888428, "sampling/importance_sampling_ratio/min": 0.48277249932289124, "sampling/sampling_logp_difference/max": 0.8292651176452637, "sampling/sampling_logp_difference/mean": 0.03793571889400482, "step": 364, "step_time": 11.53248701398843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 92.8125, "completions/mean_terminated_length": 92.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19030342809855938, "epoch": 0.0073, "frac_reward_zero_std": 0.0, "grad_norm": 0.1896112710237503, "kl": 0.5323412027209997, "learning_rate": 7.626667292802197e-06, "loss": -0.0062, "num_tokens": 16664542.0, "reward": -0.43187499046325684, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.11329185962677, "sampling/importance_sampling_ratio/mean": 0.9832653999328613, "sampling/importance_sampling_ratio/min": 0.47209790349006653, "sampling/sampling_logp_difference/max": 0.724668025970459, "sampling/sampling_logp_difference/mean": 0.03544434905052185, "step": 365, "step_time": 13.96045639600925 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 77.96875, "completions/mean_terminated_length": 77.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15887367352843285, "epoch": 0.00732, "frac_reward_zero_std": 0.25, "grad_norm": 0.8989170789718628, "kl": 0.6628592666238546, "learning_rate": 7.613989582131121e-06, "loss": -0.0279, "num_tokens": 16710196.0, "reward": 0.1993749886751175, "reward_std": 0.7289015054702759, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.7094242572784424, "sampling/importance_sampling_ratio/mean": 1.0729212760925293, "sampling/importance_sampling_ratio/min": 0.5552083253860474, "sampling/sampling_logp_difference/max": 0.834498405456543, "sampling/sampling_logp_difference/mean": 0.034817732870578766, "step": 366, "step_time": 11.382770316980896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 60.0, "completions/mean_terminated_length": 60.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14071759802754968, "epoch": 0.00734, "frac_reward_zero_std": 0.0, "grad_norm": 0.12801305949687958, "kl": 0.5121611393988132, "learning_rate": 7.6012938795153966e-06, "loss": -0.0053, "num_tokens": 16754766.0, "reward": 0.009999990463256836, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.109226107597351, "sampling/importance_sampling_ratio/mean": 1.0140695571899414, "sampling/importance_sampling_ratio/min": 0.8765963912010193, "sampling/sampling_logp_difference/max": 0.1315382719039917, "sampling/sampling_logp_difference/mean": 0.012128060683608055, "step": 367, "step_time": 11.815297144989017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 63.25, "completions/mean_terminated_length": 63.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12740323110483587, "epoch": 0.00736, "frac_reward_zero_std": 0.0, "grad_norm": 0.5238834023475647, "kl": 0.44990747794508934, "learning_rate": 7.588580352419923e-06, "loss": 0.0005, "num_tokens": 16799273.0, "reward": -0.05375000834465027, "reward_std": 0.9984559416770935, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.0856473445892334, "sampling/importance_sampling_ratio/mean": 0.9768741130828857, "sampling/importance_sampling_ratio/min": 0.5484157800674438, "sampling/sampling_logp_difference/max": 0.6036520004272461, "sampling/sampling_logp_difference/mean": 0.03031044825911522, "step": 368, "step_time": 11.331026505984482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 49.875, "completions/mean_terminated_length": 49.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1502513912273571, "epoch": 0.00738, "frac_reward_zero_std": 0.0, "grad_norm": 0.4350929856300354, "kl": 0.47855960205197334, "learning_rate": 7.575849168544717e-06, "loss": -0.0089, "num_tokens": 16843343.0, "reward": 0.1356249749660492, "reward_std": 0.8795696496963501, "rewards/rollout_reward_func/mean": 0.1356249749660492, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.0666073560714722, "sampling/importance_sampling_ratio/mean": 0.969131350517273, "sampling/importance_sampling_ratio/min": 0.551278293132782, "sampling/sampling_logp_difference/max": 0.6034870147705078, "sampling/sampling_logp_difference/mean": 0.0313495509326458, "step": 369, "step_time": 11.078343975997996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 76.4375, "completions/mean_terminated_length": 76.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13159062480553985, "epoch": 0.0074, "frac_reward_zero_std": 0.0, "grad_norm": 0.28881996870040894, "kl": 0.6002139076590538, "learning_rate": 7.5631004958227e-06, "loss": -0.0118, "num_tokens": 16888454.0, "reward": -0.053125008940696716, "reward_std": 1.0711638927459717, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.05125892162323, "sampling/importance_sampling_ratio/mean": 0.9773174524307251, "sampling/importance_sampling_ratio/min": 0.37732550501823425, "sampling/sampling_logp_difference/max": 0.975313663482666, "sampling/sampling_logp_difference/mean": 0.02845405414700508, "step": 370, "step_time": 12.012923602989758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 48.21875, "completions/mean_terminated_length": 48.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1619224390015006, "epoch": 0.00742, "frac_reward_zero_std": 0.0, "grad_norm": 0.48328620195388794, "kl": 3.9237516429275274, "learning_rate": 7.550334502417483e-06, "loss": 0.0025, "num_tokens": 16933445.0, "reward": -0.24312500655651093, "reward_std": 1.0067436695098877, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.3208322525024414, "sampling/importance_sampling_ratio/mean": 0.9969007968902588, "sampling/importance_sampling_ratio/min": 0.6230865716934204, "sampling/sampling_logp_difference/max": 0.4735574722290039, "sampling/sampling_logp_difference/mean": 0.030309408903121948, "step": 371, "step_time": 11.406517323004664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 123.21875, "completions/mean_terminated_length": 123.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11315087729599327, "epoch": 0.00744, "frac_reward_zero_std": 0.0, "grad_norm": 0.20826882123947144, "kl": 1.1136598661541939, "learning_rate": 7.537551356721149e-06, "loss": -0.0281, "num_tokens": 16981531.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0958589315414429, "sampling/importance_sampling_ratio/mean": 0.9913667440414429, "sampling/importance_sampling_ratio/min": 0.48723024129867554, "sampling/sampling_logp_difference/max": 0.7188234329223633, "sampling/sampling_logp_difference/mean": 0.02494649961590767, "step": 372, "step_time": 15.831017618009355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 84.3125, "completions/mean_terminated_length": 84.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14051671465858817, "epoch": 0.00746, "frac_reward_zero_std": 0.0, "grad_norm": 0.3668946921825409, "kl": 0.8670478202402592, "learning_rate": 7.5247512273520325e-06, "loss": -0.0047, "num_tokens": 17027338.0, "reward": -0.053125008940696716, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.4408231973648071, "sampling/importance_sampling_ratio/mean": 0.9889616966247559, "sampling/importance_sampling_ratio/min": 0.4372291564941406, "sampling/sampling_logp_difference/max": 0.8264322280883789, "sampling/sampling_logp_difference/mean": 0.04350835829973221, "step": 373, "step_time": 12.006038801002433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 36.5625, "completions/mean_terminated_length": 36.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1784485006937757, "epoch": 0.00748, "frac_reward_zero_std": 0.0, "grad_norm": 0.4423363208770752, "kl": 0.5732648149132729, "learning_rate": 7.511934283152491e-06, "loss": 0.0079, "num_tokens": 17071859.0, "reward": -0.36937499046325684, "reward_std": 0.9424567818641663, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.2388933897018433, "sampling/importance_sampling_ratio/mean": 1.0015908479690552, "sampling/importance_sampling_ratio/min": 0.5497745871543884, "sampling/sampling_logp_difference/max": 0.6053500175476074, "sampling/sampling_logp_difference/mean": 0.03606022894382477, "step": 374, "step_time": 11.457998854995822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 42.875, "completions/mean_terminated_length": 42.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07614747388288379, "epoch": 0.0075, "frac_reward_zero_std": 0.0, "grad_norm": 0.04098938778042793, "kl": 0.5463599115610123, "learning_rate": 7.499100693186684e-06, "loss": -0.0101, "num_tokens": 17113651.0, "reward": 0.1356249898672104, "reward_std": 1.0449373722076416, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.10415780544281, "sampling/importance_sampling_ratio/mean": 1.0173349380493164, "sampling/importance_sampling_ratio/min": 1.002369999885559, "sampling/sampling_logp_difference/max": 0.0769212543964386, "sampling/sampling_logp_difference/mean": 0.007915952242910862, "step": 375, "step_time": 11.181965071002196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 438.0, "completions/max_terminated_length": 438.0, "completions/mean_length": 51.6875, "completions/mean_terminated_length": 51.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15984988666605204, "epoch": 0.00752, "frac_reward_zero_std": 0.25, "grad_norm": 0.21899093687534332, "kl": 2.792703690007329, "learning_rate": 7.486250626738338e-06, "loss": -0.0185, "num_tokens": 17159486.0, "reward": -0.17937500774860382, "reward_std": 0.8012300729751587, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.4586271047592163, "sampling/importance_sampling_ratio/mean": 0.9882651567459106, "sampling/importance_sampling_ratio/min": 0.4197404980659485, "sampling/sampling_logp_difference/max": 0.8681073188781738, "sampling/sampling_logp_difference/mean": 0.0388411246240139, "step": 376, "step_time": 11.097472884008312 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 113.53125, "completions/mean_terminated_length": 113.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1510623594513163, "epoch": 0.00754, "frac_reward_zero_std": 0.0, "grad_norm": 0.46424391865730286, "kl": 0.7190752048045397, "learning_rate": 7.473384253308518e-06, "loss": -0.0037, "num_tokens": 17206945.0, "reward": -0.24250000715255737, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.1079068183898926, "sampling/importance_sampling_ratio/mean": 0.9814531803131104, "sampling/importance_sampling_ratio/min": 0.6181142330169678, "sampling/sampling_logp_difference/max": 0.480743408203125, "sampling/sampling_logp_difference/mean": 0.030977187678217888, "step": 377, "step_time": 14.48070516998996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 46.09375, "completions/mean_terminated_length": 46.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15117655764333904, "epoch": 0.00756, "frac_reward_zero_std": 0.0, "grad_norm": 0.5816274881362915, "kl": 0.6330097485333681, "learning_rate": 7.460501742613385e-06, "loss": -0.01, "num_tokens": 17252473.0, "reward": -0.24312500655651093, "reward_std": 0.9791113138198853, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.7766711711883545, "sampling/importance_sampling_ratio/mean": 1.0151845216751099, "sampling/importance_sampling_ratio/min": 0.5911933183670044, "sampling/sampling_logp_difference/max": 0.5506694316864014, "sampling/sampling_logp_difference/mean": 0.028834937140345573, "step": 378, "step_time": 11.38694153700635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 112.40625, "completions/mean_terminated_length": 112.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14114498905837536, "epoch": 0.00758, "frac_reward_zero_std": 0.0, "grad_norm": 0.24863041937351227, "kl": 0.6273983512073755, "learning_rate": 7.447603264581964e-06, "loss": -0.0181, "num_tokens": 17300202.0, "reward": -0.11625000834465027, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1175363063812256, "sampling/importance_sampling_ratio/mean": 1.001422643661499, "sampling/importance_sampling_ratio/min": 0.5727571845054626, "sampling/sampling_logp_difference/max": 0.5575151443481445, "sampling/sampling_logp_difference/mean": 0.023864857852458954, "step": 379, "step_time": 12.038950859998295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 75.875, "completions/mean_terminated_length": 75.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11000674520619214, "epoch": 0.0076, "frac_reward_zero_std": 0.0, "grad_norm": 0.1132843941450119, "kl": 1.6675729556009173, "learning_rate": 7.4346889893539e-06, "loss": 0.0006, "num_tokens": 17346731.0, "reward": -0.05437500774860382, "reward_std": 0.9330272674560547, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228137969970703, "sampling/importance_sampling_ratio/max": 1.0967415571212769, "sampling/importance_sampling_ratio/mean": 0.9978927969932556, "sampling/importance_sampling_ratio/min": 0.6207150220870972, "sampling/sampling_logp_difference/max": 0.603696346282959, "sampling/sampling_logp_difference/mean": 0.026873599737882614, "step": 380, "step_time": 11.904920530003437 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 62.8125, "completions/mean_terminated_length": 62.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14379537687636912, "epoch": 0.00762, "frac_reward_zero_std": 0.0, "grad_norm": 0.3515150547027588, "kl": 1.778793167322874, "learning_rate": 7.421759087277214e-06, "loss": -0.0279, "num_tokens": 17392991.0, "reward": -0.24312500655651093, "reward_std": 0.962119460105896, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.380409598350525, "sampling/importance_sampling_ratio/mean": 0.9643000364303589, "sampling/importance_sampling_ratio/min": 0.4887370765209198, "sampling/sampling_logp_difference/max": 0.7182493209838867, "sampling/sampling_logp_difference/mean": 0.05642513185739517, "step": 381, "step_time": 11.390267164999386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 25.03125, "completions/mean_terminated_length": 25.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16831254167482257, "epoch": 0.00764, "frac_reward_zero_std": 0.0, "grad_norm": 0.1170961782336235, "kl": 0.7352738305926323, "learning_rate": 7.408813728906053e-06, "loss": -0.0044, "num_tokens": 17438547.0, "reward": -0.5581250190734863, "reward_std": 0.8522208333015442, "rewards/rollout_reward_func/mean": -0.5581250190734863, "rewards/rollout_reward_func/std": 0.8484271168708801, "sampling/importance_sampling_ratio/max": 1.2922098636627197, "sampling/importance_sampling_ratio/mean": 1.009163498878479, "sampling/importance_sampling_ratio/min": 0.6193957328796387, "sampling/sampling_logp_difference/max": 0.47922515869140625, "sampling/sampling_logp_difference/mean": 0.026946205645799637, "step": 382, "step_time": 11.131742479003151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 402.0, "completions/max_terminated_length": 402.0, "completions/mean_length": 40.75, "completions/mean_terminated_length": 40.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12712252791970968, "epoch": 0.00766, "frac_reward_zero_std": 0.0, "grad_norm": 0.2737891674041748, "kl": 0.6485857740044594, "learning_rate": 7.395853084998448e-06, "loss": -0.0133, "num_tokens": 17483578.0, "reward": -0.17937500774860382, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.4573767185211182, "sampling/importance_sampling_ratio/mean": 1.041117548942566, "sampling/importance_sampling_ratio/min": 0.9938390254974365, "sampling/sampling_logp_difference/max": 0.3772709369659424, "sampling/sampling_logp_difference/mean": 0.01706570014357567, "step": 383, "step_time": 10.98519008600124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 78.6875, "completions/mean_terminated_length": 78.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1439447896555066, "epoch": 0.00768, "frac_reward_zero_std": 0.0, "grad_norm": 0.5699148774147034, "kl": 8.080621860921383, "learning_rate": 7.382877326514051e-06, "loss": -0.0142, "num_tokens": 17530252.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.3621498346328735, "sampling/importance_sampling_ratio/mean": 0.904076099395752, "sampling/importance_sampling_ratio/min": 0.3398604691028595, "sampling/sampling_logp_difference/max": 1.0789966583251953, "sampling/sampling_logp_difference/mean": 0.08022215962409973, "step": 384, "step_time": 11.325827208005649 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 775.0, "completions/max_terminated_length": 775.0, "completions/mean_length": 137.9375, "completions/mean_terminated_length": 137.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15075470716692507, "epoch": 0.0077, "frac_reward_zero_std": 0.0, "grad_norm": 0.4982296824455261, "kl": 0.7901153080165386, "learning_rate": 7.369886624611889e-06, "loss": -0.0346, "num_tokens": 17577987.0, "reward": 0.3881249725818634, "reward_std": 0.943366289138794, "rewards/rollout_reward_func/mean": 0.3881249725818634, "rewards/rollout_reward_func/std": 0.9508550763130188, "sampling/importance_sampling_ratio/max": 1.1117240190505981, "sampling/importance_sampling_ratio/mean": 0.9511640071868896, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.041046142578125, "sampling/sampling_logp_difference/mean": 0.04297375679016113, "step": 385, "step_time": 12.395866668986855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 67.4375, "completions/mean_terminated_length": 67.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14087544288486242, "epoch": 0.00772, "frac_reward_zero_std": 0.0, "grad_norm": 0.2266261875629425, "kl": 0.569450018927455, "learning_rate": 7.356881150648103e-06, "loss": 0.0087, "num_tokens": 17622240.0, "reward": 0.07312498986721039, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.6011508703231812, "sampling/importance_sampling_ratio/mean": 1.055627703666687, "sampling/importance_sampling_ratio/min": 0.7815988063812256, "sampling/sampling_logp_difference/max": 0.47075116634368896, "sampling/sampling_logp_difference/mean": 0.027536632493138313, "step": 386, "step_time": 11.312806205001834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 775.0, "completions/max_terminated_length": 775.0, "completions/mean_length": 79.65625, "completions/mean_terminated_length": 79.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1517039251048118, "epoch": 0.00774, "frac_reward_zero_std": 0.0, "grad_norm": 0.2473093867301941, "kl": 0.661198103800416, "learning_rate": 7.343861076173684e-06, "loss": -0.0136, "num_tokens": 17668006.0, "reward": 0.009374991059303284, "reward_std": 0.9894057512283325, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.2044334411621094, "sampling/importance_sampling_ratio/mean": 0.94927978515625, "sampling/importance_sampling_ratio/min": 0.053635455667972565, "sampling/sampling_logp_difference/max": 1.5816948413848877, "sampling/sampling_logp_difference/mean": 0.0672542154788971, "step": 387, "step_time": 12.282752975996118 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 98.21875, "completions/mean_terminated_length": 98.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20142854400910437, "epoch": 0.00776, "frac_reward_zero_std": 0.0, "grad_norm": 0.5062217116355896, "kl": 1.0820163115859032, "learning_rate": 7.330826572932217e-06, "loss": -0.0104, "num_tokens": 17716307.0, "reward": -0.3062500059604645, "reward_std": 0.9976819157600403, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 1.461521029472351, "sampling/importance_sampling_ratio/mean": 1.0219762325286865, "sampling/importance_sampling_ratio/min": 0.46137359738349915, "sampling/sampling_logp_difference/max": 0.8086135387420654, "sampling/sampling_logp_difference/mean": 0.0409238375723362, "step": 388, "step_time": 13.34314384899335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 164.625, "completions/mean_terminated_length": 164.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1737025110051036, "epoch": 0.00778, "frac_reward_zero_std": 0.5, "grad_norm": 0.3479599058628082, "kl": 1.9880050839856267, "learning_rate": 7.3177778128576124e-06, "loss": -0.0054, "num_tokens": 17766688.0, "reward": -0.053125008940696716, "reward_std": 0.5312961935997009, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.226657509803772, "sampling/importance_sampling_ratio/mean": 0.9492448568344116, "sampling/importance_sampling_ratio/min": 0.390048086643219, "sampling/sampling_logp_difference/max": 0.9415493011474609, "sampling/sampling_logp_difference/mean": 0.06144339591264725, "step": 389, "step_time": 13.960985111974878 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 75.59375, "completions/mean_terminated_length": 75.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18073402252048254, "epoch": 0.0078, "frac_reward_zero_std": 0.0, "grad_norm": 0.5052434802055359, "kl": 0.6894596442580223, "learning_rate": 7.304714968071834e-06, "loss": -0.0036, "num_tokens": 17811806.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 2.098767042160034, "sampling/importance_sampling_ratio/mean": 0.9933032989501953, "sampling/importance_sampling_ratio/min": 0.4348852336406708, "sampling/sampling_logp_difference/max": 0.8317022323608398, "sampling/sampling_logp_difference/mean": 0.04849034547805786, "step": 390, "step_time": 11.8794101400199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 80.0625, "completions/mean_terminated_length": 80.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1317917217966169, "epoch": 0.00782, "frac_reward_zero_std": 0.0, "grad_norm": 0.25168508291244507, "kl": 0.6531613208353519, "learning_rate": 7.291638210882638e-06, "loss": -0.0245, "num_tokens": 17857998.0, "reward": 0.1356249898672104, "reward_std": 0.9898844361305237, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175585746765137, "sampling/importance_sampling_ratio/max": 1.1674102544784546, "sampling/importance_sampling_ratio/mean": 1.0166802406311035, "sampling/importance_sampling_ratio/min": 0.7670629024505615, "sampling/sampling_logp_difference/max": 0.2650848627090454, "sampling/sampling_logp_difference/mean": 0.012319575063884258, "step": 391, "step_time": 10.985489230006351 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 55.78125, "completions/mean_terminated_length": 55.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16718475800007582, "epoch": 0.00784, "frac_reward_zero_std": 0.0, "grad_norm": 0.2390887290239334, "kl": 0.5150203760713339, "learning_rate": 7.278547713781288e-06, "loss": -0.0198, "num_tokens": 17903211.0, "reward": 0.009374991059303284, "reward_std": 1.0067436695098877, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.5057364702224731, "sampling/importance_sampling_ratio/mean": 1.034144401550293, "sampling/importance_sampling_ratio/min": 0.7554664611816406, "sampling/sampling_logp_difference/max": 0.40923434495925903, "sampling/sampling_logp_difference/mean": 0.02410086803138256, "step": 392, "step_time": 12.95835357199394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11837237200234085, "epoch": 0.00786, "frac_reward_zero_std": 0.0, "grad_norm": 0.08511613309383392, "kl": 0.6731577888131142, "learning_rate": 7.2654436494402955e-06, "loss": -0.0011, "num_tokens": 17945614.0, "reward": -0.05375000834465027, "reward_std": 1.0531628131866455, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.0610257387161255, "sampling/importance_sampling_ratio/mean": 0.9873326420783997, "sampling/importance_sampling_ratio/min": 0.5116664171218872, "sampling/sampling_logp_difference/max": 0.6701741218566895, "sampling/sampling_logp_difference/mean": 0.027832353487610817, "step": 393, "step_time": 9.978345846029697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 166.375, "completions/mean_terminated_length": 166.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10816910129506141, "epoch": 0.00788, "frac_reward_zero_std": 0.0, "grad_norm": 0.3705340623855591, "kl": 0.917524759657681, "learning_rate": 7.252326190711121e-06, "loss": 0.0231, "num_tokens": 17993187.0, "reward": 0.19874997437000275, "reward_std": 0.970690906047821, "rewards/rollout_reward_func/mean": 0.19874997437000275, "rewards/rollout_reward_func/std": 1.0074424743652344, "sampling/importance_sampling_ratio/max": 1.1859320402145386, "sampling/importance_sampling_ratio/mean": 1.0158342123031616, "sampling/importance_sampling_ratio/min": 0.6258675456047058, "sampling/sampling_logp_difference/max": 0.46701955795288086, "sampling/sampling_logp_difference/mean": 0.018096989020705223, "step": 394, "step_time": 14.652161731995875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 82.0625, "completions/mean_terminated_length": 82.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13168243505060673, "epoch": 0.0079, "frac_reward_zero_std": 0.0, "grad_norm": 0.22971570491790771, "kl": 0.7200445141643286, "learning_rate": 7.239195510621918e-06, "loss": -0.0205, "num_tokens": 18039608.0, "reward": -0.11625000834465027, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.158278465270996, "sampling/importance_sampling_ratio/mean": 0.986869215965271, "sampling/importance_sampling_ratio/min": 0.5480753183364868, "sampling/sampling_logp_difference/max": 0.5994420051574707, "sampling/sampling_logp_difference/mean": 0.03764059394598007, "step": 395, "step_time": 13.40071891000116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 67.4375, "completions/mean_terminated_length": 67.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14260491728782654, "epoch": 0.00792, "frac_reward_zero_std": 0.0, "grad_norm": 0.12140686064958572, "kl": 0.9297293405979872, "learning_rate": 7.22605178237523e-06, "loss": -0.0143, "num_tokens": 18085345.0, "reward": 0.009374991059303284, "reward_std": 0.9617735147476196, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.2041562795639038, "sampling/importance_sampling_ratio/mean": 1.0089964866638184, "sampling/importance_sampling_ratio/min": 0.62677401304245, "sampling/sampling_logp_difference/max": 0.4674220085144043, "sampling/sampling_logp_difference/mean": 0.024986183270812035, "step": 396, "step_time": 11.402832544001285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 29.8125, "completions/mean_terminated_length": 29.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16416102880612016, "epoch": 0.00794, "frac_reward_zero_std": 0.0, "grad_norm": 0.3337618410587311, "kl": 0.6620549019426107, "learning_rate": 7.212895179345718e-06, "loss": -0.0035, "num_tokens": 18129662.0, "reward": -0.17937500774860382, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.4542378187179565, "sampling/importance_sampling_ratio/mean": 1.0492902994155884, "sampling/importance_sampling_ratio/min": 1.0028750896453857, "sampling/sampling_logp_difference/max": 0.3752262592315674, "sampling/sampling_logp_difference/mean": 0.021023903042078018, "step": 397, "step_time": 11.064280228005373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 59.03125, "completions/mean_terminated_length": 59.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17147492989897728, "epoch": 0.00796, "frac_reward_zero_std": 0.0, "grad_norm": 0.10388927161693573, "kl": 1.1531185619533062, "learning_rate": 7.199725875077871e-06, "loss": -0.014, "num_tokens": 18176126.0, "reward": -0.4950000047683716, "reward_std": 0.9074459075927734, "rewards/rollout_reward_func/mean": -0.4950000047683716, "rewards/rollout_reward_func/std": 0.8886814713478088, "sampling/importance_sampling_ratio/max": 1.109613060951233, "sampling/importance_sampling_ratio/mean": 0.99515300989151, "sampling/importance_sampling_ratio/min": 0.6673932075500488, "sampling/sampling_logp_difference/max": 0.4043113589286804, "sampling/sampling_logp_difference/mean": 0.025967484340071678, "step": 398, "step_time": 10.986869344989827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 40.1875, "completions/mean_terminated_length": 40.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17327646946068853, "epoch": 0.00798, "frac_reward_zero_std": 0.0, "grad_norm": 0.4429763853549957, "kl": 0.630920123308897, "learning_rate": 7.186544043283715e-06, "loss": -0.0038, "num_tokens": 18220576.0, "reward": -0.11625000834465027, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 2.0495316982269287, "sampling/importance_sampling_ratio/mean": 1.0580828189849854, "sampling/importance_sampling_ratio/min": 0.9184349179267883, "sampling/sampling_logp_difference/max": 0.7177603244781494, "sampling/sampling_logp_difference/mean": 0.024272125214338303, "step": 399, "step_time": 11.353317202010658 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 86.0, "completions/mean_terminated_length": 86.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1671084112022072, "epoch": 0.008, "frac_reward_zero_std": 0.0, "grad_norm": 0.3896557688713074, "kl": 0.5563310272991657, "learning_rate": 7.173349857840521e-06, "loss": 0.004, "num_tokens": 18267059.0, "reward": -0.05437500774860382, "reward_std": 1.0161405801773071, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228137969970703, "sampling/importance_sampling_ratio/max": 1.9316089153289795, "sampling/importance_sampling_ratio/mean": 1.0379343032836914, "sampling/importance_sampling_ratio/min": 0.8071289658546448, "sampling/sampling_logp_difference/max": 0.6069653034210205, "sampling/sampling_logp_difference/mean": 0.021488768979907036, "step": 400, "step_time": 13.77267992200359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1270.0, "completions/max_terminated_length": 1270.0, "completions/mean_length": 111.5625, "completions/mean_terminated_length": 111.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16749073704704642, "epoch": 0.00802, "frac_reward_zero_std": 0.0, "grad_norm": 0.3595040440559387, "kl": 1.150387343019247, "learning_rate": 7.160143492788516e-06, "loss": 0.0005, "num_tokens": 18314670.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.6644335985183716, "sampling/importance_sampling_ratio/mean": 1.0399742126464844, "sampling/importance_sampling_ratio/min": 0.6194435358047485, "sampling/sampling_logp_difference/max": 0.47887516021728516, "sampling/sampling_logp_difference/mean": 0.03384895622730255, "step": 401, "step_time": 14.229656402028922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 47.375, "completions/mean_terminated_length": 47.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15653328830376267, "epoch": 0.00804, "frac_reward_zero_std": 0.0, "grad_norm": 0.2566053867340088, "kl": 0.9271817971020937, "learning_rate": 7.146925122328581e-06, "loss": -0.0016, "num_tokens": 18358828.0, "reward": -0.05375000834465027, "reward_std": 0.9984559416770935, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.0761475563049316, "sampling/importance_sampling_ratio/mean": 0.9526396989822388, "sampling/importance_sampling_ratio/min": 0.31082451343536377, "sampling/sampling_logp_difference/max": 0.6912360191345215, "sampling/sampling_logp_difference/mean": 0.045890044420957565, "step": 402, "step_time": 11.770744552013639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 89.21875, "completions/mean_terminated_length": 89.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.180656002368778, "epoch": 0.00806, "frac_reward_zero_std": 0.0, "grad_norm": 0.15665218234062195, "kl": 0.6737284269183874, "learning_rate": 7.133694920819958e-06, "loss": -0.0193, "num_tokens": 18405942.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.270247459411621, "sampling/importance_sampling_ratio/mean": 1.0052928924560547, "sampling/importance_sampling_ratio/min": 0.4870105981826782, "sampling/sampling_logp_difference/max": 0.7193679809570312, "sampling/sampling_logp_difference/mean": 0.026502033695578575, "step": 403, "step_time": 12.15112046100694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 60.15625, "completions/mean_terminated_length": 60.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16588166181463748, "epoch": 0.00808, "frac_reward_zero_std": 0.0, "grad_norm": 0.31271013617515564, "kl": 2.580436185002327, "learning_rate": 7.12045306277795e-06, "loss": -0.0028, "num_tokens": 18451820.0, "reward": -0.24250000715255737, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.3172662258148193, "sampling/importance_sampling_ratio/mean": 0.9648857116699219, "sampling/importance_sampling_ratio/min": 0.5560170412063599, "sampling/sampling_logp_difference/max": 0.5915241241455078, "sampling/sampling_logp_difference/mean": 0.04671734198927879, "step": 404, "step_time": 11.693916767013434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 98.78125, "completions/mean_terminated_length": 98.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1392828058451414, "epoch": 0.0081, "frac_reward_zero_std": 0.0, "grad_norm": 0.061181552708148956, "kl": 0.9152575358748436, "learning_rate": 7.107199722871614e-06, "loss": -0.009, "num_tokens": 18498104.0, "reward": 0.009374991059303284, "reward_std": 1.0062534809112549, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.138332724571228, "sampling/importance_sampling_ratio/mean": 0.9971146583557129, "sampling/importance_sampling_ratio/min": 0.4655516743659973, "sampling/sampling_logp_difference/max": 0.7642288208007812, "sampling/sampling_logp_difference/mean": 0.03017335571348667, "step": 405, "step_time": 13.403160816989839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 140.09375, "completions/mean_terminated_length": 140.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.185568479122594, "epoch": 0.00812, "frac_reward_zero_std": 0.0, "grad_norm": 0.6567308902740479, "kl": 0.8563900645822287, "learning_rate": 7.093935075921465e-06, "loss": -0.0378, "num_tokens": 18546815.0, "reward": -0.18000000715255737, "reward_std": 1.0173444747924805, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.740363359451294, "sampling/importance_sampling_ratio/mean": 1.044477105140686, "sampling/importance_sampling_ratio/min": 0.6268602013587952, "sampling/sampling_logp_difference/max": 0.5145140886306763, "sampling/sampling_logp_difference/mean": 0.03030800260603428, "step": 406, "step_time": 13.914615660010895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 66.03125, "completions/mean_terminated_length": 66.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20938065648078918, "epoch": 0.00814, "frac_reward_zero_std": 0.0, "grad_norm": 0.6342635154724121, "kl": 1.0183865278959274, "learning_rate": 7.0806592968971624e-06, "loss": 0.0005, "num_tokens": 18592697.0, "reward": -0.18125000596046448, "reward_std": 0.9696723222732544, "rewards/rollout_reward_func/mean": -0.18125000596046448, "rewards/rollout_reward_func/std": 1.0056862831115723, "sampling/importance_sampling_ratio/max": 1.3877732753753662, "sampling/importance_sampling_ratio/mean": 0.9871071577072144, "sampling/importance_sampling_ratio/min": 2.6309874417984602e-12, "sampling/sampling_logp_difference/max": 26.309024810791016, "sampling/sampling_logp_difference/mean": 0.3170478940010071, "step": 407, "step_time": 14.32103896499757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 125.0625, "completions/mean_terminated_length": 125.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14496727869845927, "epoch": 0.00816, "frac_reward_zero_std": 0.0, "grad_norm": 0.23778267204761505, "kl": 1.085901901125908, "learning_rate": 7.067372560915205e-06, "loss": -0.0273, "num_tokens": 18639269.0, "reward": 0.07249999046325684, "reward_std": 0.9347947835922241, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.192573070526123, "sampling/importance_sampling_ratio/mean": 0.9724105000495911, "sampling/importance_sampling_ratio/min": 0.5205116868019104, "sampling/sampling_logp_difference/max": 0.6528308391571045, "sampling/sampling_logp_difference/mean": 0.0355244055390358, "step": 408, "step_time": 14.239755011993111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 78.6875, "completions/mean_terminated_length": 78.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16649151989258826, "epoch": 0.00818, "frac_reward_zero_std": 0.0, "grad_norm": 0.13256265223026276, "kl": 1.089214500039816, "learning_rate": 7.054075043236623e-06, "loss": -0.0165, "num_tokens": 18685506.0, "reward": -0.17937500774860382, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.0468885898590088, "sampling/importance_sampling_ratio/mean": 0.9874179363250732, "sampling/importance_sampling_ratio/min": 0.5585072636604309, "sampling/sampling_logp_difference/max": 0.5872402191162109, "sampling/sampling_logp_difference/mean": 0.02532043121755123, "step": 409, "step_time": 11.546283608011436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 66.1875, "completions/mean_terminated_length": 66.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13401860429439694, "epoch": 0.0082, "frac_reward_zero_std": 0.0, "grad_norm": 0.1590304970741272, "kl": 0.6538475062698126, "learning_rate": 7.040766919264664e-06, "loss": -0.0117, "num_tokens": 18731189.0, "reward": 0.009999990463256836, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.05427086353302, "sampling/importance_sampling_ratio/mean": 0.9907795190811157, "sampling/importance_sampling_ratio/min": 0.5525025725364685, "sampling/sampling_logp_difference/max": 0.6002669334411621, "sampling/sampling_logp_difference/mean": 0.024756666272878647, "step": 410, "step_time": 11.800818372023059 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 71.8125, "completions/mean_terminated_length": 71.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16856624733190984, "epoch": 0.00822, "frac_reward_zero_std": 0.0, "grad_norm": 0.1496301293373108, "kl": 1.0960869323462248, "learning_rate": 7.027448364542479e-06, "loss": -0.0086, "num_tokens": 18777278.0, "reward": -0.24250000715255737, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 1.294957160949707, "sampling/importance_sampling_ratio/mean": 0.9987959265708923, "sampling/importance_sampling_ratio/min": 0.7113639116287231, "sampling/sampling_logp_difference/max": 0.35001039505004883, "sampling/sampling_logp_difference/mean": 0.030773771926760674, "step": 411, "step_time": 11.982787741981156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 847.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 94.5625, "completions/mean_terminated_length": 94.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2019884022884071, "epoch": 0.00824, "frac_reward_zero_std": 0.0, "grad_norm": 0.9032086133956909, "kl": 1.3987402208149433, "learning_rate": 7.01411955475081e-06, "loss": -0.0365, "num_tokens": 18824621.0, "reward": -0.24375000596046448, "reward_std": 1.0247212648391724, "rewards/rollout_reward_func/mean": -0.24375000596046448, "rewards/rollout_reward_func/std": 0.991960346698761, "sampling/importance_sampling_ratio/max": 1.7920243740081787, "sampling/importance_sampling_ratio/mean": 0.979035496711731, "sampling/importance_sampling_ratio/min": 0.42434656620025635, "sampling/sampling_logp_difference/max": 0.8659071922302246, "sampling/sampling_logp_difference/mean": 0.06575682759284973, "step": 412, "step_time": 12.87120399401465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 70.96875, "completions/mean_terminated_length": 70.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12974866037257016, "epoch": 0.00826, "frac_reward_zero_std": 0.0, "grad_norm": 0.32636013627052307, "kl": 0.5594551488757133, "learning_rate": 7.000780665705665e-06, "loss": -0.0168, "num_tokens": 18869416.0, "reward": 0.07312498986721039, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0605008602142334, "sampling/importance_sampling_ratio/mean": 0.9750734567642212, "sampling/importance_sampling_ratio/min": 0.45174720883369446, "sampling/sampling_logp_difference/max": 0.7943398952484131, "sampling/sampling_logp_difference/mean": 0.03371083363890648, "step": 413, "step_time": 14.105842985976778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 60.84375, "completions/mean_terminated_length": 60.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.146295431884937, "epoch": 0.00828, "frac_reward_zero_std": 0.0, "grad_norm": 0.10694678872823715, "kl": 0.8948976621031761, "learning_rate": 6.987431873356011e-06, "loss": -0.0074, "num_tokens": 18914852.0, "reward": -0.18000000715255737, "reward_std": 1.0533576011657715, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.1926387548446655, "sampling/importance_sampling_ratio/mean": 0.9923632740974426, "sampling/importance_sampling_ratio/min": 0.48375338315963745, "sampling/sampling_logp_difference/max": 0.7259974479675293, "sampling/sampling_logp_difference/mean": 0.02740565687417984, "step": 414, "step_time": 11.385130285016203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 456.0, "completions/max_terminated_length": 456.0, "completions/mean_length": 57.1875, "completions/mean_terminated_length": 57.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16752566047944129, "epoch": 0.0083, "frac_reward_zero_std": 0.0, "grad_norm": 0.27136892080307007, "kl": 2.0757262762635946, "learning_rate": 6.97407335378144e-06, "loss": -0.0134, "num_tokens": 18959970.0, "reward": -0.24312500655651093, "reward_std": 1.025570034980774, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.1498349905014038, "sampling/importance_sampling_ratio/mean": 0.938422441482544, "sampling/importance_sampling_ratio/min": 0.3936039209365845, "sampling/sampling_logp_difference/max": 0.9319822788238525, "sampling/sampling_logp_difference/mean": 0.06446763128042221, "step": 415, "step_time": 10.788961781996477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 820.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 81.5, "completions/mean_terminated_length": 81.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13391843414865434, "epoch": 0.00832, "frac_reward_zero_std": 0.0, "grad_norm": 0.23356486856937408, "kl": 1.8645704835653305, "learning_rate": 6.960705283189857e-06, "loss": -0.0132, "num_tokens": 19006893.0, "reward": -0.36937499046325684, "reward_std": 0.9427521824836731, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.089547872543335, "sampling/importance_sampling_ratio/mean": 0.9711216688156128, "sampling/importance_sampling_ratio/min": 0.33422139286994934, "sampling/sampling_logp_difference/max": 1.095876932144165, "sampling/sampling_logp_difference/mean": 0.039831846952438354, "step": 416, "step_time": 12.382824455991795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 100.46875, "completions/mean_terminated_length": 100.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15883283503353596, "epoch": 0.00834, "frac_reward_zero_std": 0.0, "grad_norm": 0.48324501514434814, "kl": 0.8003814555704594, "learning_rate": 6.94732783791515e-06, "loss": 0.0043, "num_tokens": 19054271.0, "reward": -0.18000000715255737, "reward_std": 1.0341415405273438, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.3551850318908691, "sampling/importance_sampling_ratio/mean": 1.0413726568222046, "sampling/importance_sampling_ratio/min": 0.743431806564331, "sampling/sampling_logp_difference/max": 0.2968759536743164, "sampling/sampling_logp_difference/mean": 0.023281540721654892, "step": 417, "step_time": 12.057275568025943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 51.15625, "completions/mean_terminated_length": 51.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1528986031189561, "epoch": 0.00836, "frac_reward_zero_std": 0.0, "grad_norm": 0.23636609315872192, "kl": 1.2708201743662357, "learning_rate": 6.933941194414866e-06, "loss": -0.0015, "num_tokens": 19099193.0, "reward": -0.24312500655651093, "reward_std": 1.0255701541900635, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.4862645864486694, "sampling/importance_sampling_ratio/mean": 1.0039817094802856, "sampling/importance_sampling_ratio/min": 0.42747464776039124, "sampling/sampling_logp_difference/max": 0.8498885631561279, "sampling/sampling_logp_difference/mean": 0.03702245652675629, "step": 418, "step_time": 11.562329973996384 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 173.09375, "completions/mean_terminated_length": 173.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13106774690095335, "epoch": 0.00838, "frac_reward_zero_std": 0.0, "grad_norm": 0.4881305694580078, "kl": 0.7641892656683922, "learning_rate": 6.920545529267882e-06, "loss": 0.0037, "num_tokens": 19147613.0, "reward": 0.2606250047683716, "reward_std": 1.0439103841781616, "rewards/rollout_reward_func/mean": 0.2606250047683716, "rewards/rollout_reward_func/std": 0.9921171069145203, "sampling/importance_sampling_ratio/max": 1.9470840692520142, "sampling/importance_sampling_ratio/mean": 1.0241384506225586, "sampling/importance_sampling_ratio/min": 0.5681219100952148, "sampling/sampling_logp_difference/max": 0.5935807824134827, "sampling/sampling_logp_difference/mean": 0.027684547007083893, "step": 419, "step_time": 14.193589392983995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 86.6875, "completions/mean_terminated_length": 86.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15962199750356376, "epoch": 0.0084, "frac_reward_zero_std": 0.25, "grad_norm": 0.14535732567310333, "kl": 1.3974178694188595, "learning_rate": 6.907141019172076e-06, "loss": -0.027, "num_tokens": 19195268.0, "reward": -0.3687500059604645, "reward_std": 0.7736372947692871, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.0773265361785889, "sampling/importance_sampling_ratio/mean": 0.9878302216529846, "sampling/importance_sampling_ratio/min": 0.7173070907592773, "sampling/sampling_logp_difference/max": 0.3321608901023865, "sampling/sampling_logp_difference/mean": 0.022280622273683548, "step": 420, "step_time": 13.386020213001757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 138.8125, "completions/mean_terminated_length": 138.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1278012291295454, "epoch": 0.00842, "frac_reward_zero_std": 0.0, "grad_norm": 0.2570098340511322, "kl": 0.8260579444468021, "learning_rate": 6.893727840941997e-06, "loss": -0.0164, "num_tokens": 19243423.0, "reward": 0.00812499225139618, "reward_std": 1.0777559280395508, "rewards/rollout_reward_func/mean": 0.00812499225139618, "rewards/rollout_reward_func/std": 1.0242840051651, "sampling/importance_sampling_ratio/max": 1.093625545501709, "sampling/importance_sampling_ratio/mean": 0.9982165694236755, "sampling/importance_sampling_ratio/min": 0.7455803751945496, "sampling/sampling_logp_difference/max": 0.2932724952697754, "sampling/sampling_logp_difference/mean": 0.016123009845614433, "step": 421, "step_time": 14.760583083996607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 41.1875, "completions/mean_terminated_length": 41.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1534660968463868, "epoch": 0.00844, "frac_reward_zero_std": 0.0, "grad_norm": 0.12901663780212402, "kl": 1.848303833976388, "learning_rate": 6.880306171506535e-06, "loss": -0.0026, "num_tokens": 19288233.0, "reward": -0.3056250214576721, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.4596930742263794, "sampling/importance_sampling_ratio/mean": 0.9834348559379578, "sampling/importance_sampling_ratio/min": 0.32163378596305847, "sampling/sampling_logp_difference/max": 1.1466648578643799, "sampling/sampling_logp_difference/mean": 0.04588877782225609, "step": 422, "step_time": 11.277228610983002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 97.8125, "completions/mean_terminated_length": 100.45160675048828, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15449555753730237, "epoch": 0.00846, "frac_reward_zero_std": 0.0, "grad_norm": 0.7217878699302673, "kl": 0.6616777870804071, "learning_rate": 6.866876187906582e-06, "loss": 0.0199, "num_tokens": 19333532.0, "reward": 0.13624998927116394, "reward_std": 1.0435711145401, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.3002266883850098, "sampling/importance_sampling_ratio/mean": 1.0246448516845703, "sampling/importance_sampling_ratio/min": 0.7335162162780762, "sampling/sampling_logp_difference/max": 0.29503440856933594, "sampling/sampling_logp_difference/mean": 0.02058260142803192, "step": 423, "step_time": 11.750150968007802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 92.1875, "completions/mean_terminated_length": 92.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12729254085570574, "epoch": 0.00848, "frac_reward_zero_std": 0.0, "grad_norm": 0.2543734610080719, "kl": 1.0116347968578339, "learning_rate": 6.8534380672927e-06, "loss": 0.0028, "num_tokens": 19379281.0, "reward": -0.11687500774860382, "reward_std": 0.9792624711990356, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.230726718902588, "sampling/importance_sampling_ratio/mean": 1.0135397911071777, "sampling/importance_sampling_ratio/min": 0.7850079536437988, "sampling/sampling_logp_difference/max": 0.31100040674209595, "sampling/sampling_logp_difference/mean": 0.021191878244280815, "step": 424, "step_time": 12.018193716023234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 147.09375, "completions/mean_terminated_length": 147.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11950938904192299, "epoch": 0.0085, "frac_reward_zero_std": 0.25, "grad_norm": 0.6850411295890808, "kl": 1.1638481169939041, "learning_rate": 6.839991986922785e-06, "loss": 0.0131, "num_tokens": 19426748.0, "reward": 0.07312498986721039, "reward_std": 0.7184121608734131, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 2.5628228187561035, "sampling/importance_sampling_ratio/mean": 1.0201606750488281, "sampling/importance_sampling_ratio/min": 0.12748965620994568, "sampling/sampling_logp_difference/max": 1.3932271003723145, "sampling/sampling_logp_difference/mean": 0.0497836172580719, "step": 425, "step_time": 13.843769565013645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 43.28125, "completions/mean_terminated_length": 43.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1264934060163796, "epoch": 0.00852, "frac_reward_zero_std": 0.0, "grad_norm": 0.685198187828064, "kl": 1.6039870269596577, "learning_rate": 6.826538124159727e-06, "loss": -0.0082, "num_tokens": 19471711.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.3497875928878784, "sampling/importance_sampling_ratio/mean": 1.019151210784912, "sampling/importance_sampling_ratio/min": 0.5413508415222168, "sampling/sampling_logp_difference/max": 0.6137533187866211, "sampling/sampling_logp_difference/mean": 0.02428426593542099, "step": 426, "step_time": 10.696892978994583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 99.75, "completions/mean_terminated_length": 99.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13773194490931928, "epoch": 0.00854, "frac_reward_zero_std": 0.0, "grad_norm": 0.2874240577220917, "kl": 1.0538090094923973, "learning_rate": 6.813076656469068e-06, "loss": -0.0201, "num_tokens": 19518635.0, "reward": -0.11687500774860382, "reward_std": 1.025570034980774, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.5128737688064575, "sampling/importance_sampling_ratio/mean": 1.018977403640747, "sampling/importance_sampling_ratio/min": 0.4719671905040741, "sampling/sampling_logp_difference/max": 0.7487716674804688, "sampling/sampling_logp_difference/mean": 0.02534968964755535, "step": 427, "step_time": 13.35248684000544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 112.8125, "completions/mean_terminated_length": 112.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16845631622709334, "epoch": 0.00856, "frac_reward_zero_std": 0.0, "grad_norm": 0.7256425023078918, "kl": 0.4939114861190319, "learning_rate": 6.799607761416671e-06, "loss": 0.0059, "num_tokens": 19565083.0, "reward": 0.009374991059303284, "reward_std": 1.0449373722076416, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.5966441631317139, "sampling/importance_sampling_ratio/mean": 1.020166277885437, "sampling/importance_sampling_ratio/min": 0.6525633335113525, "sampling/sampling_logp_difference/max": 0.4562389850616455, "sampling/sampling_logp_difference/mean": 0.022339917719364166, "step": 428, "step_time": 14.293062532990007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 87.9375, "completions/mean_terminated_length": 87.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13194416742771864, "epoch": 0.00858, "frac_reward_zero_std": 0.0, "grad_norm": 0.13648462295532227, "kl": 1.4748067501932383, "learning_rate": 6.786131616666364e-06, "loss": -0.0275, "num_tokens": 19611731.0, "reward": 0.009999990463256836, "reward_std": 0.9350782632827759, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.3141971826553345, "sampling/importance_sampling_ratio/mean": 0.9878854751586914, "sampling/importance_sampling_ratio/min": 0.3681078255176544, "sampling/sampling_logp_difference/max": 0.9993438720703125, "sampling/sampling_logp_difference/mean": 0.04053591564297676, "step": 429, "step_time": 13.378578515999834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 62.40625, "completions/mean_terminated_length": 62.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14112550881691277, "epoch": 0.0086, "frac_reward_zero_std": 0.0, "grad_norm": 0.719775915145874, "kl": 1.565274003893137, "learning_rate": 6.772648399977606e-06, "loss": -0.0024, "num_tokens": 19657401.0, "reward": -0.18000000715255737, "reward_std": 1.016998529434204, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.497072458267212, "sampling/importance_sampling_ratio/mean": 0.987533450126648, "sampling/importance_sampling_ratio/min": 0.6544100642204285, "sampling/sampling_logp_difference/max": 0.42315196990966797, "sampling/sampling_logp_difference/mean": 0.0402405746281147, "step": 430, "step_time": 11.206168438999157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 874.0, "completions/max_terminated_length": 874.0, "completions/mean_length": 142.75, "completions/mean_terminated_length": 142.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09754824405536056, "epoch": 0.00862, "frac_reward_zero_std": 0.0, "grad_norm": 0.17937391996383667, "kl": 1.4751742174848914, "learning_rate": 6.759158289203141e-06, "loss": -0.0186, "num_tokens": 19706215.0, "reward": 0.00812499225139618, "reward_std": 1.0247517824172974, "rewards/rollout_reward_func/mean": 0.00812499225139618, "rewards/rollout_reward_func/std": 1.0242838859558105, "sampling/importance_sampling_ratio/max": 1.238585352897644, "sampling/importance_sampling_ratio/mean": 1.002164602279663, "sampling/importance_sampling_ratio/min": 0.5380905270576477, "sampling/sampling_logp_difference/max": 0.6195969581604004, "sampling/sampling_logp_difference/mean": 0.016264010220766068, "step": 431, "step_time": 13.366515814974264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 72.9375, "completions/mean_terminated_length": 72.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11297933361493051, "epoch": 0.00864, "frac_reward_zero_std": 0.0, "grad_norm": 0.13634298741817474, "kl": 1.1456631664186716, "learning_rate": 6.745661462286648e-06, "loss": 0.009, "num_tokens": 19751777.0, "reward": -0.053125008940696716, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.670469045639038, "sampling/importance_sampling_ratio/mean": 1.0193361043930054, "sampling/importance_sampling_ratio/min": 0.43541625142097473, "sampling/sampling_logp_difference/max": 0.8397796154022217, "sampling/sampling_logp_difference/mean": 0.026583677157759666, "step": 432, "step_time": 11.440348741998605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 84.65625, "completions/mean_terminated_length": 84.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17594632506370544, "epoch": 0.00866, "frac_reward_zero_std": 0.25, "grad_norm": 0.16888348758220673, "kl": 0.9543821215629578, "learning_rate": 6.7321580972603996e-06, "loss": -0.0135, "num_tokens": 19799700.0, "reward": -0.6212499737739563, "reward_std": 0.6736763715744019, "rewards/rollout_reward_func/mean": -0.6212499737739563, "rewards/rollout_reward_func/std": 0.8010466694831848, "sampling/importance_sampling_ratio/max": 1.3464219570159912, "sampling/importance_sampling_ratio/mean": 0.955348014831543, "sampling/importance_sampling_ratio/min": 0.39486438035964966, "sampling/sampling_logp_difference/max": 0.9290695190429688, "sampling/sampling_logp_difference/mean": 0.053451601415872574, "step": 433, "step_time": 13.544986617998802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 103.84375, "completions/mean_terminated_length": 103.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14310635067522526, "epoch": 0.00868, "frac_reward_zero_std": 0.0, "grad_norm": 0.4943159818649292, "kl": 0.8385428134351969, "learning_rate": 6.718648372242909e-06, "loss": -0.0131, "num_tokens": 19846282.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.5285511016845703, "sampling/importance_sampling_ratio/mean": 1.01743745803833, "sampling/importance_sampling_ratio/min": 0.4210469722747803, "sampling/sampling_logp_difference/max": 0.8648880124092102, "sampling/sampling_logp_difference/mean": 0.038824714720249176, "step": 434, "step_time": 13.748091054003453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 124.625, "completions/mean_terminated_length": 124.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12616484612226486, "epoch": 0.0087, "frac_reward_zero_std": 0.0, "grad_norm": 0.436910480260849, "kl": 0.5268765706568956, "learning_rate": 6.705132465436579e-06, "loss": 0.0015, "num_tokens": 19893456.0, "reward": 0.009374991059303284, "reward_std": 1.0445914268493652, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.8183350563049316, "sampling/importance_sampling_ratio/mean": 1.0297834873199463, "sampling/importance_sampling_ratio/min": 0.736064612865448, "sampling/sampling_logp_difference/max": 0.3155820369720459, "sampling/sampling_logp_difference/mean": 0.017594629898667336, "step": 435, "step_time": 13.215373332997842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 168.0, "completions/max_terminated_length": 168.0, "completions/mean_length": 7.1875, "completions/mean_terminated_length": 7.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16289773536846042, "epoch": 0.00872, "frac_reward_zero_std": 0.0, "grad_norm": 0.6242549419403076, "kl": 1.1567578706890345, "learning_rate": 6.691610555125361e-06, "loss": -0.0005, "num_tokens": 19937191.0, "reward": -0.3687500059604645, "reward_std": 0.8798136115074158, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.7217662334442139, "sampling/importance_sampling_ratio/mean": 1.0620067119598389, "sampling/importance_sampling_ratio/min": 0.975145697593689, "sampling/sampling_logp_difference/max": 0.5435352325439453, "sampling/sampling_logp_difference/mean": 0.026141200214624405, "step": 436, "step_time": 10.634961358999135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 131.375, "completions/mean_terminated_length": 131.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1402748767286539, "epoch": 0.00874, "frac_reward_zero_std": 0.0, "grad_norm": 0.24796868860721588, "kl": 0.6110287830233574, "learning_rate": 6.678082819672389e-06, "loss": -0.0411, "num_tokens": 19984974.0, "reward": -0.3056250214576721, "reward_std": 0.8798531293869019, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 2.026186943054199, "sampling/importance_sampling_ratio/mean": 0.9977847933769226, "sampling/importance_sampling_ratio/min": 0.45935118198394775, "sampling/sampling_logp_difference/max": 0.7779930830001831, "sampling/sampling_logp_difference/mean": 0.040427714586257935, "step": 437, "step_time": 12.986246715998277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 92.3125, "completions/mean_terminated_length": 92.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1423318220768124, "epoch": 0.00876, "frac_reward_zero_std": 0.0, "grad_norm": 0.4408198893070221, "kl": 1.0459094755351543, "learning_rate": 6.664549437517642e-06, "loss": -0.0145, "num_tokens": 20032290.0, "reward": -0.17937500774860382, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.454612374305725, "sampling/importance_sampling_ratio/mean": 0.9846643805503845, "sampling/importance_sampling_ratio/min": 0.3980613052845001, "sampling/sampling_logp_difference/max": 0.9210186004638672, "sampling/sampling_logp_difference/mean": 0.033625632524490356, "step": 438, "step_time": 13.433738032006659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 50.46875, "completions/mean_terminated_length": 50.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1653993483632803, "epoch": 0.00878, "frac_reward_zero_std": 0.0, "grad_norm": 0.1873801201581955, "kl": 0.7168899402022362, "learning_rate": 6.651010587175577e-06, "loss": -0.0059, "num_tokens": 20078808.0, "reward": -0.5600000023841858, "reward_std": 0.8487529158592224, "rewards/rollout_reward_func/mean": -0.5600000023841858, "rewards/rollout_reward_func/std": 0.8448553085327148, "sampling/importance_sampling_ratio/max": 1.5367833375930786, "sampling/importance_sampling_ratio/mean": 1.0498943328857422, "sampling/importance_sampling_ratio/min": 0.9900561571121216, "sampling/sampling_logp_difference/max": 0.42973875999450684, "sampling/sampling_logp_difference/mean": 0.020876027643680573, "step": 439, "step_time": 12.375223096998525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1270.0, "completions/max_terminated_length": 1270.0, "completions/mean_length": 94.4375, "completions/mean_terminated_length": 94.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11859060823917389, "epoch": 0.0088, "frac_reward_zero_std": 0.0, "grad_norm": 0.23929038643836975, "kl": 0.6766565665602684, "learning_rate": 6.637466447232784e-06, "loss": -0.0134, "num_tokens": 20125635.0, "reward": -0.3062500059604645, "reward_std": 0.9338853359222412, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 1.2553308010101318, "sampling/importance_sampling_ratio/mean": 1.0326555967330933, "sampling/importance_sampling_ratio/min": 1.0038520097732544, "sampling/sampling_logp_difference/max": 0.21058857440948486, "sampling/sampling_logp_difference/mean": 0.01307680457830429, "step": 440, "step_time": 14.260570580016065 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 106.5, "completions/mean_terminated_length": 106.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1403266757261008, "epoch": 0.00882, "frac_reward_zero_std": 0.0, "grad_norm": 0.18445193767547607, "kl": 0.5981629323214293, "learning_rate": 6.623917196345622e-06, "loss": -0.0074, "num_tokens": 20172702.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.3782001733779907, "sampling/importance_sampling_ratio/mean": 1.0045487880706787, "sampling/importance_sampling_ratio/min": 0.3465018570423126, "sampling/sampling_logp_difference/max": 1.070678949356079, "sampling/sampling_logp_difference/mean": 0.034949708729982376, "step": 441, "step_time": 14.091482396994252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 71.9375, "completions/mean_terminated_length": 71.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13822519918903708, "epoch": 0.00884, "frac_reward_zero_std": 0.0, "grad_norm": 0.21469882130622864, "kl": 0.5218098647892475, "learning_rate": 6.610363013237871e-06, "loss": -0.0319, "num_tokens": 20218330.0, "reward": 0.07249999046325684, "reward_std": 0.9976819157600403, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.422894835472107, "sampling/importance_sampling_ratio/mean": 0.9940606355667114, "sampling/importance_sampling_ratio/min": 0.48298147320747375, "sampling/sampling_logp_difference/max": 0.7276570796966553, "sampling/sampling_logp_difference/mean": 0.03983263671398163, "step": 442, "step_time": 14.49971792800352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 36.90625, "completions/mean_terminated_length": 36.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1298399295192212, "epoch": 0.00886, "frac_reward_zero_std": 0.0, "grad_norm": 0.2293073683977127, "kl": 1.278594532981515, "learning_rate": 6.59680407669837e-06, "loss": 0.0005, "num_tokens": 20262455.0, "reward": -0.3056250214576721, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.1180341243743896, "sampling/importance_sampling_ratio/mean": 0.98064786195755, "sampling/importance_sampling_ratio/min": 0.433750182390213, "sampling/sampling_logp_difference/max": 0.8353771567344666, "sampling/sampling_logp_difference/mean": 0.030448103323578835, "step": 443, "step_time": 11.375516414009326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 106.3125, "completions/mean_terminated_length": 106.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17553621204569936, "epoch": 0.00888, "frac_reward_zero_std": 0.0, "grad_norm": 0.636799693107605, "kl": 2.033167521469295, "learning_rate": 6.583240565578657e-06, "loss": -0.0214, "num_tokens": 20309136.0, "reward": -0.24312500655651093, "reward_std": 1.0062534809112549, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.636398196220398, "sampling/importance_sampling_ratio/mean": 0.927544891834259, "sampling/importance_sampling_ratio/min": 0.4038752317428589, "sampling/sampling_logp_difference/max": 0.7320137023925781, "sampling/sampling_logp_difference/mean": 0.07770274579524994, "step": 444, "step_time": 14.473692014005792 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 104.84375, "completions/mean_terminated_length": 104.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12721785879693925, "epoch": 0.0089, "frac_reward_zero_std": 0.0, "grad_norm": 0.25474223494529724, "kl": 0.4786642361432314, "learning_rate": 6.569672658790611e-06, "loss": -0.0032, "num_tokens": 20356257.0, "reward": -0.3056250214576721, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.169370174407959, "sampling/importance_sampling_ratio/mean": 0.988460123538971, "sampling/importance_sampling_ratio/min": 0.5147718787193298, "sampling/sampling_logp_difference/max": 0.6638996601104736, "sampling/sampling_logp_difference/mean": 0.026815589517354965, "step": 445, "step_time": 13.472513239990803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 71.9375, "completions/mean_terminated_length": 71.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11421253299340606, "epoch": 0.00892, "frac_reward_zero_std": 0.0, "grad_norm": 0.2258012294769287, "kl": 0.7190050128847361, "learning_rate": 6.556100535304097e-06, "loss": -0.0257, "num_tokens": 20402710.0, "reward": 0.009374991059303284, "reward_std": 1.0445914268493652, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.6575592756271362, "sampling/importance_sampling_ratio/mean": 0.9513475894927979, "sampling/importance_sampling_ratio/min": 0.4427518844604492, "sampling/sampling_logp_difference/max": 0.8148090839385986, "sampling/sampling_logp_difference/mean": 0.06031103804707527, "step": 446, "step_time": 10.779214757967566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 71.8125, "completions/mean_terminated_length": 71.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10903931315988302, "epoch": 0.00894, "frac_reward_zero_std": 0.0, "grad_norm": 0.24773456156253815, "kl": 0.6459158640354872, "learning_rate": 6.542524374144598e-06, "loss": -0.0054, "num_tokens": 20447690.0, "reward": 0.008749991655349731, "reward_std": 1.0612759590148926, "rewards/rollout_reward_func/mean": 0.008749991655349731, "rewards/rollout_reward_func/std": 1.0249146223068237, "sampling/importance_sampling_ratio/max": 1.3286058902740479, "sampling/importance_sampling_ratio/mean": 1.010771632194519, "sampling/importance_sampling_ratio/min": 0.5299983620643616, "sampling/sampling_logp_difference/max": 0.6349589824676514, "sampling/sampling_logp_difference/mean": 0.02630303055047989, "step": 447, "step_time": 11.47892959901219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 90.21875, "completions/mean_terminated_length": 90.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13296979723963886, "epoch": 0.00896, "frac_reward_zero_std": 0.0, "grad_norm": 0.8967440128326416, "kl": 2.012577287852764, "learning_rate": 6.528944354390855e-06, "loss": 0.0038, "num_tokens": 20492199.0, "reward": 0.1356249898672104, "reward_std": 0.9898844361305237, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.5948280096054077, "sampling/importance_sampling_ratio/mean": 1.021984338760376, "sampling/importance_sampling_ratio/min": 0.3734176456928253, "sampling/sampling_logp_difference/max": 0.9850077629089355, "sampling/sampling_logp_difference/mean": 0.03995875269174576, "step": 448, "step_time": 11.897448572977737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 105.25, "completions/mean_terminated_length": 105.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09329217253252864, "epoch": 0.00898, "frac_reward_zero_std": 0.0, "grad_norm": 0.5519315600395203, "kl": 0.5589420106261969, "learning_rate": 6.515360655172512e-06, "loss": -0.0049, "num_tokens": 20538375.0, "reward": 0.07187499105930328, "reward_std": 0.9975189566612244, "rewards/rollout_reward_func/mean": 0.07187499105930328, "rewards/rollout_reward_func/std": 1.022985577583313, "sampling/importance_sampling_ratio/max": 1.3888740539550781, "sampling/importance_sampling_ratio/mean": 1.0184783935546875, "sampling/importance_sampling_ratio/min": 0.8990563154220581, "sampling/sampling_logp_difference/max": 0.32869982719421387, "sampling/sampling_logp_difference/mean": 0.010722077451646328, "step": 449, "step_time": 11.659547906994703 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 66.1875, "completions/mean_terminated_length": 66.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10428544552996755, "epoch": 0.009, "frac_reward_zero_std": 0.0, "grad_norm": 0.06992831826210022, "kl": 0.6388248056173325, "learning_rate": 6.501773455667742e-06, "loss": -0.018, "num_tokens": 20583459.0, "reward": -0.05375000834465027, "reward_std": 0.9153542518615723, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.3747451305389404, "sampling/importance_sampling_ratio/mean": 1.0134096145629883, "sampling/importance_sampling_ratio/min": 0.7393901348114014, "sampling/sampling_logp_difference/max": 0.3182840943336487, "sampling/sampling_logp_difference/mean": 0.013951278291642666, "step": 450, "step_time": 11.240951803018106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 73.0625, "completions/mean_terminated_length": 73.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10836401942651719, "epoch": 0.00902, "frac_reward_zero_std": 0.0, "grad_norm": 0.10247036069631577, "kl": 0.6986805871129036, "learning_rate": 6.488182935100893e-06, "loss": -0.0199, "num_tokens": 20630557.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.4111251831054688, "sampling/importance_sampling_ratio/mean": 1.0252461433410645, "sampling/importance_sampling_ratio/min": 0.733367919921875, "sampling/sampling_logp_difference/max": 0.337545245885849, "sampling/sampling_logp_difference/mean": 0.017602263018488884, "step": 451, "step_time": 11.875952518974373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 91.34375, "completions/mean_terminated_length": 91.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0831880362238735, "epoch": 0.00904, "frac_reward_zero_std": 0.0, "grad_norm": 0.31373533606529236, "kl": 0.5562594775110483, "learning_rate": 6.474589272740116e-06, "loss": 0.0095, "num_tokens": 20676342.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.1445846557617188, "sampling/importance_sampling_ratio/mean": 1.0076329708099365, "sampling/importance_sampling_ratio/min": 0.7157869935035706, "sampling/sampling_logp_difference/max": 0.34734320640563965, "sampling/sampling_logp_difference/mean": 0.011946350336074829, "step": 452, "step_time": 11.589367040010984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 69.8125, "completions/mean_terminated_length": 69.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1205929780844599, "epoch": 0.00906, "frac_reward_zero_std": 0.0, "grad_norm": 0.2309923619031906, "kl": 0.6801983527839184, "learning_rate": 6.46099264789501e-06, "loss": -0.0204, "num_tokens": 20722713.0, "reward": -0.11625000834465027, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.3327369689941406, "sampling/importance_sampling_ratio/mean": 1.0160397291183472, "sampling/importance_sampling_ratio/min": 0.5608255863189697, "sampling/sampling_logp_difference/max": 0.5782582759857178, "sampling/sampling_logp_difference/mean": 0.018329031765460968, "step": 453, "step_time": 11.51191663199279 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 118.5625, "completions/mean_terminated_length": 118.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06817652005702257, "epoch": 0.00908, "frac_reward_zero_std": 0.0, "grad_norm": 0.2895311117172241, "kl": 1.4344140756875277, "learning_rate": 6.447393239914247e-06, "loss": -0.0049, "num_tokens": 20769289.0, "reward": 0.3256250023841858, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": 0.3256250023841858, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.0623650550842285, "sampling/importance_sampling_ratio/mean": 0.9859050512313843, "sampling/importance_sampling_ratio/min": 0.34886160492897034, "sampling/sampling_logp_difference/max": 1.053001880645752, "sampling/sampling_logp_difference/mean": 0.017937462776899338, "step": 454, "step_time": 12.0458818860061 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 77.28125, "completions/mean_terminated_length": 77.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08086624834686518, "epoch": 0.0091, "frac_reward_zero_std": 0.0, "grad_norm": 0.1918015033006668, "kl": 0.6363368257880211, "learning_rate": 6.433791228183211e-06, "loss": -0.0195, "num_tokens": 20815972.0, "reward": -0.05375000834465027, "reward_std": 1.0173444747924805, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.3165920972824097, "sampling/importance_sampling_ratio/mean": 1.014622449874878, "sampling/importance_sampling_ratio/min": 0.7979792952537537, "sampling/sampling_logp_difference/max": 0.22564363479614258, "sampling/sampling_logp_difference/mean": 0.011612157337367535, "step": 455, "step_time": 10.860631176990864 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 357.0, "completions/max_terminated_length": 357.0, "completions/mean_length": 49.03125, "completions/mean_terminated_length": 49.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11391886381898075, "epoch": 0.00912, "frac_reward_zero_std": 0.0, "grad_norm": 0.3504219651222229, "kl": 1.3103700075298548, "learning_rate": 6.42018679212163e-06, "loss": -0.0245, "num_tokens": 20860271.0, "reward": 0.1356249898672104, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.8978055715560913, "sampling/importance_sampling_ratio/mean": 1.019002914428711, "sampling/importance_sampling_ratio/min": 0.525913655757904, "sampling/sampling_logp_difference/max": 0.6423144340515137, "sampling/sampling_logp_difference/mean": 0.03420864790678024, "step": 456, "step_time": 10.91183413699764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 35.0, "completions/mean_terminated_length": 35.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11666568368673325, "epoch": 0.00914, "frac_reward_zero_std": 0.0, "grad_norm": 0.34339237213134766, "kl": 0.5045151133090258, "learning_rate": 6.406580111181216e-06, "loss": -0.0191, "num_tokens": 20904387.0, "reward": -0.11687500774860382, "reward_std": 1.042907953262329, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.1318124532699585, "sampling/importance_sampling_ratio/mean": 0.9614574909210205, "sampling/importance_sampling_ratio/min": 0.45007482171058655, "sampling/sampling_logp_difference/max": 0.7982902526855469, "sampling/sampling_logp_difference/mean": 0.03692751005291939, "step": 457, "step_time": 11.238630341991666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 74.46875, "completions/mean_terminated_length": 74.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12312519841361791, "epoch": 0.00916, "frac_reward_zero_std": 0.0, "grad_norm": 0.5392564535140991, "kl": 2.8871582746505737, "learning_rate": 6.392971364843287e-06, "loss": -0.0054, "num_tokens": 20951241.0, "reward": -0.3056250214576721, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.4504345655441284, "sampling/importance_sampling_ratio/mean": 0.9552614092826843, "sampling/importance_sampling_ratio/min": 0.4547700583934784, "sampling/sampling_logp_difference/max": 0.7879278659820557, "sampling/sampling_logp_difference/mean": 0.047748446464538574, "step": 458, "step_time": 14.032660025994119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 98.90625, "completions/mean_terminated_length": 98.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08824611466843635, "epoch": 0.00918, "frac_reward_zero_std": 0.0, "grad_norm": 0.18969018757343292, "kl": 0.7841757014393806, "learning_rate": 6.379360732616405e-06, "loss": -0.0078, "num_tokens": 20997123.0, "reward": 0.1356249898672104, "reward_std": 1.0620803833007812, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175585746765137, "sampling/importance_sampling_ratio/max": 1.166688084602356, "sampling/importance_sampling_ratio/mean": 1.0045233964920044, "sampling/importance_sampling_ratio/min": 0.6104228496551514, "sampling/sampling_logp_difference/max": 0.4933452606201172, "sampling/sampling_logp_difference/mean": 0.014566617086529732, "step": 459, "step_time": 12.680313124008535 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "completions/clipped_ratio": 0.03125, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 69.875, "completions/mean_terminated_length": 71.61289978027344, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13919433893170208, "epoch": 0.0092, "frac_reward_zero_std": 0.0, "grad_norm": 0.47929811477661133, "kl": 1.8988102059811354, "learning_rate": 6.365748394034013e-06, "loss": -0.0154, "num_tokens": 21041406.0, "reward": 0.07249999046325684, "reward_std": 0.9981721639633179, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.3219929933547974, "sampling/importance_sampling_ratio/mean": 0.9762595295906067, "sampling/importance_sampling_ratio/min": 0.5370311141014099, "sampling/sampling_logp_difference/max": 0.62162184715271, "sampling/sampling_logp_difference/mean": 0.04112031310796738, "step": 460, "step_time": 14.098714583997207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 57.34375, "completions/mean_terminated_length": 57.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09839049458969384, "epoch": 0.00922, "frac_reward_zero_std": 0.0, "grad_norm": 0.529093325138092, "kl": 1.3192012272775173, "learning_rate": 6.352134528652057e-06, "loss": 0.0015, "num_tokens": 21086450.0, "reward": -0.053125008940696716, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0764241218566895, "sampling/importance_sampling_ratio/mean": 0.9915605783462524, "sampling/importance_sampling_ratio/min": 0.7561754584312439, "sampling/sampling_logp_difference/max": 0.27968597412109375, "sampling/sampling_logp_difference/mean": 0.013525355607271194, "step": 461, "step_time": 11.610547757023596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 96.53125, "completions/mean_terminated_length": 96.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14199342881329358, "epoch": 0.00924, "frac_reward_zero_std": 0.0, "grad_norm": 0.6495827436447144, "kl": 0.9842691738158464, "learning_rate": 6.3385193160466255e-06, "loss": -0.0079, "num_tokens": 21133743.0, "reward": -0.3062500059604645, "reward_std": 0.9510283470153809, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 2.0797598361968994, "sampling/importance_sampling_ratio/mean": 0.9959003925323486, "sampling/importance_sampling_ratio/min": 0.40293988585472107, "sampling/sampling_logp_difference/max": 0.9088599681854248, "sampling/sampling_logp_difference/mean": 0.04919808357954025, "step": 462, "step_time": 13.11459544498939 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 59.875, "completions/mean_terminated_length": 59.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10473196825478226, "epoch": 0.00926, "frac_reward_zero_std": 0.0, "grad_norm": 0.12111823260784149, "kl": 1.393854172900319, "learning_rate": 6.324902935811576e-06, "loss": -0.0069, "num_tokens": 21178510.0, "reward": -0.053125008940696716, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0845139026641846, "sampling/importance_sampling_ratio/mean": 0.9667661190032959, "sampling/importance_sampling_ratio/min": 0.38294658064842224, "sampling/sampling_logp_difference/max": 0.9596984386444092, "sampling/sampling_logp_difference/mean": 0.035183295607566833, "step": 463, "step_time": 11.320692974986741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 129.6875, "completions/mean_terminated_length": 129.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09587363712489605, "epoch": 0.00928, "frac_reward_zero_std": 0.0, "grad_norm": 0.19521375000476837, "kl": 0.9607531186193228, "learning_rate": 6.311285567556168e-06, "loss": -0.0169, "num_tokens": 21227574.0, "reward": -0.24312500655651093, "reward_std": 0.9418424367904663, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.0154945850372314, "sampling/importance_sampling_ratio/mean": 0.9889316558837891, "sampling/importance_sampling_ratio/min": 0.4236985146999359, "sampling/sampling_logp_difference/max": 0.8710128664970398, "sampling/sampling_logp_difference/mean": 0.014190476387739182, "step": 464, "step_time": 14.554594239001744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 49.375, "completions/mean_terminated_length": 49.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14558196993311867, "epoch": 0.0093, "frac_reward_zero_std": 0.0, "grad_norm": 0.459243506193161, "kl": 0.641282370314002, "learning_rate": 6.297667390902694e-06, "loss": -0.0205, "num_tokens": 21273001.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.2932171821594238, "sampling/importance_sampling_ratio/mean": 0.9926562309265137, "sampling/importance_sampling_ratio/min": 0.345284640789032, "sampling/sampling_logp_difference/max": 0.5559248924255371, "sampling/sampling_logp_difference/mean": 0.03467995300889015, "step": 465, "step_time": 11.09123938902485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 116.46875, "completions/mean_terminated_length": 116.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08016154984943569, "epoch": 0.00932, "frac_reward_zero_std": 0.0, "grad_norm": 0.05383894219994545, "kl": 1.2790247648954391, "learning_rate": 6.284048585484113e-06, "loss": -0.0038, "num_tokens": 21319816.0, "reward": 0.07312498986721039, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.1164313554763794, "sampling/importance_sampling_ratio/mean": 1.0111334323883057, "sampling/importance_sampling_ratio/min": 0.8839980363845825, "sampling/sampling_logp_difference/max": 0.12064409255981445, "sampling/sampling_logp_difference/mean": 0.007261891849339008, "step": 466, "step_time": 14.264772736976738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 420.0, "completions/max_terminated_length": 420.0, "completions/mean_length": 63.21875, "completions/mean_terminated_length": 63.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11531167733483016, "epoch": 0.00934, "frac_reward_zero_std": 0.0, "grad_norm": 0.18954084813594818, "kl": 1.0505227809771895, "learning_rate": 6.270429330941671e-06, "loss": -0.0213, "num_tokens": 21365522.0, "reward": 0.009374991059303284, "reward_std": 1.0617344379425049, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.0215580463409424, "sampling/importance_sampling_ratio/mean": 0.9739894866943359, "sampling/importance_sampling_ratio/min": 0.49357566237449646, "sampling/sampling_logp_difference/max": 0.7060743570327759, "sampling/sampling_logp_difference/mean": 0.022613516077399254, "step": 467, "step_time": 11.107003504010208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 48.625, "completions/mean_terminated_length": 48.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13589155062800273, "epoch": 0.00936, "frac_reward_zero_std": 0.0, "grad_norm": 0.1397532820701599, "kl": 1.3943481398746371, "learning_rate": 6.2568098069225436e-06, "loss": -0.0104, "num_tokens": 21411512.0, "reward": -0.3687500059604645, "reward_std": 0.8969566226005554, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.333235502243042, "sampling/importance_sampling_ratio/mean": 0.9963257312774658, "sampling/importance_sampling_ratio/min": 0.5213311910629272, "sampling/sampling_logp_difference/max": 0.6513018608093262, "sampling/sampling_logp_difference/mean": 0.029289763420820236, "step": 468, "step_time": 12.165261432004627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 68.28125, "completions/mean_terminated_length": 68.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12812255416065454, "epoch": 0.00938, "frac_reward_zero_std": 0.0, "grad_norm": 0.15346887707710266, "kl": 1.1461290512233973, "learning_rate": 6.243190193077457e-06, "loss": -0.0001, "num_tokens": 21457217.0, "reward": -0.11625000834465027, "reward_std": 0.9712425470352173, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0676602125167847, "sampling/importance_sampling_ratio/mean": 0.9672941565513611, "sampling/importance_sampling_ratio/min": 0.5096129179000854, "sampling/sampling_logp_difference/max": 0.6740004420280457, "sampling/sampling_logp_difference/mean": 0.029829390347003937, "step": 469, "step_time": 11.755575452007179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 109.15625, "completions/mean_terminated_length": 109.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11579065211117268, "epoch": 0.0094, "frac_reward_zero_std": 0.0, "grad_norm": 0.2968960702419281, "kl": 0.6142921876162291, "learning_rate": 6.2295706690583325e-06, "loss": -0.0111, "num_tokens": 21504437.0, "reward": 0.009999990463256836, "reward_std": 0.9350782632827759, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.7588632106781006, "sampling/importance_sampling_ratio/mean": 0.9937150478363037, "sampling/importance_sampling_ratio/min": 0.24887749552726746, "sampling/sampling_logp_difference/max": 1.3907299041748047, "sampling/sampling_logp_difference/mean": 0.03912492096424103, "step": 470, "step_time": 12.45995762298844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 429.0, "completions/max_terminated_length": 429.0, "completions/mean_length": 65.0625, "completions/mean_terminated_length": 65.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11066378280520439, "epoch": 0.00942, "frac_reward_zero_std": 0.0, "grad_norm": 0.2052629292011261, "kl": 1.1682209242135286, "learning_rate": 6.215951414515888e-06, "loss": -0.0126, "num_tokens": 21548714.0, "reward": 0.1993749886751175, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.6204464435577393, "sampling/importance_sampling_ratio/mean": 0.9992938041687012, "sampling/importance_sampling_ratio/min": 0.24059775471687317, "sampling/sampling_logp_difference/max": 1.4246139526367188, "sampling/sampling_logp_difference/mean": 0.0442623570561409, "step": 471, "step_time": 10.82544345700444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 46.25, "completions/mean_terminated_length": 46.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10585495491977781, "epoch": 0.00944, "frac_reward_zero_std": 0.0, "grad_norm": 0.025049129500985146, "kl": 1.2127960063517094, "learning_rate": 6.202332609097308e-06, "loss": -0.004, "num_tokens": 21593584.0, "reward": -0.18125000596046448, "reward_std": 1.015628457069397, "rewards/rollout_reward_func/mean": -0.18125000596046448, "rewards/rollout_reward_func/std": 1.0056734085083008, "sampling/importance_sampling_ratio/max": 1.4239543676376343, "sampling/importance_sampling_ratio/mean": 1.008749008178711, "sampling/importance_sampling_ratio/min": 0.7088268995285034, "sampling/sampling_logp_difference/max": 0.3534119129180908, "sampling/sampling_logp_difference/mean": 0.01601017266511917, "step": 472, "step_time": 11.594926807993033 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1261.0, "completions/max_terminated_length": 1261.0, "completions/mean_length": 165.9375, "completions/mean_terminated_length": 165.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0868009626865387, "epoch": 0.00946, "frac_reward_zero_std": 0.0, "grad_norm": 0.10449168086051941, "kl": 2.741363162174821, "learning_rate": 6.188714432443833e-06, "loss": -0.0158, "num_tokens": 21641978.0, "reward": 0.07187499105930328, "reward_std": 1.0527045726776123, "rewards/rollout_reward_func/mean": 0.07187499105930328, "rewards/rollout_reward_func/std": 1.022985577583313, "sampling/importance_sampling_ratio/max": 1.536906123161316, "sampling/importance_sampling_ratio/mean": 0.9903229475021362, "sampling/importance_sampling_ratio/min": 0.47711819410324097, "sampling/sampling_logp_difference/max": 0.6164236068725586, "sampling/sampling_logp_difference/mean": 0.028549883514642715, "step": 473, "step_time": 15.224551409999549 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 85.4375, "completions/mean_terminated_length": 85.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1523456524591893, "epoch": 0.00948, "frac_reward_zero_std": 0.0, "grad_norm": 0.45893028378486633, "kl": 1.817474614828825, "learning_rate": 6.175097064188427e-06, "loss": -0.0197, "num_tokens": 21689006.0, "reward": -0.3075000047683716, "reward_std": 0.933508038520813, "rewards/rollout_reward_func/mean": -0.3075000047683716, "rewards/rollout_reward_func/std": 0.9721641540527344, "sampling/importance_sampling_ratio/max": 1.1092921495437622, "sampling/importance_sampling_ratio/mean": 0.8952890038490295, "sampling/importance_sampling_ratio/min": 0.2109173834323883, "sampling/sampling_logp_difference/max": 1.5561859607696533, "sampling/sampling_logp_difference/mean": 0.08902080357074738, "step": 474, "step_time": 11.510946710994176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 61.40625, "completions/mean_terminated_length": 61.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13505524396896362, "epoch": 0.0095, "frac_reward_zero_std": 0.0, "grad_norm": 0.522661566734314, "kl": 0.7555138450115919, "learning_rate": 6.161480683953376e-06, "loss": -0.0056, "num_tokens": 21735025.0, "reward": -0.18062500655651093, "reward_std": 1.0161497592926025, "rewards/rollout_reward_func/mean": -0.18062500655651093, "rewards/rollout_reward_func/std": 1.0064499378204346, "sampling/importance_sampling_ratio/max": 1.8350551128387451, "sampling/importance_sampling_ratio/mean": 1.0393717288970947, "sampling/importance_sampling_ratio/min": 0.4180092513561249, "sampling/sampling_logp_difference/max": 0.8720993995666504, "sampling/sampling_logp_difference/mean": 0.045965466648340225, "step": 475, "step_time": 11.733598659986455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 80.21875, "completions/mean_terminated_length": 80.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1077277862932533, "epoch": 0.00952, "frac_reward_zero_std": 0.0, "grad_norm": 0.13205379247665405, "kl": 0.7565352357923985, "learning_rate": 6.147865471347945e-06, "loss": -0.0134, "num_tokens": 21781140.0, "reward": 0.009999990463256836, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.026511311531067, "sampling/importance_sampling_ratio/mean": 0.9953732490539551, "sampling/importance_sampling_ratio/min": 0.7071398496627808, "sampling/sampling_logp_difference/max": 0.3623480796813965, "sampling/sampling_logp_difference/mean": 0.012034543789923191, "step": 476, "step_time": 13.183864301987342 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 51.28125, "completions/mean_terminated_length": 51.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14557862794026732, "epoch": 0.00954, "frac_reward_zero_std": 0.0, "grad_norm": 0.1871446967124939, "kl": 0.7411675248295069, "learning_rate": 6.134251605965988e-06, "loss": -0.017, "num_tokens": 21826827.0, "reward": -0.24312500655651093, "reward_std": 1.0067436695098877, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.52890944480896, "sampling/importance_sampling_ratio/mean": 1.0084352493286133, "sampling/importance_sampling_ratio/min": 0.3131108283996582, "sampling/sampling_logp_difference/max": 1.1611547470092773, "sampling/sampling_logp_difference/mean": 0.0319286547601223, "step": 477, "step_time": 11.453061611013254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 75.15625, "completions/mean_terminated_length": 75.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1213111646939069, "epoch": 0.00956, "frac_reward_zero_std": 0.0, "grad_norm": 0.0911387950181961, "kl": 1.9146000295877457, "learning_rate": 6.1206392673835955e-06, "loss": -0.0311, "num_tokens": 21874654.0, "reward": -0.36937499046325684, "reward_std": 0.9894058108329773, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.3012515306472778, "sampling/importance_sampling_ratio/mean": 0.9877099990844727, "sampling/importance_sampling_ratio/min": 0.6122448444366455, "sampling/sampling_logp_difference/max": 0.4905238151550293, "sampling/sampling_logp_difference/mean": 0.023684632033109665, "step": 478, "step_time": 11.110561461006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 110.84375, "completions/mean_terminated_length": 110.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1201717434450984, "epoch": 0.00958, "frac_reward_zero_std": 0.0, "grad_norm": 0.26352691650390625, "kl": 0.6424729470163584, "learning_rate": 6.1070286351567154e-06, "loss": -0.0162, "num_tokens": 21922473.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.4076684713363647, "sampling/importance_sampling_ratio/mean": 1.0105478763580322, "sampling/importance_sampling_ratio/min": 0.7847713828086853, "sampling/sampling_logp_difference/max": 0.3388909101486206, "sampling/sampling_logp_difference/mean": 0.019964303821325302, "step": 479, "step_time": 12.408262750002905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 78.5625, "completions/mean_terminated_length": 78.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12415023799985647, "epoch": 0.0096, "frac_reward_zero_std": 0.0, "grad_norm": 0.3101961612701416, "kl": 0.556226409971714, "learning_rate": 6.093419888818785e-06, "loss": -0.0036, "num_tokens": 21967731.0, "reward": -0.05375000834465027, "reward_std": 1.0535087585449219, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.3990850448608398, "sampling/importance_sampling_ratio/mean": 0.9846808910369873, "sampling/importance_sampling_ratio/min": 0.5242112874984741, "sampling/sampling_logp_difference/max": 0.6454586982727051, "sampling/sampling_logp_difference/mean": 0.03243419528007507, "step": 480, "step_time": 11.832332395009871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 856.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 136.5625, "completions/mean_terminated_length": 136.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10390079172793776, "epoch": 0.00962, "frac_reward_zero_std": 0.0, "grad_norm": 0.3636389672756195, "kl": 0.6050856206566095, "learning_rate": 6.0798132078783714e-06, "loss": -0.0196, "num_tokens": 22014877.0, "reward": 0.13499999046325684, "reward_std": 1.0422446727752686, "rewards/rollout_reward_func/mean": 0.13499999046325684, "rewards/rollout_reward_func/std": 1.0170037746429443, "sampling/importance_sampling_ratio/max": 1.439229130744934, "sampling/importance_sampling_ratio/mean": 1.0226595401763916, "sampling/importance_sampling_ratio/min": 0.776625394821167, "sampling/sampling_logp_difference/max": 0.32840508222579956, "sampling/sampling_logp_difference/mean": 0.017996422946453094, "step": 481, "step_time": 12.929129909985932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 865.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 71.375, "completions/mean_terminated_length": 71.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10717450972879305, "epoch": 0.00964, "frac_reward_zero_std": 0.0, "grad_norm": 0.24617309868335724, "kl": 0.7322252951562405, "learning_rate": 6.0662087718167915e-06, "loss": -0.0185, "num_tokens": 22059961.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.4638994932174683, "sampling/importance_sampling_ratio/mean": 1.0387264490127563, "sampling/importance_sampling_ratio/min": 0.7907311320304871, "sampling/sampling_logp_difference/max": 0.3810538053512573, "sampling/sampling_logp_difference/mean": 0.020605476573109627, "step": 482, "step_time": 13.410493740993843 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 70.40625, "completions/mean_terminated_length": 70.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15888413996435702, "epoch": 0.00966, "frac_reward_zero_std": 0.0, "grad_norm": 0.44972607493400574, "kl": 2.1647740434855223, "learning_rate": 6.0526067600857556e-06, "loss": -0.0243, "num_tokens": 22106238.0, "reward": -0.3687500059604645, "reward_std": 0.9626314640045166, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.381203055381775, "sampling/importance_sampling_ratio/mean": 0.9581944346427917, "sampling/importance_sampling_ratio/min": 0.21606306731700897, "sampling/sampling_logp_difference/max": 1.5322017669677734, "sampling/sampling_logp_difference/mean": 0.06921355426311493, "step": 483, "step_time": 11.494195727012993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 222.28125, "completions/mean_terminated_length": 222.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0901545966626145, "epoch": 0.00968, "frac_reward_zero_std": 0.0, "grad_norm": 0.48441439867019653, "kl": 0.518435419537127, "learning_rate": 6.039007352104992e-06, "loss": 0.0073, "num_tokens": 22156745.0, "reward": 0.13624998927116394, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.4429231882095337, "sampling/importance_sampling_ratio/mean": 1.0156577825546265, "sampling/importance_sampling_ratio/min": 0.4890727698802948, "sampling/sampling_logp_difference/max": 0.6654990911483765, "sampling/sampling_logp_difference/mean": 0.018902916461229324, "step": 484, "step_time": 15.884723366980324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 66.03125, "completions/mean_terminated_length": 66.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12452219659462571, "epoch": 0.0097, "frac_reward_zero_std": 0.0, "grad_norm": 0.16168305277824402, "kl": 1.1493912544101477, "learning_rate": 6.025410727259885e-06, "loss": -0.0141, "num_tokens": 22202775.0, "reward": -0.05375000834465027, "reward_std": 0.9981721639633179, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.7489042282104492, "sampling/importance_sampling_ratio/mean": 0.9953100681304932, "sampling/importance_sampling_ratio/min": 0.4407593309879303, "sampling/sampling_logp_difference/max": 0.8190882205963135, "sampling/sampling_logp_difference/mean": 0.032192494720220566, "step": 485, "step_time": 11.457330919991364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 127.5625, "completions/mean_terminated_length": 127.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12386614363640547, "epoch": 0.00972, "frac_reward_zero_std": 0.0, "grad_norm": 0.09706485271453857, "kl": 0.6964193675667048, "learning_rate": 6.01181706489911e-06, "loss": -0.0272, "num_tokens": 22252122.0, "reward": -0.17937500774860382, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.4939501285552979, "sampling/importance_sampling_ratio/mean": 1.0228910446166992, "sampling/importance_sampling_ratio/min": 0.48626676201820374, "sampling/sampling_logp_difference/max": 0.7250585556030273, "sampling/sampling_logp_difference/mean": 0.02235836535692215, "step": 486, "step_time": 13.110827219017665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 30.375, "completions/mean_terminated_length": 30.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.124159867875278, "epoch": 0.00974, "frac_reward_zero_std": 0.0, "grad_norm": 0.12437690794467926, "kl": 1.0944392755627632, "learning_rate": 5.99822654433226e-06, "loss": -0.0044, "num_tokens": 22297421.0, "reward": -0.3687500059604645, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.0363181829452515, "sampling/importance_sampling_ratio/mean": 0.9878647327423096, "sampling/importance_sampling_ratio/min": 0.5839795470237732, "sampling/sampling_logp_difference/max": 0.5379233360290527, "sampling/sampling_logp_difference/mean": 0.017971165478229523, "step": 487, "step_time": 11.616425004998746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 126.375, "completions/mean_terminated_length": 126.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12859815428964794, "epoch": 0.00976, "frac_reward_zero_std": 0.0, "grad_norm": 0.17084218561649323, "kl": 0.7840884355828166, "learning_rate": 5.984639344827491e-06, "loss": -0.0152, "num_tokens": 22345760.0, "reward": 0.1356249898672104, "reward_std": 0.9791113138198853, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.032917857170105, "sampling/importance_sampling_ratio/mean": 0.9922895431518555, "sampling/importance_sampling_ratio/min": 0.6204957962036133, "sampling/sampling_logp_difference/max": 0.4770331382751465, "sampling/sampling_logp_difference/mean": 0.014911474660038948, "step": 488, "step_time": 13.651594406997901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 429.0, "completions/max_terminated_length": 429.0, "completions/mean_length": 22.78125, "completions/mean_terminated_length": 22.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12262968311551958, "epoch": 0.00978, "frac_reward_zero_std": 0.0, "grad_norm": 0.1198078915476799, "kl": 0.6882536113262177, "learning_rate": 5.971055645609147e-06, "loss": -0.0082, "num_tokens": 22389705.0, "reward": -0.053125008940696716, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.1108978986740112, "sampling/importance_sampling_ratio/mean": 0.9830993413925171, "sampling/importance_sampling_ratio/min": 0.2970772385597229, "sampling/sampling_logp_difference/max": 1.2136726379394531, "sampling/sampling_logp_difference/mean": 0.033034421503543854, "step": 489, "step_time": 10.7481872999997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 90.90625, "completions/mean_terminated_length": 90.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12753501400584355, "epoch": 0.0098, "frac_reward_zero_std": 0.0, "grad_norm": 0.1749672293663025, "kl": 0.6325374264270067, "learning_rate": 5.957475625855404e-06, "loss": -0.0153, "num_tokens": 22435606.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.398097038269043, "sampling/importance_sampling_ratio/mean": 0.9973615407943726, "sampling/importance_sampling_ratio/min": 0.22450551390647888, "sampling/sampling_logp_difference/max": 1.4938035011291504, "sampling/sampling_logp_difference/mean": 0.027824604883790016, "step": 490, "step_time": 14.444149490002019 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 78.46875, "completions/mean_terminated_length": 78.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12616711610462517, "epoch": 0.00982, "frac_reward_zero_std": 0.0, "grad_norm": 0.18641512095928192, "kl": 1.5475651379674673, "learning_rate": 5.943899464695905e-06, "loss": -0.0185, "num_tokens": 22481773.0, "reward": -0.11750000715255737, "reward_std": 0.9885983467102051, "rewards/rollout_reward_func/mean": -0.11750000715255737, "rewards/rollout_reward_func/std": 1.0166834592819214, "sampling/importance_sampling_ratio/max": 1.0223405361175537, "sampling/importance_sampling_ratio/mean": 0.9665659666061401, "sampling/importance_sampling_ratio/min": 0.3337937593460083, "sampling/sampling_logp_difference/max": 1.0971059799194336, "sampling/sampling_logp_difference/mean": 0.029970519244670868, "step": 491, "step_time": 11.034828829018807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 820.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 181.09375, "completions/mean_terminated_length": 181.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09138780669309199, "epoch": 0.00984, "frac_reward_zero_std": 0.0, "grad_norm": 0.10915009677410126, "kl": 0.9254063554108143, "learning_rate": 5.930327341209392e-06, "loss": -0.0094, "num_tokens": 22531801.0, "reward": 0.19874998927116394, "reward_std": 0.9347947835922241, "rewards/rollout_reward_func/mean": 0.19874998927116394, "rewards/rollout_reward_func/std": 1.0074424743652344, "sampling/importance_sampling_ratio/max": 1.7842612266540527, "sampling/importance_sampling_ratio/mean": 1.0076909065246582, "sampling/importance_sampling_ratio/min": 0.37052255868911743, "sampling/sampling_logp_difference/max": 0.9929251670837402, "sampling/sampling_logp_difference/mean": 0.02494887262582779, "step": 492, "step_time": 12.209000636015844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 73.0, "completions/mean_terminated_length": 73.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1580596145358868, "epoch": 0.00986, "frac_reward_zero_std": 0.0, "grad_norm": 0.40747180581092834, "kl": 0.6175853200256824, "learning_rate": 5.916759434421345e-06, "loss": -0.0134, "num_tokens": 22577791.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.5737913846969604, "sampling/importance_sampling_ratio/mean": 1.0255582332611084, "sampling/importance_sampling_ratio/min": 0.7920922040939331, "sampling/sampling_logp_difference/max": 0.4431338310241699, "sampling/sampling_logp_difference/mean": 0.030888434499502182, "step": 493, "step_time": 12.43926111496694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 73.90625, "completions/mean_terminated_length": 73.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10841526254080236, "epoch": 0.00988, "frac_reward_zero_std": 0.0, "grad_norm": 0.6666508316993713, "kl": 0.5973530896008015, "learning_rate": 5.903195923301632e-06, "loss": -0.0126, "num_tokens": 22622166.0, "reward": 0.26249998807907104, "reward_std": 1.0074068307876587, "rewards/rollout_reward_func/mean": 0.26249998807907104, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.3973463773727417, "sampling/importance_sampling_ratio/mean": 1.02023184299469, "sampling/importance_sampling_ratio/min": 0.8172483444213867, "sampling/sampling_logp_difference/max": 0.3345947265625, "sampling/sampling_logp_difference/mean": 0.01323620043694973, "step": 494, "step_time": 14.013224651978817 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 411.0, "completions/max_terminated_length": 411.0, "completions/mean_length": 35.0, "completions/mean_terminated_length": 35.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12225154088810086, "epoch": 0.0099, "frac_reward_zero_std": 0.0, "grad_norm": 0.2582184970378876, "kl": 1.1208812184631824, "learning_rate": 5.88963698676213e-06, "loss": -0.0105, "num_tokens": 22666748.0, "reward": -0.05375000834465027, "reward_std": 1.0705006122589111, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.219050407409668, "sampling/importance_sampling_ratio/mean": 1.0008790493011475, "sampling/importance_sampling_ratio/min": 0.7101035118103027, "sampling/sampling_logp_difference/max": 0.34335970878601074, "sampling/sampling_logp_difference/mean": 0.017012732103466988, "step": 495, "step_time": 10.74309939199884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 79.96875, "completions/mean_terminated_length": 79.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10025808052159846, "epoch": 0.00992, "frac_reward_zero_std": 0.0, "grad_norm": 0.1301981657743454, "kl": 1.2337675243616104, "learning_rate": 5.87608280365438e-06, "loss": -0.0121, "num_tokens": 22711526.0, "reward": 0.13437499105930328, "reward_std": 0.7579507827758789, "rewards/rollout_reward_func/mean": 0.13437499105930328, "rewards/rollout_reward_func/std": 1.0164483785629272, "sampling/importance_sampling_ratio/max": 1.0330486297607422, "sampling/importance_sampling_ratio/mean": 0.9780780076980591, "sampling/importance_sampling_ratio/min": 0.3624856770038605, "sampling/sampling_logp_difference/max": 1.0148733854293823, "sampling/sampling_logp_difference/mean": 0.023850396275520325, "step": 496, "step_time": 14.145048141028383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 483.0, "completions/max_terminated_length": 483.0, "completions/mean_length": 38.375, "completions/mean_terminated_length": 38.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12785830441862345, "epoch": 0.00994, "frac_reward_zero_std": 0.0, "grad_norm": 0.2591380774974823, "kl": 0.5864848140627146, "learning_rate": 5.862533552767218e-06, "loss": -0.0088, "num_tokens": 22756985.0, "reward": -0.3056250214576721, "reward_std": 0.9521816968917847, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.80045747756958, "sampling/importance_sampling_ratio/mean": 1.0168187618255615, "sampling/importance_sampling_ratio/min": 0.2927408814430237, "sampling/sampling_logp_difference/max": 1.2276325225830078, "sampling/sampling_logp_difference/mean": 0.032569173723459244, "step": 497, "step_time": 10.922434341002372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 97.90625, "completions/mean_terminated_length": 97.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1224248461658135, "epoch": 0.00996, "frac_reward_zero_std": 0.0, "grad_norm": 0.18152743577957153, "kl": 0.5464247222989798, "learning_rate": 5.848989412824425e-06, "loss": -0.0238, "num_tokens": 22803989.0, "reward": 0.1993749886751175, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.3234143257141113, "sampling/importance_sampling_ratio/mean": 1.047245740890503, "sampling/importance_sampling_ratio/min": 0.8292075395584106, "sampling/sampling_logp_difference/max": 0.842970609664917, "sampling/sampling_logp_difference/mean": 0.01809995248913765, "step": 498, "step_time": 11.712586905989156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 93.125, "completions/mean_terminated_length": 93.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08492048736661673, "epoch": 0.00998, "frac_reward_zero_std": 0.0, "grad_norm": 0.0524761900305748, "kl": 0.515866519883275, "learning_rate": 5.8354505624823585e-06, "loss": -0.0128, "num_tokens": 22850096.0, "reward": 0.1356249898672104, "reward_std": 1.0449373722076416, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.113249659538269, "sampling/importance_sampling_ratio/mean": 1.0172758102416992, "sampling/importance_sampling_ratio/min": 0.9917030930519104, "sampling/sampling_logp_difference/max": 0.10727125406265259, "sampling/sampling_logp_difference/mean": 0.006936509162187576, "step": 499, "step_time": 11.815950365999015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 112.125, "completions/mean_terminated_length": 112.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15439302055165172, "epoch": 0.01, "frac_reward_zero_std": 0.0, "grad_norm": 0.348704993724823, "kl": 0.7261798679828644, "learning_rate": 5.821917180327612e-06, "loss": -0.0188, "num_tokens": 22898095.0, "reward": -0.17937500774860382, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.0618009567260742, "sampling/importance_sampling_ratio/mean": 0.9775233268737793, "sampling/importance_sampling_ratio/min": 0.670846164226532, "sampling/sampling_logp_difference/max": 0.3991568684577942, "sampling/sampling_logp_difference/mean": 0.021319124847650528, "step": 500, "step_time": 11.732068568999239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 123.21875, "completions/mean_terminated_length": 123.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1176214138395153, "epoch": 0.01002, "frac_reward_zero_std": 0.0, "grad_norm": 0.18178528547286987, "kl": 0.6771227847784758, "learning_rate": 5.808389444874641e-06, "loss": -0.02, "num_tokens": 22945437.0, "reward": -0.053125008940696716, "reward_std": 0.9521816968917847, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.6198877096176147, "sampling/importance_sampling_ratio/mean": 1.034301519393921, "sampling/importance_sampling_ratio/min": 0.8530016541481018, "sampling/sampling_logp_difference/max": 0.46933186054229736, "sampling/sampling_logp_difference/mean": 0.01561569981276989, "step": 501, "step_time": 14.048699165010476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 838.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 79.9375, "completions/mean_terminated_length": 79.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08993125514825806, "epoch": 0.01004, "frac_reward_zero_std": 0.0, "grad_norm": 0.12311014533042908, "kl": 0.6347655355930328, "learning_rate": 5.794867534563422e-06, "loss": -0.0164, "num_tokens": 22990155.0, "reward": 0.3256250023841858, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": 0.3256250023841858, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.1796826124191284, "sampling/importance_sampling_ratio/mean": 0.9976932406425476, "sampling/importance_sampling_ratio/min": 0.3983171880245209, "sampling/sampling_logp_difference/max": 0.9204854965209961, "sampling/sampling_logp_difference/mean": 0.020324040204286575, "step": 502, "step_time": 13.244141835995833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 72.78125, "completions/mean_terminated_length": 72.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.152937984385062, "epoch": 0.01006, "frac_reward_zero_std": 0.0, "grad_norm": 0.22527366876602173, "kl": 0.6059077372774482, "learning_rate": 5.781351627757093e-06, "loss": -0.0162, "num_tokens": 23036453.0, "reward": -0.05437500774860382, "reward_std": 1.0336833000183105, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228264331817627, "sampling/importance_sampling_ratio/max": 1.2727735042572021, "sampling/importance_sampling_ratio/mean": 1.014758825302124, "sampling/importance_sampling_ratio/min": 0.5479498505592346, "sampling/sampling_logp_difference/max": 0.6063308715820312, "sampling/sampling_logp_difference/mean": 0.02695804089307785, "step": 503, "step_time": 11.635440099009429 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 18.15625, "completions/mean_terminated_length": 18.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12146571435732767, "epoch": 0.01008, "frac_reward_zero_std": 0.0, "grad_norm": 0.15672293305397034, "kl": 0.6757204998284578, "learning_rate": 5.767841902739602e-06, "loss": 0.0061, "num_tokens": 23079528.0, "reward": -0.11687500774860382, "reward_std": 0.9427522420883179, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.5921247005462646, "sampling/importance_sampling_ratio/mean": 1.030606746673584, "sampling/importance_sampling_ratio/min": 1.0007681846618652, "sampling/sampling_logp_difference/max": 0.4650411605834961, "sampling/sampling_logp_difference/mean": 0.012630168348550797, "step": 504, "step_time": 11.944680114997027 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 156.09375, "completions/mean_terminated_length": 156.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17544160230318084, "epoch": 0.0101, "frac_reward_zero_std": 0.0, "grad_norm": 0.4217784106731415, "kl": 0.7504643639549613, "learning_rate": 5.754338537713353e-06, "loss": -0.0239, "num_tokens": 23129039.0, "reward": -0.11687500774860382, "reward_std": 1.0617344379425049, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.3999767303466797, "sampling/importance_sampling_ratio/mean": 1.0127758979797363, "sampling/importance_sampling_ratio/min": 0.31897422671318054, "sampling/sampling_logp_difference/max": 1.1804635524749756, "sampling/sampling_logp_difference/mean": 0.0407465435564518, "step": 505, "step_time": 14.620145719993161 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 438.0, "completions/max_terminated_length": 438.0, "completions/mean_length": 24.75, "completions/mean_terminated_length": 24.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12163565459195524, "epoch": 0.01012, "frac_reward_zero_std": 0.0, "grad_norm": 0.11102747917175293, "kl": 0.5741633139550686, "learning_rate": 5.740841710796861e-06, "loss": -0.0039, "num_tokens": 23172491.0, "reward": 0.07249999046325684, "reward_std": 0.9984559416770935, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.0636651515960693, "sampling/importance_sampling_ratio/mean": 1.0092207193374634, "sampling/importance_sampling_ratio/min": 0.9367585182189941, "sampling/sampling_logp_difference/max": 0.06492447853088379, "sampling/sampling_logp_difference/mean": 0.0061827003955841064, "step": 506, "step_time": 10.69004435800889 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 91.21875, "completions/mean_terminated_length": 91.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12903197953710333, "epoch": 0.01014, "frac_reward_zero_std": 0.0, "grad_norm": 0.10411938279867172, "kl": 0.6382095068693161, "learning_rate": 5.727351600022396e-06, "loss": 0.019, "num_tokens": 23218475.0, "reward": -0.3062500059604645, "reward_std": 0.9981721639633179, "rewards/rollout_reward_func/mean": -0.3062500059604645, "rewards/rollout_reward_func/std": 0.9738971590995789, "sampling/importance_sampling_ratio/max": 1.251806616783142, "sampling/importance_sampling_ratio/mean": 1.0209803581237793, "sampling/importance_sampling_ratio/min": 0.9155924916267395, "sampling/sampling_logp_difference/max": 0.13546299934387207, "sampling/sampling_logp_difference/mean": 0.010966548696160316, "step": 507, "step_time": 12.140375284005131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 103.5625, "completions/mean_terminated_length": 103.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1784300389699638, "epoch": 0.01016, "frac_reward_zero_std": 0.0, "grad_norm": 0.3240290582180023, "kl": 1.4239275585860014, "learning_rate": 5.7138683833336385e-06, "loss": -0.005, "num_tokens": 23266295.0, "reward": -0.24312502145767212, "reward_std": 0.9430981874465942, "rewards/rollout_reward_func/mean": -0.24312502145767212, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.1757057905197144, "sampling/importance_sampling_ratio/mean": 0.9863675832748413, "sampling/importance_sampling_ratio/min": 0.39166879653930664, "sampling/sampling_logp_difference/max": 0.9372978210449219, "sampling/sampling_logp_difference/mean": 0.033719904720783234, "step": 508, "step_time": 14.853952233999735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 83.03125, "completions/mean_terminated_length": 83.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17620736663229764, "epoch": 0.01018, "frac_reward_zero_std": 0.0, "grad_norm": 0.34655943512916565, "kl": 0.5716280657798052, "learning_rate": 5.70039223858333e-06, "loss": -0.0108, "num_tokens": 23313926.0, "reward": -0.17937500774860382, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.4785722494125366, "sampling/importance_sampling_ratio/mean": 1.0155391693115234, "sampling/importance_sampling_ratio/min": 0.5431088209152222, "sampling/sampling_logp_difference/max": 0.6103307008743286, "sampling/sampling_logp_difference/mean": 0.034206703305244446, "step": 509, "step_time": 11.519080781996308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 41.46875, "completions/mean_terminated_length": 41.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17473642295226455, "epoch": 0.0102, "frac_reward_zero_std": 0.0, "grad_norm": 0.15997470915317535, "kl": 0.8879991415888071, "learning_rate": 5.686923343530932e-06, "loss": -0.0052, "num_tokens": 23359486.0, "reward": -0.3687500059604645, "reward_std": 0.9626314640045166, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.1416630744934082, "sampling/importance_sampling_ratio/mean": 1.0121251344680786, "sampling/importance_sampling_ratio/min": 0.8005579710006714, "sampling/sampling_logp_difference/max": 0.22249174118041992, "sampling/sampling_logp_difference/mean": 0.015458576381206512, "step": 510, "step_time": 12.334630448000098 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 168.59375, "completions/mean_terminated_length": 168.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16611181607004255, "epoch": 0.01022, "frac_reward_zero_std": 0.0, "grad_norm": 0.2961507737636566, "kl": 0.9518023394048214, "learning_rate": 5.673461875840275e-06, "loss": -0.029, "num_tokens": 23408824.0, "reward": 0.009999990463256836, "reward_std": 0.9626314640045166, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.2090398073196411, "sampling/importance_sampling_ratio/mean": 0.9997588992118835, "sampling/importance_sampling_ratio/min": 0.6732812523841858, "sampling/sampling_logp_difference/max": 0.39539289474487305, "sampling/sampling_logp_difference/mean": 0.018518492579460144, "step": 511, "step_time": 13.270248202003131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 119.28125, "completions/mean_terminated_length": 119.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18504807632416487, "epoch": 0.01024, "frac_reward_zero_std": 0.0, "grad_norm": 0.23022738099098206, "kl": 2.083823896944523, "learning_rate": 5.6600080130772166e-06, "loss": -0.0055, "num_tokens": 23457121.0, "reward": -0.3056250214576721, "reward_std": 0.9521816968917847, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.1263847351074219, "sampling/importance_sampling_ratio/mean": 0.9877668023109436, "sampling/importance_sampling_ratio/min": 0.47903144359588623, "sampling/sampling_logp_difference/max": 0.7359890937805176, "sampling/sampling_logp_difference/mean": 0.030148927122354507, "step": 512, "step_time": 14.48821185300767 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 829.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 152.875, "completions/mean_terminated_length": 152.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13825353875290602, "epoch": 0.01026, "frac_reward_zero_std": 0.0, "grad_norm": 0.24710991978645325, "kl": 0.6659236997365952, "learning_rate": 5.646561932707302e-06, "loss": -0.0128, "num_tokens": 23505084.0, "reward": 0.07187499105930328, "reward_std": 1.0527045726776123, "rewards/rollout_reward_func/mean": 0.07187499105930328, "rewards/rollout_reward_func/std": 1.022985577583313, "sampling/importance_sampling_ratio/max": 1.0545812845230103, "sampling/importance_sampling_ratio/mean": 1.0052456855773926, "sampling/importance_sampling_ratio/min": 0.6586941480636597, "sampling/sampling_logp_difference/max": 0.44157397747039795, "sampling/sampling_logp_difference/mean": 0.011536860838532448, "step": 513, "step_time": 12.885788644998684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 77.5625, "completions/mean_terminated_length": 77.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14592200960032642, "epoch": 0.01028, "frac_reward_zero_std": 0.0, "grad_norm": 0.15175513923168182, "kl": 0.5869811158627272, "learning_rate": 5.633123812093419e-06, "loss": -0.0168, "num_tokens": 23550586.0, "reward": 0.009374991059303284, "reward_std": 1.0790722370147705, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.340126395225525, "sampling/importance_sampling_ratio/mean": 1.0141717195510864, "sampling/importance_sampling_ratio/min": 0.7468549609184265, "sampling/sampling_logp_difference/max": 0.3411903381347656, "sampling/sampling_logp_difference/mean": 0.01636044681072235, "step": 514, "step_time": 10.881744881022314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 258.0, "completions/max_terminated_length": 258.0, "completions/mean_length": 25.4375, "completions/mean_terminated_length": 25.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14895748283015564, "epoch": 0.0103, "frac_reward_zero_std": 0.0, "grad_norm": 0.1886836439371109, "kl": 0.8427586797624826, "learning_rate": 5.619693828493467e-06, "loss": -0.0099, "num_tokens": 23593489.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0401054620742798, "sampling/importance_sampling_ratio/mean": 0.9908137917518616, "sampling/importance_sampling_ratio/min": 0.6271243095397949, "sampling/sampling_logp_difference/max": 0.46656930446624756, "sampling/sampling_logp_difference/mean": 0.017963824793696404, "step": 515, "step_time": 10.169735748007952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 54.375, "completions/mean_terminated_length": 54.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16898954345379025, "epoch": 0.01032, "frac_reward_zero_std": 0.0, "grad_norm": 0.5030161142349243, "kl": 0.8273781575262547, "learning_rate": 5.606272159058005e-06, "loss": -0.0152, "num_tokens": 23638621.0, "reward": 0.07124999165534973, "reward_std": 1.0162526369094849, "rewards/rollout_reward_func/mean": 0.07124999165534973, "rewards/rollout_reward_func/std": 1.0223935842514038, "sampling/importance_sampling_ratio/max": 1.372999668121338, "sampling/importance_sampling_ratio/mean": 0.9844696521759033, "sampling/importance_sampling_ratio/min": 0.43058639764785767, "sampling/sampling_logp_difference/max": 0.8424608707427979, "sampling/sampling_logp_difference/mean": 0.0362909734249115, "step": 516, "step_time": 11.854980249991058 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 49.90625, "completions/mean_terminated_length": 49.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14167966204695404, "epoch": 0.01034, "frac_reward_zero_std": 0.0, "grad_norm": 0.24195510149002075, "kl": 0.5922452565282583, "learning_rate": 5.5928589808279266e-06, "loss": 0.0018, "num_tokens": 23682041.0, "reward": 0.07249999046325684, "reward_std": 1.0343364477157593, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.107442021369934, "sampling/importance_sampling_ratio/mean": 1.0015809535980225, "sampling/importance_sampling_ratio/min": 0.5744540691375732, "sampling/sampling_logp_difference/max": 0.5654529929161072, "sampling/sampling_logp_difference/mean": 0.014357459731400013, "step": 517, "step_time": 11.508985248998215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 64.90625, "completions/mean_terminated_length": 64.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13910723908338696, "epoch": 0.01036, "frac_reward_zero_std": 0.0, "grad_norm": 0.06435997784137726, "kl": 0.878341706469655, "learning_rate": 5.5794544707321184e-06, "loss": -0.0109, "num_tokens": 23727484.0, "reward": 0.07249999046325684, "reward_std": 0.9981721639633179, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.1849384307861328, "sampling/importance_sampling_ratio/mean": 1.0036782026290894, "sampling/importance_sampling_ratio/min": 0.6233552694320679, "sampling/sampling_logp_difference/max": 0.4734358787536621, "sampling/sampling_logp_difference/mean": 0.01750672236084938, "step": 518, "step_time": 11.273131135989388 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 847.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 68.15625, "completions/mean_terminated_length": 68.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1330374832614325, "epoch": 0.01038, "frac_reward_zero_std": 0.0, "grad_norm": 0.09336567670106888, "kl": 0.8079373724758625, "learning_rate": 5.566058805585135e-06, "loss": -0.0185, "num_tokens": 23772463.0, "reward": 0.07312498986721039, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.2638217210769653, "sampling/importance_sampling_ratio/mean": 0.9836471080780029, "sampling/importance_sampling_ratio/min": 0.3456859886646271, "sampling/sampling_logp_difference/max": 1.0621473789215088, "sampling/sampling_logp_difference/mean": 0.035741791129112244, "step": 519, "step_time": 12.9840035810048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 85.4375, "completions/mean_terminated_length": 85.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16829940135357901, "epoch": 0.0104, "frac_reward_zero_std": 0.0, "grad_norm": 0.12214615941047668, "kl": 0.5641338117420673, "learning_rate": 5.55267216208485e-06, "loss": -0.0262, "num_tokens": 23819514.0, "reward": -0.11625000834465027, "reward_std": 0.9712425470352173, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1156630516052246, "sampling/importance_sampling_ratio/mean": 1.0089304447174072, "sampling/importance_sampling_ratio/min": 0.7978159189224243, "sampling/sampling_logp_difference/max": 0.22707843780517578, "sampling/sampling_logp_difference/mean": 0.0106731615960598, "step": 520, "step_time": 13.15962297898659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 91.875, "completions/mean_terminated_length": 91.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17191234120400622, "epoch": 0.01042, "frac_reward_zero_std": 0.0, "grad_norm": 0.21093721687793732, "kl": 0.8270219974219799, "learning_rate": 5.539294716810144e-06, "loss": -0.0106, "num_tokens": 23866154.0, "reward": 0.07312498986721039, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.2762705087661743, "sampling/importance_sampling_ratio/mean": 1.018491506576538, "sampling/importance_sampling_ratio/min": 0.805279016494751, "sampling/sampling_logp_difference/max": 0.24213558435440063, "sampling/sampling_logp_difference/mean": 0.01556338369846344, "step": 521, "step_time": 12.19270491401403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 438.0, "completions/max_terminated_length": 438.0, "completions/mean_length": 68.28125, "completions/mean_terminated_length": 68.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12101910333149135, "epoch": 0.01044, "frac_reward_zero_std": 0.0, "grad_norm": 0.2256370484828949, "kl": 0.8062322195619345, "learning_rate": 5.525926646218561e-06, "loss": -0.0148, "num_tokens": 23910027.0, "reward": 0.38749998807907104, "reward_std": 0.9893723726272583, "rewards/rollout_reward_func/mean": 0.38749998807907104, "rewards/rollout_reward_func/std": 0.9504328370094299, "sampling/importance_sampling_ratio/max": 1.0457854270935059, "sampling/importance_sampling_ratio/mean": 0.998424768447876, "sampling/importance_sampling_ratio/min": 0.6194084882736206, "sampling/sampling_logp_difference/max": 0.481020450592041, "sampling/sampling_logp_difference/mean": 0.01102009229362011, "step": 522, "step_time": 11.054187912995985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 78.15625, "completions/mean_terminated_length": 80.16128540039062, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21249385178089142, "epoch": 0.01046, "frac_reward_zero_std": 0.0, "grad_norm": 0.4295911192893982, "kl": 0.9105945024639368, "learning_rate": 5.512568126643991e-06, "loss": 0.0025, "num_tokens": 23956991.0, "reward": -0.36875003576278687, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767791748047, "sampling/importance_sampling_ratio/max": 1.1198872327804565, "sampling/importance_sampling_ratio/mean": 0.9892424941062927, "sampling/importance_sampling_ratio/min": 0.5181503891944885, "sampling/sampling_logp_difference/max": 0.6810343265533447, "sampling/sampling_logp_difference/mean": 0.0255885012447834, "step": 523, "step_time": 11.753804776002653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 52.0, "completions/mean_terminated_length": 52.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15179727639770135, "epoch": 0.01048, "frac_reward_zero_std": 0.0, "grad_norm": 0.25948619842529297, "kl": 0.8784563355147839, "learning_rate": 5.499219334294335e-06, "loss": -0.011, "num_tokens": 24001071.0, "reward": 0.009374991059303284, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.058421015739441, "sampling/importance_sampling_ratio/mean": 1.0023612976074219, "sampling/importance_sampling_ratio/min": 0.7317319512367249, "sampling/sampling_logp_difference/max": 0.39404964447021484, "sampling/sampling_logp_difference/mean": 0.012844488024711609, "step": 524, "step_time": 11.926581140993221 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 66.1875, "completions/mean_terminated_length": 66.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16449320642277598, "epoch": 0.0105, "frac_reward_zero_std": 0.0, "grad_norm": 0.09221015125513077, "kl": 0.7949921749532223, "learning_rate": 5.485880445249192e-06, "loss": -0.0164, "num_tokens": 24046450.0, "reward": 0.009374991059303284, "reward_std": 0.9897516965866089, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.0402705669403076, "sampling/importance_sampling_ratio/mean": 0.9966321587562561, "sampling/importance_sampling_ratio/min": 0.6219860911369324, "sampling/sampling_logp_difference/max": 0.47519874572753906, "sampling/sampling_logp_difference/mean": 0.0164828859269619, "step": 525, "step_time": 11.328425726977002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 106.9375, "completions/mean_terminated_length": 106.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2065329069737345, "epoch": 0.01052, "frac_reward_zero_std": 0.0, "grad_norm": 0.24547529220581055, "kl": 1.0730864685028791, "learning_rate": 5.472551635457521e-06, "loss": -0.0111, "num_tokens": 24093857.0, "reward": -0.3056250214576721, "reward_std": 0.8798531293869019, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.6542751789093018, "sampling/importance_sampling_ratio/mean": 0.9870142340660095, "sampling/importance_sampling_ratio/min": 0.20534773170948029, "sampling/sampling_logp_difference/max": 1.1023762226104736, "sampling/sampling_logp_difference/mean": 0.046938683837652206, "step": 526, "step_time": 13.494290620008542 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 87.8125, "completions/mean_terminated_length": 87.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13249660341534764, "epoch": 0.01054, "frac_reward_zero_std": 0.0, "grad_norm": 0.34832626581192017, "kl": 0.8507236782461405, "learning_rate": 5.459233080735336e-06, "loss": -0.0168, "num_tokens": 24139478.0, "reward": 0.38874998688697815, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": 0.38874998688697815, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.097183346748352, "sampling/importance_sampling_ratio/mean": 0.9923315048217773, "sampling/importance_sampling_ratio/min": 0.5936371684074402, "sampling/sampling_logp_difference/max": 0.5213932991027832, "sampling/sampling_logp_difference/mean": 0.01605076715350151, "step": 527, "step_time": 13.912787771019794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 61.25, "completions/mean_terminated_length": 61.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16403298126533628, "epoch": 0.01056, "frac_reward_zero_std": 0.0, "grad_norm": 0.2210760861635208, "kl": 0.9259470086544752, "learning_rate": 5.4459249567633776e-06, "loss": -0.0113, "num_tokens": 24184730.0, "reward": 0.009999990463256836, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.0708612203598022, "sampling/importance_sampling_ratio/mean": 0.9897053241729736, "sampling/importance_sampling_ratio/min": 0.5516730546951294, "sampling/sampling_logp_difference/max": 0.5961513519287109, "sampling/sampling_logp_difference/mean": 0.02334980107843876, "step": 528, "step_time": 10.902748962995247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 87.09375, "completions/mean_terminated_length": 87.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1280130761442706, "epoch": 0.01058, "frac_reward_zero_std": 0.0, "grad_norm": 0.1945999264717102, "kl": 0.6682311557233334, "learning_rate": 5.432627439084797e-06, "loss": -0.0113, "num_tokens": 24230843.0, "reward": 0.1981249898672104, "reward_std": 1.0338244438171387, "rewards/rollout_reward_func/mean": 0.1981249898672104, "rewards/rollout_reward_func/std": 1.0069226026535034, "sampling/importance_sampling_ratio/max": 1.1396689414978027, "sampling/importance_sampling_ratio/mean": 0.9825143814086914, "sampling/importance_sampling_ratio/min": 0.5492082238197327, "sampling/sampling_logp_difference/max": 0.6005992889404297, "sampling/sampling_logp_difference/mean": 0.02194903790950775, "step": 529, "step_time": 11.28455650399701 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 99.46875, "completions/mean_terminated_length": 99.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15444949339143932, "epoch": 0.0106, "frac_reward_zero_std": 0.0, "grad_norm": 0.3108198642730713, "kl": 0.7060983404517174, "learning_rate": 5.419340703102839e-06, "loss": -0.0013, "num_tokens": 24276848.0, "reward": -0.05437500774860382, "reward_std": 1.0336731672286987, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228137969970703, "sampling/importance_sampling_ratio/max": 1.4053839445114136, "sampling/importance_sampling_ratio/mean": 1.0186047554016113, "sampling/importance_sampling_ratio/min": 0.7923842072486877, "sampling/sampling_logp_difference/max": 0.33752888441085815, "sampling/sampling_logp_difference/mean": 0.019257502630352974, "step": 530, "step_time": 11.830499317009526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 110.03125, "completions/mean_terminated_length": 110.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1399952108040452, "epoch": 0.01062, "frac_reward_zero_std": 0.0, "grad_norm": 0.13768425583839417, "kl": 0.5442140679806471, "learning_rate": 5.406064924078536e-06, "loss": 0.0007, "num_tokens": 24323830.0, "reward": -0.053125008940696716, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.128252625465393, "sampling/importance_sampling_ratio/mean": 1.0192720890045166, "sampling/importance_sampling_ratio/min": 0.9365860223770142, "sampling/sampling_logp_difference/max": 0.12079165875911713, "sampling/sampling_logp_difference/mean": 0.009648195467889309, "step": 531, "step_time": 11.128527822998876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 483.0, "completions/max_terminated_length": 483.0, "completions/mean_length": 51.28125, "completions/mean_terminated_length": 51.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11832743528066203, "epoch": 0.01064, "frac_reward_zero_std": 0.0, "grad_norm": 0.17610517144203186, "kl": 1.383729686960578, "learning_rate": 5.392800277128386e-06, "loss": -0.0061, "num_tokens": 24367409.0, "reward": 0.2618749737739563, "reward_std": 0.8605483770370483, "rewards/rollout_reward_func/mean": 0.2618749737739563, "rewards/rollout_reward_func/std": 0.9930903911590576, "sampling/importance_sampling_ratio/max": 1.207804560661316, "sampling/importance_sampling_ratio/mean": 0.9910366535186768, "sampling/importance_sampling_ratio/min": 0.641412079334259, "sampling/sampling_logp_difference/max": 0.4441179037094116, "sampling/sampling_logp_difference/mean": 0.026562128216028214, "step": 532, "step_time": 11.069151743984548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 41.1875, "completions/mean_terminated_length": 41.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1748791600111872, "epoch": 0.01066, "frac_reward_zero_std": 0.0, "grad_norm": 0.18063189089298248, "kl": 1.1017994116991758, "learning_rate": 5.3795469372220525e-06, "loss": -0.0082, "num_tokens": 24412418.0, "reward": -0.18000000715255737, "reward_std": 1.0173444747924805, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.0488922595977783, "sampling/importance_sampling_ratio/mean": 0.9998754262924194, "sampling/importance_sampling_ratio/min": 0.8055081367492676, "sampling/sampling_logp_difference/max": 0.21625423431396484, "sampling/sampling_logp_difference/mean": 0.013755814172327518, "step": 533, "step_time": 13.522645541997917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 82.1875, "completions/mean_terminated_length": 82.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14687756600324064, "epoch": 0.01068, "frac_reward_zero_std": 0.0, "grad_norm": 0.280310720205307, "kl": 1.0907997377216816, "learning_rate": 5.366305079180043e-06, "loss": -0.0284, "num_tokens": 24459166.0, "reward": -0.17937500774860382, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.0843554735183716, "sampling/importance_sampling_ratio/mean": 0.9510749578475952, "sampling/importance_sampling_ratio/min": 0.41933372616767883, "sampling/sampling_logp_difference/max": 0.8695683479309082, "sampling/sampling_logp_difference/mean": 0.04528815299272537, "step": 534, "step_time": 11.664353117994324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 135.15625, "completions/mean_terminated_length": 135.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10938795027323067, "epoch": 0.0107, "frac_reward_zero_std": 0.0, "grad_norm": 0.2815229594707489, "kl": 0.7927504684776068, "learning_rate": 5.35307487767142e-06, "loss": -0.0116, "num_tokens": 24507930.0, "reward": 0.00812499225139618, "reward_std": 1.0243281126022339, "rewards/rollout_reward_func/mean": 0.00812499225139618, "rewards/rollout_reward_func/std": 1.0242713689804077, "sampling/importance_sampling_ratio/max": 1.0776677131652832, "sampling/importance_sampling_ratio/mean": 0.9926071166992188, "sampling/importance_sampling_ratio/min": 0.6207592487335205, "sampling/sampling_logp_difference/max": 0.4759392738342285, "sampling/sampling_logp_difference/mean": 0.01940442994236946, "step": 535, "step_time": 11.462219759981963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 90.90625, "completions/mean_terminated_length": 90.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1319435485638678, "epoch": 0.01072, "frac_reward_zero_std": 0.0, "grad_norm": 0.15585528314113617, "kl": 1.0265500620007515, "learning_rate": 5.339856507211485e-06, "loss": -0.0219, "num_tokens": 24555699.0, "reward": -0.24312500655651093, "reward_std": 0.9791113138198853, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.0742051601409912, "sampling/importance_sampling_ratio/mean": 1.0033721923828125, "sampling/importance_sampling_ratio/min": 0.7912507653236389, "sampling/sampling_logp_difference/max": 0.23412275314331055, "sampling/sampling_logp_difference/mean": 0.011876551434397697, "step": 536, "step_time": 13.40607020000607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 74.875, "completions/mean_terminated_length": 74.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09579382673837245, "epoch": 0.01074, "frac_reward_zero_std": 0.0, "grad_norm": 0.14843982458114624, "kl": 0.5147187365218997, "learning_rate": 5.326650142159479e-06, "loss": -0.0121, "num_tokens": 24600127.0, "reward": 0.3256250023841858, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": 0.3256250023841858, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.6802552938461304, "sampling/importance_sampling_ratio/mean": 1.0267595052719116, "sampling/importance_sampling_ratio/min": 0.8651338815689087, "sampling/sampling_logp_difference/max": 0.5189776420593262, "sampling/sampling_logp_difference/mean": 0.012880812399089336, "step": 537, "step_time": 10.901887484993495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 65.5, "completions/mean_terminated_length": 65.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11797494854545221, "epoch": 0.01076, "frac_reward_zero_std": 0.0, "grad_norm": 0.10276440531015396, "kl": 0.5861163381487131, "learning_rate": 5.313455956716286e-06, "loss": 0.0037, "num_tokens": 24646613.0, "reward": -0.3687500059604645, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.0978038311004639, "sampling/importance_sampling_ratio/mean": 1.0131421089172363, "sampling/importance_sampling_ratio/min": 0.9668715000152588, "sampling/sampling_logp_difference/max": 0.07918554544448853, "sampling/sampling_logp_difference/mean": 0.007939948700368404, "step": 538, "step_time": 11.954090681989328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 71.375, "completions/mean_terminated_length": 71.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10359319782583043, "epoch": 0.01078, "frac_reward_zero_std": 0.0, "grad_norm": 0.45918112993240356, "kl": 3.7750383019447327, "learning_rate": 5.3002741249221305e-06, "loss": -0.0152, "num_tokens": 24691871.0, "reward": -0.05375000834465027, "reward_std": 1.0705006122589111, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.1057771444320679, "sampling/importance_sampling_ratio/mean": 0.9754543304443359, "sampling/importance_sampling_ratio/min": 0.20262281596660614, "sampling/sampling_logp_difference/max": 1.5963134765625, "sampling/sampling_logp_difference/mean": 0.03650575876235962, "step": 539, "step_time": 11.463073905993951 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 131.9375, "completions/mean_terminated_length": 131.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10891493537928909, "epoch": 0.0108, "frac_reward_zero_std": 0.0, "grad_norm": 0.2793061435222626, "kl": 0.5955506972968578, "learning_rate": 5.287104820654283e-06, "loss": 0.003, "num_tokens": 24739856.0, "reward": 0.009374991059303284, "reward_std": 1.0062534809112549, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.426864743232727, "sampling/importance_sampling_ratio/mean": 0.9497174024581909, "sampling/importance_sampling_ratio/min": 0.3871139585971832, "sampling/sampling_logp_difference/max": 0.9597777128219604, "sampling/sampling_logp_difference/mean": 0.05368587374687195, "step": 540, "step_time": 13.095667409994348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 50.4375, "completions/mean_terminated_length": 50.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0834359037107788, "epoch": 0.01082, "frac_reward_zero_std": 0.0, "grad_norm": 0.12289531528949738, "kl": 0.6592752188444138, "learning_rate": 5.273948217624771e-06, "loss": -0.0053, "num_tokens": 24784310.0, "reward": 0.1356249898672104, "reward_std": 1.0259159803390503, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.1811895370483398, "sampling/importance_sampling_ratio/mean": 1.0136878490447998, "sampling/importance_sampling_ratio/min": 0.8416572213172913, "sampling/sampling_logp_difference/max": 0.17186778783798218, "sampling/sampling_logp_difference/mean": 0.010178297758102417, "step": 541, "step_time": 11.209713637981622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 65.625, "completions/mean_terminated_length": 65.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0863465522415936, "epoch": 0.01084, "frac_reward_zero_std": 0.0, "grad_norm": 0.19823643565177917, "kl": 0.6098579913377762, "learning_rate": 5.260804489378083e-06, "loss": -0.0146, "num_tokens": 24829683.0, "reward": -0.053125008940696716, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0729892253875732, "sampling/importance_sampling_ratio/mean": 1.0064141750335693, "sampling/importance_sampling_ratio/min": 0.6364196538925171, "sampling/sampling_logp_difference/max": 0.45206165313720703, "sampling/sampling_logp_difference/mean": 0.01344133261591196, "step": 542, "step_time": 11.887504763020843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 76.71875, "completions/mean_terminated_length": 76.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08594664942938834, "epoch": 0.01086, "frac_reward_zero_std": 0.0, "grad_norm": 0.24751493334770203, "kl": 0.6204926297068596, "learning_rate": 5.2476738092888805e-06, "loss": -0.0105, "num_tokens": 24875572.0, "reward": -0.053125008940696716, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.0279133319854736, "sampling/importance_sampling_ratio/mean": 0.9919206500053406, "sampling/importance_sampling_ratio/min": 0.4386151432991028, "sampling/sampling_logp_difference/max": 0.8244071006774902, "sampling/sampling_logp_difference/mean": 0.015021691098809242, "step": 543, "step_time": 11.946263404977799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 79.25, "completions/mean_terminated_length": 79.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.059455305396113545, "epoch": 0.01088, "frac_reward_zero_std": 0.0, "grad_norm": 0.045733653008937836, "kl": 0.5273599550127983, "learning_rate": 5.234556350559705e-06, "loss": -0.0156, "num_tokens": 24920318.0, "reward": 0.3256250023841858, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": 0.3256250023841858, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.0194824934005737, "sampling/importance_sampling_ratio/mean": 1.0068777799606323, "sampling/importance_sampling_ratio/min": 0.9867115616798401, "sampling/sampling_logp_difference/max": 0.015383727848529816, "sampling/sampling_logp_difference/mean": 0.003233205759897828, "step": 544, "step_time": 11.804797657008749 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 63.9375, "completions/mean_terminated_length": 63.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1012823173077777, "epoch": 0.0109, "frac_reward_zero_std": 0.0, "grad_norm": 0.2725133001804352, "kl": 0.8723727017641068, "learning_rate": 5.221452286218712e-06, "loss": -0.0127, "num_tokens": 24965596.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.3125333786010742, "sampling/importance_sampling_ratio/mean": 1.0147802829742432, "sampling/importance_sampling_ratio/min": 0.6961807608604431, "sampling/sampling_logp_difference/max": 0.3622264862060547, "sampling/sampling_logp_difference/mean": 0.014951546676456928, "step": 545, "step_time": 10.899394502994255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 27.84375, "completions/mean_terminated_length": 27.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.045707412296906114, "epoch": 0.01092, "frac_reward_zero_std": 0.0, "grad_norm": 0.25108325481414795, "kl": 1.1321278847754002, "learning_rate": 5.2083617891173625e-06, "loss": -0.0049, "num_tokens": 25006985.0, "reward": 0.3256250023841858, "reward_std": 0.8522208333015442, "rewards/rollout_reward_func/mean": 0.3256250023841858, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.0231529474258423, "sampling/importance_sampling_ratio/mean": 0.9972430467605591, "sampling/importance_sampling_ratio/min": 0.6953749656677246, "sampling/sampling_logp_difference/max": 0.3633613586425781, "sampling/sampling_logp_difference/mean": 0.008813465014100075, "step": 546, "step_time": 11.164503746993432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 125.1875, "completions/mean_terminated_length": 125.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11075935792177916, "epoch": 0.01094, "frac_reward_zero_std": 0.0, "grad_norm": 0.35641515254974365, "kl": 0.6821503918617964, "learning_rate": 5.195285031928168e-06, "loss": -0.0129, "num_tokens": 25055849.0, "reward": -0.18062500655651093, "reward_std": 0.9976600408554077, "rewards/rollout_reward_func/mean": -0.18062500655651093, "rewards/rollout_reward_func/std": 1.0064371824264526, "sampling/importance_sampling_ratio/max": 1.4577962160110474, "sampling/importance_sampling_ratio/mean": 1.0244405269622803, "sampling/importance_sampling_ratio/min": 0.5377727150917053, "sampling/sampling_logp_difference/max": 0.620211124420166, "sampling/sampling_logp_difference/mean": 0.021001094952225685, "step": 547, "step_time": 12.27773860898742 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 40.46875, "completions/mean_terminated_length": 40.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12564799608662724, "epoch": 0.01096, "frac_reward_zero_std": 0.0, "grad_norm": 0.22658951580524445, "kl": 0.9561644084751606, "learning_rate": 5.182222187142388e-06, "loss": -0.0198, "num_tokens": 25101286.0, "reward": -0.18000000715255737, "reward_std": 1.0343364477157593, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.1423485279083252, "sampling/importance_sampling_ratio/mean": 0.9769607782363892, "sampling/importance_sampling_ratio/min": 0.5576478838920593, "sampling/sampling_logp_difference/max": 0.731196403503418, "sampling/sampling_logp_difference/mean": 0.03306327015161514, "step": 548, "step_time": 10.852082908000739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 802.0, "completions/max_terminated_length": 802.0, "completions/mean_length": 112.125, "completions/mean_terminated_length": 112.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08021511096740142, "epoch": 0.01098, "frac_reward_zero_std": 0.0, "grad_norm": 0.08117262274026871, "kl": 1.4972630608826876, "learning_rate": 5.169173427067784e-06, "loss": -0.0104, "num_tokens": 25148705.0, "reward": -0.18000000715255737, "reward_std": 1.033846139907837, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.1827070713043213, "sampling/importance_sampling_ratio/mean": 1.0055723190307617, "sampling/importance_sampling_ratio/min": 0.686041533946991, "sampling/sampling_logp_difference/max": 0.37671470642089844, "sampling/sampling_logp_difference/mean": 0.016268331557512283, "step": 549, "step_time": 12.448698078005691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 856.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 73.75, "completions/mean_terminated_length": 73.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08873021724866703, "epoch": 0.011, "frac_reward_zero_std": 0.0, "grad_norm": 0.30836963653564453, "kl": 0.6056941840797663, "learning_rate": 5.156138923826317e-06, "loss": -0.0245, "num_tokens": 25193835.0, "reward": 0.13499999046325684, "reward_std": 1.0614171028137207, "rewards/rollout_reward_func/mean": 0.13499999046325684, "rewards/rollout_reward_func/std": 1.0170037746429443, "sampling/importance_sampling_ratio/max": 1.3605598211288452, "sampling/importance_sampling_ratio/mean": 1.0130369663238525, "sampling/importance_sampling_ratio/min": 0.8399679660797119, "sampling/sampling_logp_difference/max": 0.2961618900299072, "sampling/sampling_logp_difference/mean": 0.012634620070457458, "step": 550, "step_time": 13.149455298022076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 102.84375, "completions/mean_terminated_length": 102.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04918464686488733, "epoch": 0.01102, "frac_reward_zero_std": 0.0, "grad_norm": 0.1619061380624771, "kl": 1.5105866491794586, "learning_rate": 5.143118849351898e-06, "loss": -0.0058, "num_tokens": 25239356.0, "reward": 0.2618749737739563, "reward_std": 0.9795305728912354, "rewards/rollout_reward_func/mean": 0.2618749737739563, "rewards/rollout_reward_func/std": 0.9930903911590576, "sampling/importance_sampling_ratio/max": 1.1337847709655762, "sampling/importance_sampling_ratio/mean": 0.9810464382171631, "sampling/importance_sampling_ratio/min": 0.427288293838501, "sampling/sampling_logp_difference/max": 0.8557581901550293, "sampling/sampling_logp_difference/mean": 0.026642486453056335, "step": 551, "step_time": 13.326278903979983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 136.40625, "completions/mean_terminated_length": 136.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0764274641405791, "epoch": 0.01104, "frac_reward_zero_std": 0.0, "grad_norm": 0.2026679366827011, "kl": 2.1556399278342724, "learning_rate": 5.1301133753881115e-06, "loss": -0.0339, "num_tokens": 25288544.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.156590461730957, "sampling/importance_sampling_ratio/mean": 0.9902660250663757, "sampling/importance_sampling_ratio/min": 0.512337327003479, "sampling/sampling_logp_difference/max": 0.6689338684082031, "sampling/sampling_logp_difference/mean": 0.021332615986466408, "step": 552, "step_time": 14.22121748601785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 98.90625, "completions/mean_terminated_length": 98.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05263965763151646, "epoch": 0.01106, "frac_reward_zero_std": 0.0, "grad_norm": 0.12363690137863159, "kl": 1.6280135940760374, "learning_rate": 5.1171226734859505e-06, "loss": -0.0141, "num_tokens": 25334027.0, "reward": 0.3256250023841858, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": 0.3256250023841858, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.1627123355865479, "sampling/importance_sampling_ratio/mean": 0.9884999990463257, "sampling/importance_sampling_ratio/min": 0.4620749354362488, "sampling/sampling_logp_difference/max": 0.7719695568084717, "sampling/sampling_logp_difference/mean": 0.019804615527391434, "step": 553, "step_time": 11.818200513007469 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 102.84375, "completions/mean_terminated_length": 102.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08897599822375923, "epoch": 0.01108, "frac_reward_zero_std": 0.0, "grad_norm": 0.647014856338501, "kl": 0.8026885576546192, "learning_rate": 5.1041469150015535e-06, "loss": -0.0144, "num_tokens": 25381026.0, "reward": 0.13499999046325684, "reward_std": 1.02540123462677, "rewards/rollout_reward_func/mean": 0.13499999046325684, "rewards/rollout_reward_func/std": 1.0170037746429443, "sampling/importance_sampling_ratio/max": 1.1657823324203491, "sampling/importance_sampling_ratio/mean": 0.9818546175956726, "sampling/importance_sampling_ratio/min": 0.24747829139232635, "sampling/sampling_logp_difference/max": 1.396254539489746, "sampling/sampling_logp_difference/mean": 0.029454590752720833, "step": 554, "step_time": 13.340951917009079 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 99.34375, "completions/mean_terminated_length": 99.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08025117550278082, "epoch": 0.0111, "frac_reward_zero_std": 0.0, "grad_norm": 0.0737067461013794, "kl": 0.583886630833149, "learning_rate": 5.091186271093949e-06, "loss": -0.0158, "num_tokens": 25429000.0, "reward": -0.24250000715255737, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.0525158643722534, "sampling/importance_sampling_ratio/mean": 1.0097582340240479, "sampling/importance_sampling_ratio/min": 0.9296212792396545, "sampling/sampling_logp_difference/max": 0.07300320267677307, "sampling/sampling_logp_difference/mean": 0.005923807621002197, "step": 555, "step_time": 14.655804800000624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 125.59375, "completions/mean_terminated_length": 125.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11744428763631731, "epoch": 0.01112, "frac_reward_zero_std": 0.0, "grad_norm": 0.4526209235191345, "kl": 0.6316033378243446, "learning_rate": 5.078240912722787e-06, "loss": -0.0175, "num_tokens": 25476835.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.430519700050354, "sampling/importance_sampling_ratio/mean": 1.0259466171264648, "sampling/importance_sampling_ratio/min": 0.6329838037490845, "sampling/sampling_logp_difference/max": 0.45737218856811523, "sampling/sampling_logp_difference/mean": 0.026305459439754486, "step": 556, "step_time": 11.738406431999465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 118.28125, "completions/mean_terminated_length": 118.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06182819255627692, "epoch": 0.01114, "frac_reward_zero_std": 0.0, "grad_norm": 0.1306697279214859, "kl": 0.5911995116621256, "learning_rate": 5.065311010646101e-06, "loss": -0.0077, "num_tokens": 25524518.0, "reward": 0.13624998927116394, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1001801490783691, "sampling/importance_sampling_ratio/mean": 1.014103651046753, "sampling/importance_sampling_ratio/min": 1.0004258155822754, "sampling/sampling_logp_difference/max": 0.09566622972488403, "sampling/sampling_logp_difference/mean": 0.005348962731659412, "step": 557, "step_time": 11.505770514995675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 81.65625, "completions/mean_terminated_length": 81.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05257180711487308, "epoch": 0.01116, "frac_reward_zero_std": 0.0, "grad_norm": 0.0796174630522728, "kl": 0.6179434042423964, "learning_rate": 5.052396735418037e-06, "loss": 0.0, "num_tokens": 25569570.0, "reward": 0.07124999165534973, "reward_std": 1.0332906246185303, "rewards/rollout_reward_func/mean": 0.07124999165534973, "rewards/rollout_reward_func/std": 1.0223809480667114, "sampling/importance_sampling_ratio/max": 1.071343183517456, "sampling/importance_sampling_ratio/mean": 1.0117112398147583, "sampling/importance_sampling_ratio/min": 1.000665307044983, "sampling/sampling_logp_difference/max": 0.06743954122066498, "sampling/sampling_logp_difference/mean": 0.005793428048491478, "step": 558, "step_time": 11.91361299900018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 77.0, "completions/mean_terminated_length": 77.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06022365519311279, "epoch": 0.01118, "frac_reward_zero_std": 0.0, "grad_norm": 0.16661472618579865, "kl": 0.6599197555333376, "learning_rate": 5.039498257386617e-06, "loss": -0.014, "num_tokens": 25614456.0, "reward": 0.13624998927116394, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.6519657373428345, "sampling/importance_sampling_ratio/mean": 1.02866792678833, "sampling/importance_sampling_ratio/min": 1.000592827796936, "sampling/sampling_logp_difference/max": 0.4994114637374878, "sampling/sampling_logp_difference/mean": 0.009487796574831009, "step": 559, "step_time": 11.29678936998971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 104.40625, "completions/mean_terminated_length": 104.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07310086500365287, "epoch": 0.0112, "frac_reward_zero_std": 0.25, "grad_norm": 0.25854334235191345, "kl": 1.5980904344469309, "learning_rate": 5.026615746691483e-06, "loss": 0.0011, "num_tokens": 25662119.0, "reward": -0.18125000596046448, "reward_std": 0.7995399236679077, "rewards/rollout_reward_func/mean": -0.18125000596046448, "rewards/rollout_reward_func/std": 1.0056734085083008, "sampling/importance_sampling_ratio/max": 1.145042061805725, "sampling/importance_sampling_ratio/mean": 1.0067957639694214, "sampling/importance_sampling_ratio/min": 0.6952292323112488, "sampling/sampling_logp_difference/max": 0.3641190528869629, "sampling/sampling_logp_difference/mean": 0.013915371149778366, "step": 560, "step_time": 14.37818609799433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 94.5625, "completions/mean_terminated_length": 94.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10335572413168848, "epoch": 0.01122, "frac_reward_zero_std": 0.0, "grad_norm": 0.2924056649208069, "kl": 1.7251445017755032, "learning_rate": 5.013749373261662e-06, "loss": -0.0133, "num_tokens": 25708416.0, "reward": 0.009374991059303284, "reward_std": 1.0790722370147705, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.093603253364563, "sampling/importance_sampling_ratio/mean": 0.978102445602417, "sampling/importance_sampling_ratio/min": 0.6109132766723633, "sampling/sampling_logp_difference/max": 0.49317455291748047, "sampling/sampling_logp_difference/mean": 0.02057623863220215, "step": 561, "step_time": 13.284334871990723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 78.5625, "completions/mean_terminated_length": 78.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06184029008727521, "epoch": 0.01124, "frac_reward_zero_std": 0.0, "grad_norm": 0.1889553815126419, "kl": 1.0254182070493698, "learning_rate": 5.0008993068133165e-06, "loss": -0.0097, "num_tokens": 25753476.0, "reward": 0.009374991059303284, "reward_std": 1.0617344379425049, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.343482494354248, "sampling/importance_sampling_ratio/mean": 0.9807530641555786, "sampling/importance_sampling_ratio/min": 0.34026026725769043, "sampling/sampling_logp_difference/max": 1.0774974822998047, "sampling/sampling_logp_difference/mean": 0.03718913346529007, "step": 562, "step_time": 12.014542946992151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 93.3125, "completions/mean_terminated_length": 93.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.060920970630832016, "epoch": 0.01126, "frac_reward_zero_std": 0.0, "grad_norm": 0.08256185054779053, "kl": 1.158499464392662, "learning_rate": 4.98806571684751e-06, "loss": -0.0001, "num_tokens": 25799433.0, "reward": -0.11687500774860382, "reward_std": 1.0259160995483398, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.10491943359375, "sampling/importance_sampling_ratio/mean": 1.0000888109207153, "sampling/importance_sampling_ratio/min": 0.7689727544784546, "sampling/sampling_logp_difference/max": 0.2627435326576233, "sampling/sampling_logp_difference/mean": 0.009474317543208599, "step": 563, "step_time": 14.249762711013318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 53.40625, "completions/mean_terminated_length": 53.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08868486905703321, "epoch": 0.01128, "frac_reward_zero_std": 0.0, "grad_norm": 0.15540994703769684, "kl": 0.5201428849250078, "learning_rate": 4.975248772647969e-06, "loss": -0.004, "num_tokens": 25843692.0, "reward": -0.053125008940696716, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.5008801221847534, "sampling/importance_sampling_ratio/mean": 1.0335910320281982, "sampling/importance_sampling_ratio/min": 0.905432939529419, "sampling/sampling_logp_difference/max": 0.40383613109588623, "sampling/sampling_logp_difference/mean": 0.01853346638381481, "step": 564, "step_time": 11.302404966998438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 62.40625, "completions/mean_terminated_length": 62.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09836052590981126, "epoch": 0.0113, "frac_reward_zero_std": 0.0, "grad_norm": 0.11752650886774063, "kl": 0.9462688621133566, "learning_rate": 4.962448643278852e-06, "loss": -0.0171, "num_tokens": 25888786.0, "reward": -0.17937500774860382, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.210747241973877, "sampling/importance_sampling_ratio/mean": 1.0005534887313843, "sampling/importance_sampling_ratio/min": 0.5562036633491516, "sampling/sampling_logp_difference/max": 0.5864911079406738, "sampling/sampling_logp_difference/mean": 0.017515743151307106, "step": 565, "step_time": 13.40962316099467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 84.59375, "completions/mean_terminated_length": 84.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08590234932489693, "epoch": 0.01132, "frac_reward_zero_std": 0.0, "grad_norm": 0.2348526567220688, "kl": 1.090903026983142, "learning_rate": 4.949665497582519e-06, "loss": -0.0066, "num_tokens": 25935297.0, "reward": -0.30687499046325684, "reward_std": 1.0161497592926025, "rewards/rollout_reward_func/mean": -0.30687499046325684, "rewards/rollout_reward_func/std": 0.9730378985404968, "sampling/importance_sampling_ratio/max": 1.0520943403244019, "sampling/importance_sampling_ratio/mean": 0.9749699831008911, "sampling/importance_sampling_ratio/min": 0.5104434490203857, "sampling/sampling_logp_difference/max": 0.6723227500915527, "sampling/sampling_logp_difference/mean": 0.02226971462368965, "step": 566, "step_time": 11.812646430014865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1535.0, "completions/max_terminated_length": 1535.0, "completions/mean_length": 129.40625, "completions/mean_terminated_length": 129.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07340411003679037, "epoch": 0.01134, "frac_reward_zero_std": 0.0, "grad_norm": 0.44498786330223083, "kl": 0.6392688788473606, "learning_rate": 4.936899504177303e-06, "loss": -0.0214, "num_tokens": 25982894.0, "reward": -0.05437500774860382, "reward_std": 1.0526506900787354, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228137969970703, "sampling/importance_sampling_ratio/max": 1.0564043521881104, "sampling/importance_sampling_ratio/mean": 0.9798837900161743, "sampling/importance_sampling_ratio/min": 0.39828789234161377, "sampling/sampling_logp_difference/max": 0.920517086982727, "sampling/sampling_logp_difference/mean": 0.02497684210538864, "step": 567, "step_time": 15.577431094985513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 86.5625, "completions/mean_terminated_length": 86.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10795261501334608, "epoch": 0.01136, "frac_reward_zero_std": 0.0, "grad_norm": 0.29863300919532776, "kl": 0.7040136158466339, "learning_rate": 4.9241508314552856e-06, "loss": -0.0042, "num_tokens": 26028868.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.3076155185699463, "sampling/importance_sampling_ratio/mean": 1.0090851783752441, "sampling/importance_sampling_ratio/min": 0.47937828302383423, "sampling/sampling_logp_difference/max": 0.7353241443634033, "sampling/sampling_logp_difference/mean": 0.019444050267338753, "step": 568, "step_time": 11.51693764099764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 123.34375, "completions/mean_terminated_length": 123.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.061616319871973246, "epoch": 0.01138, "frac_reward_zero_std": 0.0, "grad_norm": 0.18944796919822693, "kl": 0.6191706582903862, "learning_rate": 4.911419647580079e-06, "loss": -0.0059, "num_tokens": 26077070.0, "reward": 0.07312498986721039, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 2.250889301300049, "sampling/importance_sampling_ratio/mean": 1.0646569728851318, "sampling/importance_sampling_ratio/min": 1.000882863998413, "sampling/sampling_logp_difference/max": 0.7984527349472046, "sampling/sampling_logp_difference/mean": 0.018466953188180923, "step": 569, "step_time": 14.476704320019053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 56.5, "completions/mean_terminated_length": 56.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06417849287390709, "epoch": 0.0114, "frac_reward_zero_std": 0.0, "grad_norm": 0.1311764419078827, "kl": 0.9895175416022539, "learning_rate": 4.898706120484606e-06, "loss": -0.0052, "num_tokens": 26121132.0, "reward": 0.1356249898672104, "reward_std": 1.0259160995483398, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.4353594779968262, "sampling/importance_sampling_ratio/mean": 1.017707347869873, "sampling/importance_sampling_ratio/min": 0.7842482924461365, "sampling/sampling_logp_difference/max": 0.3614082932472229, "sampling/sampling_logp_difference/mean": 0.013147793710231781, "step": 570, "step_time": 11.911349841000629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 114.78125, "completions/mean_terminated_length": 114.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07568623474799097, "epoch": 0.01142, "frac_reward_zero_std": 0.0, "grad_norm": 0.13444505631923676, "kl": 0.5878918338567019, "learning_rate": 4.886010417868881e-06, "loss": -0.0158, "num_tokens": 26168717.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.3531196117401123, "sampling/importance_sampling_ratio/mean": 1.018056869506836, "sampling/importance_sampling_ratio/min": 0.6354138851165771, "sampling/sampling_logp_difference/max": 0.4533785879611969, "sampling/sampling_logp_difference/mean": 0.016286097466945648, "step": 571, "step_time": 11.604901255988807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 58.875, "completions/mean_terminated_length": 58.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05993982299696654, "epoch": 0.01144, "frac_reward_zero_std": 0.0, "grad_norm": 0.37100115418434143, "kl": 0.5503669045865536, "learning_rate": 4.873332707197804e-06, "loss": -0.0054, "num_tokens": 26212652.0, "reward": 0.13499999046325684, "reward_std": 1.0250985622406006, "rewards/rollout_reward_func/mean": 0.13499999046325684, "rewards/rollout_reward_func/std": 1.0170037746429443, "sampling/importance_sampling_ratio/max": 1.1003799438476562, "sampling/importance_sampling_ratio/mean": 1.012576937675476, "sampling/importance_sampling_ratio/min": 1.0008344650268555, "sampling/sampling_logp_difference/max": 0.09590339660644531, "sampling/sampling_logp_difference/mean": 0.005311944056302309, "step": 572, "step_time": 11.185531142007676 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 402.0, "completions/max_terminated_length": 402.0, "completions/mean_length": 45.09375, "completions/mean_terminated_length": 45.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0721192317432724, "epoch": 0.01146, "frac_reward_zero_std": 0.0, "grad_norm": 0.6464943885803223, "kl": 0.8511787801980972, "learning_rate": 4.860673155698953e-06, "loss": -0.0112, "num_tokens": 26256187.0, "reward": 0.1993749886751175, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.243135929107666, "sampling/importance_sampling_ratio/mean": 1.0496935844421387, "sampling/importance_sampling_ratio/min": 0.5264409780502319, "sampling/sampling_logp_difference/max": 0.8080425262451172, "sampling/sampling_logp_difference/mean": 0.029065236449241638, "step": 573, "step_time": 11.062137871995219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 114.375, "completions/mean_terminated_length": 114.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06072381522972137, "epoch": 0.01148, "frac_reward_zero_std": 0.0, "grad_norm": 0.08162599056959152, "kl": 0.8247157018631697, "learning_rate": 4.848031930360365e-06, "loss": -0.0052, "num_tokens": 26303314.0, "reward": 0.07249999046325684, "reward_std": 1.01747727394104, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.0722737312316895, "sampling/importance_sampling_ratio/mean": 0.9988789558410645, "sampling/importance_sampling_ratio/min": 0.6130412220954895, "sampling/sampling_logp_difference/max": 0.4903082847595215, "sampling/sampling_logp_difference/mean": 0.010252648033201694, "step": 574, "step_time": 14.167790209990926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 45.40625, "completions/mean_terminated_length": 45.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07660842419136316, "epoch": 0.0115, "frac_reward_zero_std": 0.0, "grad_norm": 0.12980587780475616, "kl": 0.6304411888122559, "learning_rate": 4.835409197928351e-06, "loss": 0.006, "num_tokens": 26346715.0, "reward": 0.009999990463256836, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.3726791143417358, "sampling/importance_sampling_ratio/mean": 1.0200397968292236, "sampling/importance_sampling_ratio/min": 0.9387426376342773, "sampling/sampling_logp_difference/max": 0.31730037927627563, "sampling/sampling_logp_difference/mean": 0.00966724380850792, "step": 575, "step_time": 14.154145001011784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 69.28125, "completions/mean_terminated_length": 69.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09430272015742958, "epoch": 0.01152, "frac_reward_zero_std": 0.0, "grad_norm": 0.11517735570669174, "kl": 0.5992597956210375, "learning_rate": 4.822805124905282e-06, "loss": -0.0119, "num_tokens": 26391861.0, "reward": 0.19687499105930328, "reward_std": 1.0157405138015747, "rewards/rollout_reward_func/mean": 0.19687499105930328, "rewards/rollout_reward_func/std": 1.0058937072753906, "sampling/importance_sampling_ratio/max": 1.8066056966781616, "sampling/importance_sampling_ratio/mean": 1.0161223411560059, "sampling/importance_sampling_ratio/min": 0.44770726561546326, "sampling/sampling_logp_difference/max": 0.8035461902618408, "sampling/sampling_logp_difference/mean": 0.03019648790359497, "step": 576, "step_time": 11.698479793012666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 77.0, "completions/mean_terminated_length": 77.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0754314084770158, "epoch": 0.01154, "frac_reward_zero_std": 0.0, "grad_norm": 0.12263064831495285, "kl": 0.8327435571700335, "learning_rate": 4.810219877547406e-06, "loss": -0.0249, "num_tokens": 26437764.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.406607985496521, "sampling/importance_sampling_ratio/mean": 1.011328935623169, "sampling/importance_sampling_ratio/min": 0.48188939690589905, "sampling/sampling_logp_difference/max": 0.7338337898254395, "sampling/sampling_logp_difference/mean": 0.01960848458111286, "step": 577, "step_time": 13.859779340004025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 192.21875, "completions/mean_terminated_length": 192.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07764509989647195, "epoch": 0.01156, "frac_reward_zero_std": 0.0, "grad_norm": 0.48019251227378845, "kl": 0.7252974640578032, "learning_rate": 4.797653621862637e-06, "loss": 0.0039, "num_tokens": 26487955.0, "reward": -0.17937500774860382, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.1061307191848755, "sampling/importance_sampling_ratio/mean": 1.0076065063476562, "sampling/importance_sampling_ratio/min": 0.8839288949966431, "sampling/sampling_logp_difference/max": 0.4539000988006592, "sampling/sampling_logp_difference/mean": 0.014538224786520004, "step": 578, "step_time": 14.690803861005406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 95.6875, "completions/mean_terminated_length": 95.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09771455626469105, "epoch": 0.01158, "frac_reward_zero_std": 0.0, "grad_norm": 0.10655543208122253, "kl": 0.8755654748529196, "learning_rate": 4.785106523608388e-06, "loss": -0.0061, "num_tokens": 26534889.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.087059736251831, "sampling/importance_sampling_ratio/mean": 0.9916887879371643, "sampling/importance_sampling_ratio/min": 0.3789786994457245, "sampling/sampling_logp_difference/max": 0.9702460765838623, "sampling/sampling_logp_difference/mean": 0.020518874749541283, "step": 579, "step_time": 11.884171353020065 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 77.15625, "completions/mean_terminated_length": 77.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1130868437467143, "epoch": 0.0116, "frac_reward_zero_std": 0.0, "grad_norm": 0.32657647132873535, "kl": 0.8007909059524536, "learning_rate": 4.7725787482893645e-06, "loss": -0.0081, "num_tokens": 26581243.0, "reward": -0.17937500774860382, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.8102750778198242, "sampling/importance_sampling_ratio/mean": 1.0261037349700928, "sampling/importance_sampling_ratio/min": 0.24484260380268097, "sampling/sampling_logp_difference/max": 1.4417729377746582, "sampling/sampling_logp_difference/mean": 0.03942514955997467, "step": 580, "step_time": 14.217541239020647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 101.71875, "completions/mean_terminated_length": 101.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12663862080080435, "epoch": 0.01162, "frac_reward_zero_std": 0.0, "grad_norm": 0.2633477747440338, "kl": 1.013329828158021, "learning_rate": 4.760070461155393e-06, "loss": -0.031, "num_tokens": 26629450.0, "reward": -0.3056250214576721, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.3798282146453857, "sampling/importance_sampling_ratio/mean": 0.9607820510864258, "sampling/importance_sampling_ratio/min": 0.42614200711250305, "sampling/sampling_logp_difference/max": 0.852881908416748, "sampling/sampling_logp_difference/mean": 0.041857652366161346, "step": 581, "step_time": 14.431586186008644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 36.40625, "completions/mean_terminated_length": 36.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09387948055518791, "epoch": 0.01164, "frac_reward_zero_std": 0.0, "grad_norm": 0.09144340455532074, "kl": 1.0699041541665792, "learning_rate": 4.7475818271992335e-06, "loss": -0.0067, "num_tokens": 26674478.0, "reward": -0.24312500655651093, "reward_std": 1.0257649421691895, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.3743350505828857, "sampling/importance_sampling_ratio/mean": 0.9947389364242554, "sampling/importance_sampling_ratio/min": 0.4202808737754822, "sampling/sampling_logp_difference/max": 0.8630776405334473, "sampling/sampling_logp_difference/mean": 0.025150451809167862, "step": 582, "step_time": 11.48707807900064 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 59.15625, "completions/mean_terminated_length": 59.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10285429039504379, "epoch": 0.01166, "frac_reward_zero_std": 0.0, "grad_norm": 0.10627772659063339, "kl": 1.3444875236600637, "learning_rate": 4.735113011154413e-06, "loss": -0.0167, "num_tokens": 26721323.0, "reward": -0.3056250214576721, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.118252158164978, "sampling/importance_sampling_ratio/mean": 0.9981601238250732, "sampling/importance_sampling_ratio/min": 0.6925393342971802, "sampling/sampling_logp_difference/max": 0.3674149513244629, "sampling/sampling_logp_difference/mean": 0.01569504849612713, "step": 583, "step_time": 11.882840136007871 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 155.0, "completions/mean_terminated_length": 155.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1262463852763176, "epoch": 0.01168, "frac_reward_zero_std": 0.25, "grad_norm": 0.22200287878513336, "kl": 1.225284568965435, "learning_rate": 4.722664177493042e-06, "loss": 0.0004, "num_tokens": 26770353.0, "reward": -0.3687500059604645, "reward_std": 0.7564942836761475, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.2864943742752075, "sampling/importance_sampling_ratio/mean": 0.9399862885475159, "sampling/importance_sampling_ratio/min": 0.30920276045799255, "sampling/sampling_logp_difference/max": 1.1865469217300415, "sampling/sampling_logp_difference/mean": 0.05076811835169792, "step": 584, "step_time": 14.886956168993493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 80.34375, "completions/mean_terminated_length": 80.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1196078626671806, "epoch": 0.0117, "frac_reward_zero_std": 0.25, "grad_norm": 0.1323367804288864, "kl": 0.961980851367116, "learning_rate": 4.710235490423655e-06, "loss": -0.0156, "num_tokens": 26818219.0, "reward": -0.17937502264976501, "reward_std": 0.784087061882019, "rewards/rollout_reward_func/mean": -0.17937502264976501, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.3393611907958984, "sampling/importance_sampling_ratio/mean": 1.0051767826080322, "sampling/importance_sampling_ratio/min": 0.6484857201576233, "sampling/sampling_logp_difference/max": 0.4463663101196289, "sampling/sampling_logp_difference/mean": 0.01835249550640583, "step": 585, "step_time": 11.706978058005916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 27.5625, "completions/mean_terminated_length": 27.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08106366044376045, "epoch": 0.01172, "frac_reward_zero_std": 0.0, "grad_norm": 0.10017642378807068, "kl": 1.1015009433031082, "learning_rate": 4.697827113889034e-06, "loss": 0.0037, "num_tokens": 26861039.0, "reward": 0.07312498986721039, "reward_std": 1.0711638927459717, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.1982156038284302, "sampling/importance_sampling_ratio/mean": 1.0128297805786133, "sampling/importance_sampling_ratio/min": 0.895507276058197, "sampling/sampling_logp_difference/max": 0.18077901005744934, "sampling/sampling_logp_difference/mean": 0.010102875530719757, "step": 586, "step_time": 11.10815337299573 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 92.03125, "completions/mean_terminated_length": 92.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09410964255221188, "epoch": 0.01174, "frac_reward_zero_std": 0.0, "grad_norm": 0.07164983451366425, "kl": 0.5760734342038631, "learning_rate": 4.685439211564055e-06, "loss": 0.0002, "num_tokens": 26907365.0, "reward": -0.053125008940696716, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0995584726333618, "sampling/importance_sampling_ratio/mean": 0.9977320432662964, "sampling/importance_sampling_ratio/min": 0.4745931625366211, "sampling/sampling_logp_difference/max": 0.7453477382659912, "sampling/sampling_logp_difference/mean": 0.0172455795109272, "step": 587, "step_time": 12.186213094981213 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 92.3125, "completions/mean_terminated_length": 92.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1094304890721105, "epoch": 0.01176, "frac_reward_zero_std": 0.0, "grad_norm": 0.3478833734989166, "kl": 1.0077988244593143, "learning_rate": 4.67307194685352e-06, "loss": -0.0043, "num_tokens": 26953375.0, "reward": 0.009374991059303284, "reward_std": 1.0252747535705566, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.9280236959457397, "sampling/importance_sampling_ratio/mean": 1.008674144744873, "sampling/importance_sampling_ratio/min": 0.266512930393219, "sampling/sampling_logp_difference/max": 1.3344765901565552, "sampling/sampling_logp_difference/mean": 0.0318903923034668, "step": 588, "step_time": 13.72294918998523 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 856.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 106.21875, "completions/mean_terminated_length": 106.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12572555360384285, "epoch": 0.01178, "frac_reward_zero_std": 0.0, "grad_norm": 0.15686631202697754, "kl": 1.032922375947237, "learning_rate": 4.660725482890016e-06, "loss": -0.0327, "num_tokens": 27000399.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.493194818496704, "sampling/importance_sampling_ratio/mean": 1.0337750911712646, "sampling/importance_sampling_ratio/min": 0.5107699036598206, "sampling/sampling_logp_difference/max": 0.6718723177909851, "sampling/sampling_logp_difference/mean": 0.027603408321738243, "step": 589, "step_time": 12.902701006001735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 79.9375, "completions/mean_terminated_length": 79.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12035390606615692, "epoch": 0.0118, "frac_reward_zero_std": 0.0, "grad_norm": 0.19405993819236755, "kl": 0.8006457630544901, "learning_rate": 4.648399982531745e-06, "loss": -0.0294, "num_tokens": 27047112.0, "reward": 0.07249999046325684, "reward_std": 0.9983232021331787, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.6259294748306274, "sampling/importance_sampling_ratio/mean": 1.0180643796920776, "sampling/importance_sampling_ratio/min": 0.6824719309806824, "sampling/sampling_logp_difference/max": 0.47203201055526733, "sampling/sampling_logp_difference/mean": 0.023156922310590744, "step": 590, "step_time": 13.469036429996777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 52.28125, "completions/mean_terminated_length": 52.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1126758421305567, "epoch": 0.01182, "frac_reward_zero_std": 0.0, "grad_norm": 0.15235240757465363, "kl": 0.9211512617766857, "learning_rate": 4.636095608360393e-06, "loss": 0.0024, "num_tokens": 27091464.0, "reward": -0.17937500774860382, "reward_std": 0.9160174131393433, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.0079617500305176, "sampling/importance_sampling_ratio/max": 1.2915494441986084, "sampling/importance_sampling_ratio/mean": 1.0016255378723145, "sampling/importance_sampling_ratio/min": 0.4811961352825165, "sampling/sampling_logp_difference/max": 0.7337241172790527, "sampling/sampling_logp_difference/mean": 0.024512924253940582, "step": 591, "step_time": 11.234141375003674 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 61.5625, "completions/mean_terminated_length": 61.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12831789604388177, "epoch": 0.01184, "frac_reward_zero_std": 0.0, "grad_norm": 0.1323562115430832, "kl": 1.2959853149950504, "learning_rate": 4.623812522678968e-06, "loss": -0.0061, "num_tokens": 27137490.0, "reward": -0.36937499046325684, "reward_std": 0.9062925577163696, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.0723497867584229, "sampling/importance_sampling_ratio/mean": 0.9775223731994629, "sampling/importance_sampling_ratio/min": 0.6765545606613159, "sampling/sampling_logp_difference/max": 0.4047935903072357, "sampling/sampling_logp_difference/mean": 0.023123573511838913, "step": 592, "step_time": 11.901125660981052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 483.0, "completions/max_terminated_length": 483.0, "completions/mean_length": 30.65625, "completions/mean_terminated_length": 30.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13396913663018495, "epoch": 0.01186, "frac_reward_zero_std": 0.0, "grad_norm": 0.137761190533638, "kl": 0.707866732031107, "learning_rate": 4.611550887509677e-06, "loss": -0.0085, "num_tokens": 27181524.0, "reward": -0.18000000715255737, "reward_std": 1.053357720375061, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.3993937969207764, "sampling/importance_sampling_ratio/mean": 0.9984247088432312, "sampling/importance_sampling_ratio/min": 0.5538579821586609, "sampling/sampling_logp_difference/max": 0.5908617973327637, "sampling/sampling_logp_difference/mean": 0.030250592157244682, "step": 593, "step_time": 10.85884351599816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 115.90625, "completions/mean_terminated_length": 115.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14425523584941402, "epoch": 0.01188, "frac_reward_zero_std": 0.0, "grad_norm": 0.3988918662071228, "kl": 0.8031718246638775, "learning_rate": 4.599310864591776e-06, "loss": -0.0398, "num_tokens": 27230095.0, "reward": -0.24312500655651093, "reward_std": 0.942456841468811, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.285235047340393, "sampling/importance_sampling_ratio/mean": 1.0237741470336914, "sampling/importance_sampling_ratio/min": 0.7057235836982727, "sampling/sampling_logp_difference/max": 0.3482537269592285, "sampling/sampling_logp_difference/mean": 0.01674959808588028, "step": 594, "step_time": 13.90327647702361 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 69.96875, "completions/mean_terminated_length": 69.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11787585727870464, "epoch": 0.0119, "frac_reward_zero_std": 0.0, "grad_norm": 0.1086195856332779, "kl": 0.6399760339409113, "learning_rate": 4.587092615379442e-06, "loss": -0.0054, "num_tokens": 27275332.0, "reward": -0.053125008940696716, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.1620253324508667, "sampling/importance_sampling_ratio/mean": 1.005674123764038, "sampling/importance_sampling_ratio/min": 0.6861900687217712, "sampling/sampling_logp_difference/max": 0.37655723094940186, "sampling/sampling_logp_difference/mean": 0.012780481949448586, "step": 595, "step_time": 11.352337495991378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 111.28125, "completions/mean_terminated_length": 111.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13094393047504127, "epoch": 0.01192, "frac_reward_zero_std": 0.0, "grad_norm": 0.28010106086730957, "kl": 1.045478107407689, "learning_rate": 4.574896301039641e-06, "loss": -0.0243, "num_tokens": 27322958.0, "reward": -0.053125008940696716, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.4970943927764893, "sampling/importance_sampling_ratio/mean": 1.0282981395721436, "sampling/importance_sampling_ratio/min": 0.5471473336219788, "sampling/sampling_logp_difference/max": 0.6058554649353027, "sampling/sampling_logp_difference/mean": 0.0286891870200634, "step": 596, "step_time": 11.787342742012697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 43.15625, "completions/mean_terminated_length": 43.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1463239078875631, "epoch": 0.01194, "frac_reward_zero_std": 0.0, "grad_norm": 0.22704799473285675, "kl": 0.9620367344468832, "learning_rate": 4.562722082450004e-06, "loss": -0.009, "num_tokens": 27368248.0, "reward": -0.24312500655651093, "reward_std": 0.9791113138198853, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.3854864835739136, "sampling/importance_sampling_ratio/mean": 1.0082118511199951, "sampling/importance_sampling_ratio/min": 0.5428876876831055, "sampling/sampling_logp_difference/max": 0.6125597953796387, "sampling/sampling_logp_difference/mean": 0.025153590366244316, "step": 597, "step_time": 11.201812929975858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 65.5, "completions/mean_terminated_length": 65.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13892251392826438, "epoch": 0.01196, "frac_reward_zero_std": 0.0, "grad_norm": 0.0983496829867363, "kl": 1.3090643864125013, "learning_rate": 4.550570120196702e-06, "loss": -0.0011, "num_tokens": 27413996.0, "reward": -0.24250000715255737, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.1373027563095093, "sampling/importance_sampling_ratio/mean": 1.0103203058242798, "sampling/importance_sampling_ratio/min": 0.6192297339439392, "sampling/sampling_logp_difference/max": 0.4793229103088379, "sampling/sampling_logp_difference/mean": 0.016440972685813904, "step": 598, "step_time": 11.827423692004231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 40.90625, "completions/mean_terminated_length": 40.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13272136740852147, "epoch": 0.01198, "frac_reward_zero_std": 0.0, "grad_norm": 0.23271946609020233, "kl": 1.012389536947012, "learning_rate": 4.538440574572337e-06, "loss": -0.0075, "num_tokens": 27456937.0, "reward": 0.19874998927116394, "reward_std": 1.01747727394104, "rewards/rollout_reward_func/mean": 0.19874998927116394, "rewards/rollout_reward_func/std": 1.0074424743652344, "sampling/importance_sampling_ratio/max": 1.1093446016311646, "sampling/importance_sampling_ratio/mean": 0.9842174053192139, "sampling/importance_sampling_ratio/min": 0.5105825066566467, "sampling/sampling_logp_difference/max": 0.6720247268676758, "sampling/sampling_logp_difference/mean": 0.02420365810394287, "step": 599, "step_time": 10.825065675002406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 63.375, "completions/mean_terminated_length": 63.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11198003991739824, "epoch": 0.012, "frac_reward_zero_std": 0.0, "grad_norm": 0.3744818866252899, "kl": 1.161890265531838, "learning_rate": 4.5263336055738115e-06, "loss": -0.0097, "num_tokens": 27501449.0, "reward": 0.13624998927116394, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.9392236471176147, "sampling/importance_sampling_ratio/mean": 0.9959464073181152, "sampling/importance_sampling_ratio/min": 0.5113499164581299, "sampling/sampling_logp_difference/max": 0.6703391075134277, "sampling/sampling_logp_difference/mean": 0.03811439126729965, "step": 600, "step_time": 14.217737584986025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 483.0, "completions/max_terminated_length": 483.0, "completions/mean_length": 43.28125, "completions/mean_terminated_length": 43.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14531105023343116, "epoch": 0.01202, "frac_reward_zero_std": 0.0, "grad_norm": 0.13446709513664246, "kl": 0.6145585030317307, "learning_rate": 4.5142493729002316e-06, "loss": -0.0179, "num_tokens": 27547639.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.0887748003005981, "sampling/importance_sampling_ratio/mean": 1.013608694076538, "sampling/importance_sampling_ratio/min": 0.778660237789154, "sampling/sampling_logp_difference/max": 0.25024378299713135, "sampling/sampling_logp_difference/mean": 0.012466039508581161, "step": 601, "step_time": 11.268537750976975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 802.0, "completions/max_terminated_length": 802.0, "completions/mean_length": 59.59375, "completions/mean_terminated_length": 59.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12926704646088183, "epoch": 0.01204, "frac_reward_zero_std": 0.0, "grad_norm": 0.13873465359210968, "kl": 0.6720793917775154, "learning_rate": 4.502188035950794e-06, "loss": -0.008, "num_tokens": 27593326.0, "reward": -0.17937500774860382, "reward_std": 0.9883459806442261, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.1227467060089111, "sampling/importance_sampling_ratio/mean": 1.00711989402771, "sampling/importance_sampling_ratio/min": 0.7001131176948547, "sampling/sampling_logp_difference/max": 0.3563652038574219, "sampling/sampling_logp_difference/mean": 0.020220253616571426, "step": 602, "step_time": 11.944605406002665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.0, "completions/max_terminated_length": 294.0, "completions/mean_length": 19.125, "completions/mean_terminated_length": 19.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15605287515791133, "epoch": 0.01206, "frac_reward_zero_std": 0.0, "grad_norm": 0.1511034518480301, "kl": 0.626587837934494, "learning_rate": 4.490149753822689e-06, "loss": -0.0154, "num_tokens": 27637961.0, "reward": -0.36937499046325684, "reward_std": 0.9894058108329773, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.2261996269226074, "sampling/importance_sampling_ratio/mean": 0.9824404120445251, "sampling/importance_sampling_ratio/min": 0.4450828731060028, "sampling/sampling_logp_difference/max": 0.8094010353088379, "sampling/sampling_logp_difference/mean": 0.03881000354886055, "step": 603, "step_time": 10.735029681003653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 114.09375, "completions/mean_terminated_length": 114.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11700316151836887, "epoch": 0.01208, "frac_reward_zero_std": 0.0, "grad_norm": 0.3872441053390503, "kl": 0.5128823649138212, "learning_rate": 4.478134685308997e-06, "loss": -0.0183, "num_tokens": 27684519.0, "reward": 0.26124998927116394, "reward_std": 1.0254011154174805, "rewards/rollout_reward_func/mean": 0.26124998927116394, "rewards/rollout_reward_func/std": 0.9926040768623352, "sampling/importance_sampling_ratio/max": 1.1708825826644897, "sampling/importance_sampling_ratio/mean": 1.0129650831222534, "sampling/importance_sampling_ratio/min": 0.8352982997894287, "sampling/sampling_logp_difference/max": 0.15432357788085938, "sampling/sampling_logp_difference/mean": 0.009908640757203102, "step": 604, "step_time": 13.47833666799852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 106.65625, "completions/mean_terminated_length": 106.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.135073360404931, "epoch": 0.0121, "frac_reward_zero_std": 0.0, "grad_norm": 0.3814758062362671, "kl": 1.2185408808290958, "learning_rate": 4.4661429888965894e-06, "loss": -0.0161, "num_tokens": 27730633.0, "reward": -0.05375000834465027, "reward_std": 1.0167031288146973, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.5688036680221558, "sampling/importance_sampling_ratio/mean": 0.9956215620040894, "sampling/importance_sampling_ratio/min": 0.5081806182861328, "sampling/sampling_logp_difference/max": 0.6926870942115784, "sampling/sampling_logp_difference/mean": 0.04303973168134689, "step": 605, "step_time": 13.095936568985053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 55.25, "completions/mean_terminated_length": 55.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09743727097520605, "epoch": 0.01212, "frac_reward_zero_std": 0.0, "grad_norm": 0.3106207251548767, "kl": 0.5614286307245493, "learning_rate": 4.454174822764049e-06, "loss": 0.0067, "num_tokens": 27773925.0, "reward": 0.07312498986721039, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0904686450958252, "sampling/importance_sampling_ratio/mean": 1.004775047302246, "sampling/importance_sampling_ratio/min": 0.7181035280227661, "sampling/sampling_logp_difference/max": 0.3309764862060547, "sampling/sampling_logp_difference/mean": 0.010731739923357964, "step": 606, "step_time": 14.647352867978043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 438.0, "completions/max_terminated_length": 438.0, "completions/mean_length": 30.78125, "completions/mean_terminated_length": 30.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1422928385436535, "epoch": 0.01214, "frac_reward_zero_std": 0.0, "grad_norm": 0.07509998977184296, "kl": 0.7071763128042221, "learning_rate": 4.4422303447795816e-06, "loss": -0.0126, "num_tokens": 27819703.0, "reward": -0.3056250214576721, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.0436484813690186, "sampling/importance_sampling_ratio/mean": 1.0195376873016357, "sampling/importance_sampling_ratio/min": 1.0012952089309692, "sampling/sampling_logp_difference/max": 0.04262349009513855, "sampling/sampling_logp_difference/mean": 0.008839379996061325, "step": 607, "step_time": 10.611718881991692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1261.0, "completions/max_terminated_length": 1261.0, "completions/mean_length": 128.875, "completions/mean_terminated_length": 128.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14944694272708148, "epoch": 0.01216, "frac_reward_zero_std": 0.0, "grad_norm": 0.5179942846298218, "kl": 0.5896652620285749, "learning_rate": 4.430309712498924e-06, "loss": 0.0244, "num_tokens": 27867365.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 2.3984336853027344, "sampling/importance_sampling_ratio/mean": 1.0463635921478271, "sampling/importance_sampling_ratio/min": 0.5946359038352966, "sampling/sampling_logp_difference/max": 0.8429486751556396, "sampling/sampling_logp_difference/mean": 0.03403257578611374, "step": 608, "step_time": 14.735686542997428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 24.65625, "completions/mean_terminated_length": 24.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18875614373246208, "epoch": 0.01218, "frac_reward_zero_std": 0.0, "grad_norm": 0.20371343195438385, "kl": 0.5668531786650419, "learning_rate": 4.418413083163273e-06, "loss": -0.0138, "num_tokens": 27910340.0, "reward": 0.07312498986721039, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.0848791599273682, "sampling/importance_sampling_ratio/mean": 0.992321252822876, "sampling/importance_sampling_ratio/min": 0.44927704334259033, "sampling/sampling_logp_difference/max": 0.7999725341796875, "sampling/sampling_logp_difference/mean": 0.026079978793859482, "step": 609, "step_time": 11.142179170012241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 77.71875, "completions/mean_terminated_length": 77.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16683812206611037, "epoch": 0.0122, "frac_reward_zero_std": 0.0, "grad_norm": 0.24052008986473083, "kl": 1.0526088327169418, "learning_rate": 4.406540613697213e-06, "loss": -0.0125, "num_tokens": 27957239.0, "reward": -0.3056250214576721, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.202890157699585, "sampling/importance_sampling_ratio/mean": 0.9873532056808472, "sampling/importance_sampling_ratio/min": 0.5493487119674683, "sampling/sampling_logp_difference/max": 0.5958676338195801, "sampling/sampling_logp_difference/mean": 0.03195439279079437, "step": 610, "step_time": 12.934907831971941 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 56.46875, "completions/mean_terminated_length": 56.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10657317511504516, "epoch": 0.01222, "frac_reward_zero_std": 0.0, "grad_norm": 0.10341405123472214, "kl": 1.0627641081809998, "learning_rate": 4.394692460706643e-06, "loss": -0.0096, "num_tokens": 28001599.0, "reward": 0.1981249898672104, "reward_std": 1.0164865255355835, "rewards/rollout_reward_func/mean": 0.1981249898672104, "rewards/rollout_reward_func/std": 1.0069226026535034, "sampling/importance_sampling_ratio/max": 1.0468628406524658, "sampling/importance_sampling_ratio/mean": 0.9878334403038025, "sampling/importance_sampling_ratio/min": 0.6666595935821533, "sampling/sampling_logp_difference/max": 0.4055347442626953, "sampling/sampling_logp_difference/mean": 0.018952751532197, "step": 611, "step_time": 11.095491167994624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 83.46875, "completions/mean_terminated_length": 83.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16243749530985951, "epoch": 0.01224, "frac_reward_zero_std": 0.0, "grad_norm": 0.4139536917209625, "kl": 1.0507881194353104, "learning_rate": 4.382868780476709e-06, "loss": -0.0175, "num_tokens": 28048375.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.1505231857299805, "sampling/importance_sampling_ratio/mean": 0.9820664525032043, "sampling/importance_sampling_ratio/min": 0.5510441064834595, "sampling/sampling_logp_difference/max": 0.5964446067810059, "sampling/sampling_logp_difference/mean": 0.029182162135839462, "step": 612, "step_time": 11.878762792017369 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 55.65625, "completions/mean_terminated_length": 55.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13789900275878608, "epoch": 0.01226, "frac_reward_zero_std": 0.0, "grad_norm": 0.3905765414237976, "kl": 0.8295777961611748, "learning_rate": 4.371069728969745e-06, "loss": -0.0046, "num_tokens": 28093171.0, "reward": -0.053125008940696716, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.2110954523086548, "sampling/importance_sampling_ratio/mean": 1.0026848316192627, "sampling/importance_sampling_ratio/min": 0.5514171719551086, "sampling/sampling_logp_difference/max": 0.5915870666503906, "sampling/sampling_logp_difference/mean": 0.023261941969394684, "step": 613, "step_time": 11.389540565010975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 820.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 55.09375, "completions/mean_terminated_length": 55.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10997498157666996, "epoch": 0.01228, "frac_reward_zero_std": 0.0, "grad_norm": 0.10596805810928345, "kl": 0.5779695454984903, "learning_rate": 4.359295461823221e-06, "loss": -0.0139, "num_tokens": 28136561.0, "reward": 0.009374991059303284, "reward_std": 1.0259159803390503, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.1427768468856812, "sampling/importance_sampling_ratio/mean": 0.996644139289856, "sampling/importance_sampling_ratio/min": 0.6255896091461182, "sampling/sampling_logp_difference/max": 0.46836555004119873, "sampling/sampling_logp_difference/mean": 0.015948282554745674, "step": 614, "step_time": 12.439730307982245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 116.625, "completions/mean_terminated_length": 116.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13461880397517234, "epoch": 0.0123, "frac_reward_zero_std": 0.0, "grad_norm": 0.22976955771446228, "kl": 0.8278344869613647, "learning_rate": 4.347546134347673e-06, "loss": 0.0079, "num_tokens": 28184301.0, "reward": -0.18000000715255737, "reward_std": 0.9706909656524658, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.8282828330993652, "sampling/importance_sampling_ratio/mean": 1.0381392240524292, "sampling/importance_sampling_ratio/min": 0.897543728351593, "sampling/sampling_logp_difference/max": 0.582314133644104, "sampling/sampling_logp_difference/mean": 0.015160472132265568, "step": 615, "step_time": 12.098103562995675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 65.34375, "completions/mean_terminated_length": 65.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1263985356199555, "epoch": 0.01232, "frac_reward_zero_std": 0.0, "grad_norm": 0.06557196378707886, "kl": 0.8173848204314709, "learning_rate": 4.33582190152468e-06, "loss": -0.0157, "num_tokens": 28230534.0, "reward": 0.008749991655349731, "reward_std": 1.0252528190612793, "rewards/rollout_reward_func/mean": 0.008749991655349731, "rewards/rollout_reward_func/std": 1.0249019861221313, "sampling/importance_sampling_ratio/max": 1.0723429918289185, "sampling/importance_sampling_ratio/mean": 1.0134530067443848, "sampling/importance_sampling_ratio/min": 0.8038418889045715, "sampling/sampling_logp_difference/max": 0.21817517280578613, "sampling/sampling_logp_difference/mean": 0.011197278276085854, "step": 616, "step_time": 10.802757996985747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 100.34375, "completions/mean_terminated_length": 100.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14608654443873093, "epoch": 0.01234, "frac_reward_zero_std": 0.0, "grad_norm": 0.1918897181749344, "kl": 0.9082058444619179, "learning_rate": 4.324122918004795e-06, "loss": -0.0018, "num_tokens": 28277049.0, "reward": -0.24250000715255737, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.1852598190307617, "sampling/importance_sampling_ratio/mean": 0.9883103966712952, "sampling/importance_sampling_ratio/min": 0.4978969693183899, "sampling/sampling_logp_difference/max": 0.6974115371704102, "sampling/sampling_logp_difference/mean": 0.027509266510605812, "step": 617, "step_time": 12.120600971000385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 67.03125, "completions/mean_terminated_length": 67.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13100550085073337, "epoch": 0.01236, "frac_reward_zero_std": 0.0, "grad_norm": 0.11952674388885498, "kl": 1.3062387183308601, "learning_rate": 4.3124493381055204e-06, "loss": -0.009, "num_tokens": 28322017.0, "reward": 0.07249997556209564, "reward_std": 0.9345266222953796, "rewards/rollout_reward_func/mean": 0.07249997556209564, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.5752376317977905, "sampling/importance_sampling_ratio/mean": 1.0264674425125122, "sampling/importance_sampling_ratio/min": 0.6250993013381958, "sampling/sampling_logp_difference/max": 0.4696938991546631, "sampling/sampling_logp_difference/mean": 0.02293183095753193, "step": 618, "step_time": 12.007515194993175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 160.46875, "completions/mean_terminated_length": 160.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14287982613313943, "epoch": 0.01238, "frac_reward_zero_std": 0.0, "grad_norm": 0.4876423180103302, "kl": 0.5590190812945366, "learning_rate": 4.300801315809264e-06, "loss": 0.0029, "num_tokens": 28370420.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.7786335945129395, "sampling/importance_sampling_ratio/mean": 1.026139497756958, "sampling/importance_sampling_ratio/min": 0.6796888113021851, "sampling/sampling_logp_difference/max": 0.5643721222877502, "sampling/sampling_logp_difference/mean": 0.02279030531644821, "step": 619, "step_time": 13.392608911002753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 49.1875, "completions/mean_terminated_length": 49.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09498039807658643, "epoch": 0.0124, "frac_reward_zero_std": 0.0, "grad_norm": 0.156129851937294, "kl": 0.74077376909554, "learning_rate": 4.289179004761318e-06, "loss": -0.0104, "num_tokens": 28414257.0, "reward": 0.1993749886751175, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.0387954711914062, "sampling/importance_sampling_ratio/mean": 1.0074107646942139, "sampling/importance_sampling_ratio/min": 0.7996578812599182, "sampling/sampling_logp_difference/max": 0.2234327793121338, "sampling/sampling_logp_difference/mean": 0.00897021684795618, "step": 620, "step_time": 11.184888059004152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 64.9375, "completions/mean_terminated_length": 64.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12163483549375087, "epoch": 0.01242, "frac_reward_zero_std": 0.0, "grad_norm": 0.12236274033784866, "kl": 0.5441413074731827, "learning_rate": 4.277582558267819e-06, "loss": 0.0025, "num_tokens": 28458687.0, "reward": 0.009374991059303284, "reward_std": 1.0790722370147705, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.092965841293335, "sampling/importance_sampling_ratio/mean": 1.0152209997177124, "sampling/importance_sampling_ratio/min": 0.9527847170829773, "sampling/sampling_logp_difference/max": 0.08974498510360718, "sampling/sampling_logp_difference/mean": 0.008484935387969017, "step": 621, "step_time": 14.298546063997492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 44.0, "completions/mean_terminated_length": 44.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12883361475542188, "epoch": 0.01244, "frac_reward_zero_std": 0.0, "grad_norm": 0.19366039335727692, "kl": 0.7506234720349312, "learning_rate": 4.266012129293736e-06, "loss": -0.006, "num_tokens": 28501974.0, "reward": 0.009999990463256836, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.3250452280044556, "sampling/importance_sampling_ratio/mean": 0.9886037111282349, "sampling/importance_sampling_ratio/min": 0.5515533685684204, "sampling/sampling_logp_difference/max": 0.5966048240661621, "sampling/sampling_logp_difference/mean": 0.03128908574581146, "step": 622, "step_time": 11.3381482120094 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 811.0, "completions/max_terminated_length": 811.0, "completions/mean_length": 77.40625, "completions/mean_terminated_length": 77.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12353482819162309, "epoch": 0.01246, "frac_reward_zero_std": 0.0, "grad_norm": 0.06653329730033875, "kl": 1.015518307685852, "learning_rate": 4.254467870460848e-06, "loss": -0.0139, "num_tokens": 28548215.0, "reward": 0.07249999046325684, "reward_std": 0.9345266222953796, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.2890784740447998, "sampling/importance_sampling_ratio/mean": 1.01719331741333, "sampling/importance_sampling_ratio/min": 0.7034249901771545, "sampling/sampling_logp_difference/max": 0.3515300750732422, "sampling/sampling_logp_difference/mean": 0.014676334336400032, "step": 623, "step_time": 12.38934523500211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 107.625, "completions/mean_terminated_length": 107.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11853664659429342, "epoch": 0.01248, "frac_reward_zero_std": 0.0, "grad_norm": 0.16750866174697876, "kl": 0.7601927481591702, "learning_rate": 4.2429499340457344e-06, "loss": -0.0098, "num_tokens": 28594517.0, "reward": 0.1356249898672104, "reward_std": 1.042907953262329, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.0382403135299683, "sampling/importance_sampling_ratio/mean": 0.9762675166130066, "sampling/importance_sampling_ratio/min": 0.2524718642234802, "sampling/sampling_logp_difference/max": 1.3762097358703613, "sampling/sampling_logp_difference/mean": 0.029091935604810715, "step": 624, "step_time": 13.57607118699525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 115.21875, "completions/mean_terminated_length": 115.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1371942552505061, "epoch": 0.0125, "frac_reward_zero_std": 0.0, "grad_norm": 0.4047044515609741, "kl": 1.0036449506878853, "learning_rate": 4.231458471977764e-06, "loss": 0.0035, "num_tokens": 28641976.0, "reward": -0.11687500774860382, "reward_std": 1.0252747535705566, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.636838436126709, "sampling/importance_sampling_ratio/mean": 1.0301053524017334, "sampling/importance_sampling_ratio/min": 0.4498448073863983, "sampling/sampling_logp_difference/max": 0.8591328263282776, "sampling/sampling_logp_difference/mean": 0.03684449940919876, "step": 625, "step_time": 12.05034523399081 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 52.84375, "completions/mean_terminated_length": 52.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1614636309677735, "epoch": 0.01252, "frac_reward_zero_std": 0.0, "grad_norm": 0.10000040382146835, "kl": 0.995142562314868, "learning_rate": 4.219993635837086e-06, "loss": -0.0058, "num_tokens": 28687257.0, "reward": -0.3687500059604645, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.1024667024612427, "sampling/importance_sampling_ratio/mean": 0.998736560344696, "sampling/importance_sampling_ratio/min": 0.5798836946487427, "sampling/sampling_logp_difference/max": 0.5440008640289307, "sampling/sampling_logp_difference/mean": 0.02322515845298767, "step": 626, "step_time": 11.356288078983198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 59.4375, "completions/mean_terminated_length": 59.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1038496351102367, "epoch": 0.01254, "frac_reward_zero_std": 0.0, "grad_norm": 0.06426869332790375, "kl": 0.5723085943609476, "learning_rate": 4.208555576852641e-06, "loss": -0.0094, "num_tokens": 28731490.0, "reward": 0.19874998927116394, "reward_std": 0.9156380891799927, "rewards/rollout_reward_func/mean": 0.19874998927116394, "rewards/rollout_reward_func/std": 1.0074424743652344, "sampling/importance_sampling_ratio/max": 1.1158803701400757, "sampling/importance_sampling_ratio/mean": 1.017940640449524, "sampling/importance_sampling_ratio/min": 1.001387357711792, "sampling/sampling_logp_difference/max": 0.10459771752357483, "sampling/sampling_logp_difference/mean": 0.007680006790906191, "step": 627, "step_time": 11.402772733003076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 103.40625, "completions/mean_terminated_length": 103.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1468039754545316, "epoch": 0.01256, "frac_reward_zero_std": 0.0, "grad_norm": 0.11634251475334167, "kl": 0.633569898083806, "learning_rate": 4.19714444590016e-06, "loss": -0.0067, "num_tokens": 28779187.0, "reward": -0.24312500655651093, "reward_std": 0.942456841468811, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.3505690097808838, "sampling/importance_sampling_ratio/mean": 1.0198228359222412, "sampling/importance_sampling_ratio/min": 0.7293250560760498, "sampling/sampling_logp_difference/max": 0.31545209884643555, "sampling/sampling_logp_difference/mean": 0.020611243322491646, "step": 628, "step_time": 11.569130013005633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 134.1875, "completions/mean_terminated_length": 134.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17050700122490525, "epoch": 0.01258, "frac_reward_zero_std": 0.0, "grad_norm": 0.13987834751605988, "kl": 1.365913711488247, "learning_rate": 4.185760393500174e-06, "loss": -0.026, "num_tokens": 28828800.0, "reward": -0.36937499046325684, "reward_std": 0.9894058108329773, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 1.1391547918319702, "sampling/importance_sampling_ratio/mean": 0.9936071038246155, "sampling/importance_sampling_ratio/min": 0.4247649013996124, "sampling/sampling_logp_difference/max": 0.851229190826416, "sampling/sampling_logp_difference/mean": 0.02396092750132084, "step": 629, "step_time": 12.12349701399944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1261.0, "completions/max_terminated_length": 1261.0, "completions/mean_length": 161.28125, "completions/mean_terminated_length": 161.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13193744560703635, "epoch": 0.0126, "frac_reward_zero_std": 0.0, "grad_norm": 0.4254908263683319, "kl": 0.927348930388689, "learning_rate": 4.17440356981603e-06, "loss": -0.0268, "num_tokens": 28877589.0, "reward": 0.26124995946884155, "reward_std": 0.9890264272689819, "rewards/rollout_reward_func/mean": 0.26124995946884155, "rewards/rollout_reward_func/std": 0.99260413646698, "sampling/importance_sampling_ratio/max": 1.5850200653076172, "sampling/importance_sampling_ratio/mean": 1.0145978927612305, "sampling/importance_sampling_ratio/min": 0.6278302669525146, "sampling/sampling_logp_difference/max": 0.46556854248046875, "sampling/sampling_logp_difference/mean": 0.019718514755368233, "step": 630, "step_time": 14.410524200007785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 90.46875, "completions/mean_terminated_length": 90.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1550169144757092, "epoch": 0.01262, "frac_reward_zero_std": 0.0, "grad_norm": 0.23352240025997162, "kl": 0.9743329081684351, "learning_rate": 4.163074124651908e-06, "loss": -0.0308, "num_tokens": 28925482.0, "reward": 0.009374991059303284, "reward_std": 1.0259160995483398, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.2586214542388916, "sampling/importance_sampling_ratio/mean": 1.0015138387680054, "sampling/importance_sampling_ratio/min": 0.5138646960258484, "sampling/sampling_logp_difference/max": 0.6656320095062256, "sampling/sampling_logp_difference/mean": 0.02114729769527912, "step": 631, "step_time": 11.477881823993812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 101.875, "completions/mean_terminated_length": 101.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18394029780756682, "epoch": 0.01264, "frac_reward_zero_std": 0.0, "grad_norm": 0.243626669049263, "kl": 1.197657397016883, "learning_rate": 4.15177220745085e-06, "loss": -0.0237, "num_tokens": 28973437.0, "reward": -0.24250000715255737, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.222476840019226, "sampling/importance_sampling_ratio/mean": 0.9664139747619629, "sampling/importance_sampling_ratio/min": 0.4898117780685425, "sampling/sampling_logp_difference/max": 0.7179827690124512, "sampling/sampling_logp_difference/mean": 0.044841453433036804, "step": 632, "step_time": 11.936304887007282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 53.4375, "completions/mean_terminated_length": 53.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.163618344347924, "epoch": 0.01266, "frac_reward_zero_std": 0.0, "grad_norm": 0.42963770031929016, "kl": 0.6146498434245586, "learning_rate": 4.140497967292782e-06, "loss": -0.0017, "num_tokens": 29018248.0, "reward": 0.009374991059303284, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.2348681688308716, "sampling/importance_sampling_ratio/mean": 1.024897575378418, "sampling/importance_sampling_ratio/min": 1.0009411573410034, "sampling/sampling_logp_difference/max": 0.19842588901519775, "sampling/sampling_logp_difference/mean": 0.011595778167247772, "step": 633, "step_time": 11.453733016023762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 83.46875, "completions/mean_terminated_length": 83.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10238810098962858, "epoch": 0.01268, "frac_reward_zero_std": 0.0, "grad_norm": 0.07610706984996796, "kl": 1.1113581079989672, "learning_rate": 4.1292515528925584e-06, "loss": -0.0035, "num_tokens": 29063092.0, "reward": 0.13624998927116394, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0546337366104126, "sampling/importance_sampling_ratio/mean": 0.9845280051231384, "sampling/importance_sampling_ratio/min": 0.45178109407424927, "sampling/sampling_logp_difference/max": 0.794435977935791, "sampling/sampling_logp_difference/mean": 0.022664129734039307, "step": 634, "step_time": 12.021569282995188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 53.5625, "completions/mean_terminated_length": 53.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10463419702136889, "epoch": 0.0127, "frac_reward_zero_std": 0.0, "grad_norm": 0.07545195519924164, "kl": 0.6783203929662704, "learning_rate": 4.11803311259798e-06, "loss": -0.0035, "num_tokens": 29107623.0, "reward": 0.07187499105930328, "reward_std": 1.0334783792495728, "rewards/rollout_reward_func/mean": 0.07187499105930328, "rewards/rollout_reward_func/std": 1.0229730606079102, "sampling/importance_sampling_ratio/max": 1.0619620084762573, "sampling/importance_sampling_ratio/mean": 0.999582827091217, "sampling/importance_sampling_ratio/min": 0.5573829412460327, "sampling/sampling_logp_difference/max": 0.5842909812927246, "sampling/sampling_logp_difference/mean": 0.014681683853268623, "step": 635, "step_time": 11.742149680016155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 112.6875, "completions/mean_terminated_length": 112.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14455986442044377, "epoch": 0.01272, "frac_reward_zero_std": 0.0, "grad_norm": 0.12367838621139526, "kl": 0.7085087448358536, "learning_rate": 4.106842794387864e-06, "loss": -0.02, "num_tokens": 29154930.0, "reward": -0.05375000834465027, "reward_std": 1.016998529434204, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.3875175714492798, "sampling/importance_sampling_ratio/mean": 1.0345247983932495, "sampling/importance_sampling_ratio/min": 1.001147747039795, "sampling/sampling_logp_difference/max": 0.3252525329589844, "sampling/sampling_logp_difference/mean": 0.01313737127929926, "step": 636, "step_time": 13.425566680001793 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 874.0, "completions/max_terminated_length": 874.0, "completions/mean_length": 139.21875, "completions/mean_terminated_length": 139.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10692158085294068, "epoch": 0.01274, "frac_reward_zero_std": 0.0, "grad_norm": 0.1931964010000229, "kl": 0.6141359135508537, "learning_rate": 4.095680745870069e-06, "loss": -0.0134, "num_tokens": 29202572.0, "reward": 0.32499998807907104, "reward_std": 0.9518023729324341, "rewards/rollout_reward_func/mean": 0.32499998807907104, "rewards/rollout_reward_func/std": 0.974315345287323, "sampling/importance_sampling_ratio/max": 1.5327259302139282, "sampling/importance_sampling_ratio/mean": 1.043830156326294, "sampling/importance_sampling_ratio/min": 0.7342653870582581, "sampling/sampling_logp_difference/max": 0.4035426378250122, "sampling/sampling_logp_difference/mean": 0.0193936824798584, "step": 637, "step_time": 13.514178445999278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 90.90625, "completions/mean_terminated_length": 90.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15916825557360426, "epoch": 0.01276, "frac_reward_zero_std": 0.0, "grad_norm": 0.18148696422576904, "kl": 0.9140604957938194, "learning_rate": 4.084547114279564e-06, "loss": -0.0089, "num_tokens": 29250164.0, "reward": -0.3056250214576721, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.4507098197937012, "sampling/importance_sampling_ratio/mean": 1.0171509981155396, "sampling/importance_sampling_ratio/min": 0.7039870023727417, "sampling/sampling_logp_difference/max": 0.3507382869720459, "sampling/sampling_logp_difference/mean": 0.02670961245894432, "step": 638, "step_time": 12.014260870018916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 69.28125, "completions/mean_terminated_length": 69.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12288976489799097, "epoch": 0.01278, "frac_reward_zero_std": 0.0, "grad_norm": 0.24589374661445618, "kl": 0.5997172202914953, "learning_rate": 4.073442046476472e-06, "loss": -0.0075, "num_tokens": 29295823.0, "reward": 0.07249999046325684, "reward_std": 0.8878730535507202, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.4641633033752441, "sampling/importance_sampling_ratio/mean": 1.0207035541534424, "sampling/importance_sampling_ratio/min": 0.7363333106040955, "sampling/sampling_logp_difference/max": 0.3756941556930542, "sampling/sampling_logp_difference/mean": 0.02176969312131405, "step": 639, "step_time": 11.545236433004902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 122.09375, "completions/mean_terminated_length": 122.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12554278981406242, "epoch": 0.0128, "frac_reward_zero_std": 0.0, "grad_norm": 0.388205885887146, "kl": 1.7457222659140825, "learning_rate": 4.062365688944147e-06, "loss": -0.0199, "num_tokens": 29344499.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.0688858032226562, "sampling/importance_sampling_ratio/mean": 0.972071647644043, "sampling/importance_sampling_ratio/min": 0.38468435406684875, "sampling/sampling_logp_difference/max": 0.9740180969238281, "sampling/sampling_logp_difference/mean": 0.028159573674201965, "step": 640, "step_time": 12.259886938009004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 75.3125, "completions/mean_terminated_length": 75.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15002098790137097, "epoch": 0.01282, "frac_reward_zero_std": 0.0, "grad_norm": 0.11629022657871246, "kl": 0.7606674879789352, "learning_rate": 4.051318187787231e-06, "loss": -0.0227, "num_tokens": 29390366.0, "reward": -0.11625000834465027, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.4781732559204102, "sampling/importance_sampling_ratio/mean": 0.9867714047431946, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0827370882034302, "sampling/sampling_logp_difference/mean": 0.04275720566511154, "step": 641, "step_time": 11.88388551101525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 83.75, "completions/mean_terminated_length": 83.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1316715992288664, "epoch": 0.01284, "frac_reward_zero_std": 0.0, "grad_norm": 0.46573376655578613, "kl": 0.8831786550581455, "learning_rate": 4.040299688729734e-06, "loss": -0.006, "num_tokens": 29436638.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 2.072195529937744, "sampling/importance_sampling_ratio/mean": 1.0276365280151367, "sampling/importance_sampling_ratio/min": 0.3407055735588074, "sampling/sampling_logp_difference/max": 1.0837914943695068, "sampling/sampling_logp_difference/mean": 0.04349315166473389, "step": 642, "step_time": 11.869452249011374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 63.375, "completions/mean_terminated_length": 63.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12129782768897712, "epoch": 0.01286, "frac_reward_zero_std": 0.0, "grad_norm": 0.11869508773088455, "kl": 0.7763811927288771, "learning_rate": 4.029310337113106e-06, "loss": -0.0093, "num_tokens": 29482947.0, "reward": -0.17937500774860382, "reward_std": 0.9160174131393433, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.1698981523513794, "sampling/importance_sampling_ratio/mean": 1.016765832901001, "sampling/importance_sampling_ratio/min": 0.784538209438324, "sampling/sampling_logp_difference/max": 0.24112796783447266, "sampling/sampling_logp_difference/mean": 0.012615252286195755, "step": 643, "step_time": 10.928529948978394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 47.21875, "completions/mean_terminated_length": 47.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1295291385613382, "epoch": 0.01288, "frac_reward_zero_std": 0.0, "grad_norm": 0.1887221783399582, "kl": 0.6785506512969732, "learning_rate": 4.018350277894324e-06, "loss": -0.017, "num_tokens": 29528643.0, "reward": -0.11625000834465027, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.140906572341919, "sampling/importance_sampling_ratio/mean": 0.9815216064453125, "sampling/importance_sampling_ratio/min": 0.43323591351509094, "sampling/sampling_logp_difference/max": 0.8362894058227539, "sampling/sampling_logp_difference/mean": 0.03124150261282921, "step": 644, "step_time": 11.376043910015142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 83.1875, "completions/mean_terminated_length": 83.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08523404086008668, "epoch": 0.0129, "frac_reward_zero_std": 0.0, "grad_norm": 0.37856706976890564, "kl": 0.61711585521698, "learning_rate": 4.007419655643977e-06, "loss": -0.0071, "num_tokens": 29573392.0, "reward": 0.2618749737739563, "reward_std": 0.9621194005012512, "rewards/rollout_reward_func/mean": 0.2618749737739563, "rewards/rollout_reward_func/std": 0.9930903911590576, "sampling/importance_sampling_ratio/max": 1.1457209587097168, "sampling/importance_sampling_ratio/mean": 1.0077205896377563, "sampling/importance_sampling_ratio/min": 0.7649416923522949, "sampling/sampling_logp_difference/max": 0.26796388626098633, "sampling/sampling_logp_difference/mean": 0.009817507117986679, "step": 645, "step_time": 11.495204303006176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 88.53125, "completions/mean_terminated_length": 88.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13477018632693216, "epoch": 0.01292, "frac_reward_zero_std": 0.0, "grad_norm": 0.18513520061969757, "kl": 2.3674109932035208, "learning_rate": 3.996518614544363e-06, "loss": -0.0232, "num_tokens": 29620966.0, "reward": -0.4318750202655792, "reward_std": 0.935038685798645, "rewards/rollout_reward_func/mean": -0.4318750202655792, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.025718331336975, "sampling/importance_sampling_ratio/mean": 0.935966968536377, "sampling/importance_sampling_ratio/min": 0.43575775623321533, "sampling/sampling_logp_difference/max": 0.8560380935668945, "sampling/sampling_logp_difference/mean": 0.04887464642524719, "step": 646, "step_time": 13.48068253701058 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 81.375, "completions/mean_terminated_length": 81.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09180072566960007, "epoch": 0.01294, "frac_reward_zero_std": 0.0, "grad_norm": 0.24055933952331543, "kl": 0.7510105073451996, "learning_rate": 3.985647298387584e-06, "loss": -0.0052, "num_tokens": 29666013.0, "reward": -0.05375000834465027, "reward_std": 1.0703058242797852, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.1588214635849, "sampling/importance_sampling_ratio/mean": 0.971504807472229, "sampling/importance_sampling_ratio/min": 0.415147066116333, "sampling/sampling_logp_difference/max": 0.8789334297180176, "sampling/sampling_logp_difference/mean": 0.034079406410455704, "step": 647, "step_time": 14.617971828985901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 92.46875, "completions/mean_terminated_length": 92.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07166203745873645, "epoch": 0.01296, "frac_reward_zero_std": 0.0, "grad_norm": 0.18147514760494232, "kl": 2.353877266868949, "learning_rate": 3.9748058505736474e-06, "loss": -0.0017, "num_tokens": 29712159.0, "reward": -0.11687500774860382, "reward_std": 1.0445914268493652, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.0362516641616821, "sampling/importance_sampling_ratio/mean": 0.9940236210823059, "sampling/importance_sampling_ratio/min": 0.42026287317276, "sampling/sampling_logp_difference/max": 0.8668465614318848, "sampling/sampling_logp_difference/mean": 0.017123937606811523, "step": 648, "step_time": 14.391848839986778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 125.90625, "completions/mean_terminated_length": 125.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10321666451636702, "epoch": 0.01298, "frac_reward_zero_std": 0.0, "grad_norm": 0.29241612553596497, "kl": 0.6303191483020782, "learning_rate": 3.9639944141085785e-06, "loss": -0.0105, "num_tokens": 29759824.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.2157665491104126, "sampling/importance_sampling_ratio/mean": 0.9664614200592041, "sampling/importance_sampling_ratio/min": 0.29204100370407104, "sampling/sampling_logp_difference/max": 1.2306828498840332, "sampling/sampling_logp_difference/mean": 0.03850643336772919, "step": 649, "step_time": 11.623699039002531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 83.75, "completions/mean_terminated_length": 83.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09801206947304308, "epoch": 0.013, "frac_reward_zero_std": 0.0, "grad_norm": 0.2074030190706253, "kl": 1.0752319190651178, "learning_rate": 3.953213131602535e-06, "loss": -0.0193, "num_tokens": 29806957.0, "reward": -0.43187499046325684, "reward_std": 0.8883851170539856, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.606750726699829, "sampling/importance_sampling_ratio/mean": 1.0264723300933838, "sampling/importance_sampling_ratio/min": 0.8839622735977173, "sampling/sampling_logp_difference/max": 0.467235803604126, "sampling/sampling_logp_difference/mean": 0.014326468110084534, "step": 650, "step_time": 11.834267526006442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 153.84375, "completions/mean_terminated_length": 153.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08743255224544555, "epoch": 0.01302, "frac_reward_zero_std": 0.0, "grad_norm": 0.2738213837146759, "kl": 0.6641209572553635, "learning_rate": 3.942462145267919e-06, "loss": -0.028, "num_tokens": 29856175.0, "reward": -0.11750000715255737, "reward_std": 1.0250580310821533, "rewards/rollout_reward_func/mean": -0.11750000715255737, "rewards/rollout_reward_func/std": 1.0166834592819214, "sampling/importance_sampling_ratio/max": 1.1194627285003662, "sampling/importance_sampling_ratio/mean": 0.9969936609268188, "sampling/importance_sampling_ratio/min": 0.5504066944122314, "sampling/sampling_logp_difference/max": 0.6005921363830566, "sampling/sampling_logp_difference/mean": 0.015850216150283813, "step": 651, "step_time": 13.778392225009156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 72.78125, "completions/mean_terminated_length": 72.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09553707961458713, "epoch": 0.01304, "frac_reward_zero_std": 0.0, "grad_norm": 0.269928514957428, "kl": 1.9900635555386543, "learning_rate": 3.93174159691751e-06, "loss": -0.0118, "num_tokens": 29902452.0, "reward": -0.11687500774860382, "reward_std": 1.0445914268493652, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.4490545988082886, "sampling/importance_sampling_ratio/mean": 0.9797345995903015, "sampling/importance_sampling_ratio/min": 0.539223849773407, "sampling/sampling_logp_difference/max": 0.6323096752166748, "sampling/sampling_logp_difference/mean": 0.03056861087679863, "step": 652, "step_time": 11.805192080020788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 120.28125, "completions/mean_terminated_length": 120.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12962181330658495, "epoch": 0.01306, "frac_reward_zero_std": 0.0, "grad_norm": 0.4582325518131256, "kl": 1.116364173591137, "learning_rate": 3.921051627962586e-06, "loss": 0.0027, "num_tokens": 29948908.0, "reward": -0.053125008940696716, "reward_std": 1.0711638927459717, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.567206621170044, "sampling/importance_sampling_ratio/mean": 0.9937304258346558, "sampling/importance_sampling_ratio/min": 0.18513035774230957, "sampling/sampling_logp_difference/max": 1.726426124572754, "sampling/sampling_logp_difference/mean": 0.05562888830900192, "step": 653, "step_time": 14.2600451330145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 84.3125, "completions/mean_terminated_length": 84.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10776988475117832, "epoch": 0.01308, "frac_reward_zero_std": 0.0, "grad_norm": 0.1692894697189331, "kl": 1.079891886562109, "learning_rate": 3.9103923794110645e-06, "loss": -0.0168, "num_tokens": 29995483.0, "reward": -0.24250000715255737, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.199310541152954, "sampling/importance_sampling_ratio/mean": 0.9928701519966125, "sampling/importance_sampling_ratio/min": 0.44144755601882935, "sampling/sampling_logp_difference/max": 0.8522524833679199, "sampling/sampling_logp_difference/mean": 0.022579509764909744, "step": 654, "step_time": 11.620740307967935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 856.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 69.6875, "completions/mean_terminated_length": 69.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12668574915733188, "epoch": 0.0131, "frac_reward_zero_std": 0.25, "grad_norm": 0.2527756989002228, "kl": 1.1346510667353868, "learning_rate": 3.89976399186564e-06, "loss": -0.0165, "num_tokens": 30041318.0, "reward": -0.3056250214576721, "reward_std": 0.8012300729751587, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.0610846281051636, "sampling/importance_sampling_ratio/mean": 0.979562520980835, "sampling/importance_sampling_ratio/min": 0.6369218826293945, "sampling/sampling_logp_difference/max": 0.45125532150268555, "sampling/sampling_logp_difference/mean": 0.021231042221188545, "step": 655, "step_time": 13.304490989008627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 43.4375, "completions/mean_terminated_length": 43.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09693385439459234, "epoch": 0.01312, "frac_reward_zero_std": 0.0, "grad_norm": 0.0775994211435318, "kl": 0.6067631617188454, "learning_rate": 3.8891666055219305e-06, "loss": -0.0051, "num_tokens": 30085489.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1323295831680298, "sampling/importance_sampling_ratio/mean": 1.0064642429351807, "sampling/importance_sampling_ratio/min": 0.7943315505981445, "sampling/sampling_logp_difference/max": 0.23012912273406982, "sampling/sampling_logp_difference/mean": 0.011686515994369984, "step": 656, "step_time": 11.396050297014881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 60.15625, "completions/mean_terminated_length": 60.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12482075113803148, "epoch": 0.01314, "frac_reward_zero_std": 0.0, "grad_norm": 0.10519778728485107, "kl": 0.7656336668878794, "learning_rate": 3.878600360166621e-06, "loss": -0.0031, "num_tokens": 30131983.0, "reward": -0.36875003576278687, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 2.038677215576172, "sampling/importance_sampling_ratio/mean": 1.0242716073989868, "sampling/importance_sampling_ratio/min": 0.4801904261112213, "sampling/sampling_logp_difference/max": 0.7365446090698242, "sampling/sampling_logp_difference/mean": 0.03363513946533203, "step": 657, "step_time": 11.476595333013393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 55.375, "completions/mean_terminated_length": 55.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08439109096070752, "epoch": 0.01316, "frac_reward_zero_std": 0.0, "grad_norm": 0.1694517731666565, "kl": 0.9578372519463301, "learning_rate": 3.868065395175636e-06, "loss": -0.0111, "num_tokens": 30176547.0, "reward": -0.05375000834465027, "reward_std": 1.0533576011657715, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.0375473499298096, "sampling/importance_sampling_ratio/mean": 0.988469123840332, "sampling/importance_sampling_ratio/min": 0.48369649052619934, "sampling/sampling_logp_difference/max": 0.7262430191040039, "sampling/sampling_logp_difference/mean": 0.01610548608005047, "step": 658, "step_time": 11.936592066012963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 75.1875, "completions/mean_terminated_length": 75.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10863230924587697, "epoch": 0.01318, "frac_reward_zero_std": 0.0, "grad_norm": 0.27496951818466187, "kl": 0.6095473598688841, "learning_rate": 3.857561849512283e-06, "loss": 0.0011, "num_tokens": 30221253.0, "reward": -0.05375000834465027, "reward_std": 1.0531628131866455, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.0367807149887085, "sampling/importance_sampling_ratio/mean": 0.9867693781852722, "sampling/importance_sampling_ratio/min": 0.6727477312088013, "sampling/sampling_logp_difference/max": 0.39642536640167236, "sampling/sampling_logp_difference/mean": 0.014703522436320782, "step": 659, "step_time": 11.368197212992527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 17.15625, "completions/mean_terminated_length": 17.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10316307452740148, "epoch": 0.0132, "frac_reward_zero_std": 0.0, "grad_norm": 0.32339558005332947, "kl": 0.8179991375654936, "learning_rate": 3.847089861725429e-06, "loss": -0.0017, "num_tokens": 30263831.0, "reward": 0.009999990463256836, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 2.0176172256469727, "sampling/importance_sampling_ratio/mean": 1.0428787469863892, "sampling/importance_sampling_ratio/min": 0.6150956153869629, "sampling/sampling_logp_difference/max": 0.7019628286361694, "sampling/sampling_logp_difference/mean": 0.029859211295843124, "step": 660, "step_time": 10.100093663015286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 874.0, "completions/max_terminated_length": 874.0, "completions/mean_length": 143.59375, "completions/mean_terminated_length": 143.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16049998067319393, "epoch": 0.01322, "frac_reward_zero_std": 0.0, "grad_norm": 0.5232264995574951, "kl": 1.1022763140499592, "learning_rate": 3.836649569947673e-06, "loss": -0.0454, "num_tokens": 30312816.0, "reward": -0.11625000834465027, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.5859906673431396, "sampling/importance_sampling_ratio/mean": 1.029942512512207, "sampling/importance_sampling_ratio/min": 0.6708168983459473, "sampling/sampling_logp_difference/max": 0.7845467329025269, "sampling/sampling_logp_difference/mean": 0.04162426292896271, "step": 661, "step_time": 13.574014729005285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 98.21875, "completions/mean_terminated_length": 98.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1015267571201548, "epoch": 0.01324, "frac_reward_zero_std": 0.0, "grad_norm": 0.14124317467212677, "kl": 0.8104436155408621, "learning_rate": 3.82624111189352e-06, "loss": -0.0184, "num_tokens": 30358925.0, "reward": -0.053125008940696716, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.293580412864685, "sampling/importance_sampling_ratio/mean": 0.988673210144043, "sampling/importance_sampling_ratio/min": 0.3706483244895935, "sampling/sampling_logp_difference/max": 0.991429328918457, "sampling/sampling_logp_difference/mean": 0.02905716747045517, "step": 662, "step_time": 14.084472220987664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 132.21875, "completions/mean_terminated_length": 132.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1264798572519794, "epoch": 0.01326, "frac_reward_zero_std": 0.0, "grad_norm": 0.3562740385532379, "kl": 1.2197004035115242, "learning_rate": 3.81586462485757e-06, "loss": 0.0001, "num_tokens": 30407405.0, "reward": -0.18000000715255737, "reward_std": 1.0343364477157593, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 2.1157612800598145, "sampling/importance_sampling_ratio/mean": 1.0275105237960815, "sampling/importance_sampling_ratio/min": 0.11087698489427567, "sampling/sampling_logp_difference/max": 2.1763103008270264, "sampling/sampling_logp_difference/mean": 0.061136599630117416, "step": 663, "step_time": 13.512702900989098 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 95.8125, "completions/mean_terminated_length": 95.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1358727682963945, "epoch": 0.01328, "frac_reward_zero_std": 0.0, "grad_norm": 0.17009498178958893, "kl": 1.1723740752786398, "learning_rate": 3.8055202457127015e-06, "loss": -0.0245, "num_tokens": 30454567.0, "reward": -0.3056250214576721, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.398444414138794, "sampling/importance_sampling_ratio/mean": 0.9753265976905823, "sampling/importance_sampling_ratio/min": 0.27977654337882996, "sampling/sampling_logp_difference/max": 1.273576259613037, "sampling/sampling_logp_difference/mean": 0.03776643052697182, "step": 664, "step_time": 14.41449706901767 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 829.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 91.1875, "completions/mean_terminated_length": 91.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10162153001874685, "epoch": 0.0133, "frac_reward_zero_std": 0.0, "grad_norm": 0.570281982421875, "kl": 0.7697831746190786, "learning_rate": 3.795208110908265e-06, "loss": -0.014, "num_tokens": 30500208.0, "reward": -0.05375000834465027, "reward_std": 1.0703058242797852, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.940388560295105, "sampling/importance_sampling_ratio/mean": 1.044562578201294, "sampling/importance_sampling_ratio/min": 0.5049768090248108, "sampling/sampling_logp_difference/max": 0.6832723617553711, "sampling/sampling_logp_difference/mean": 0.039902374148368835, "step": 665, "step_time": 12.424115770001663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 67.4375, "completions/mean_terminated_length": 67.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08860861253924668, "epoch": 0.01332, "frac_reward_zero_std": 0.0, "grad_norm": 1.8546886444091797, "kl": 9.776646373793483, "learning_rate": 3.784928356468293e-06, "loss": 0.0002, "num_tokens": 30546087.0, "reward": -0.053125008940696716, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.079466462135315, "sampling/importance_sampling_ratio/mean": 0.978266179561615, "sampling/importance_sampling_ratio/min": 0.3252388834953308, "sampling/sampling_logp_difference/max": 1.1232128143310547, "sampling/sampling_logp_difference/mean": 0.028793534263968468, "step": 666, "step_time": 10.985219450994919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 124.0625, "completions/mean_terminated_length": 124.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08217632351443172, "epoch": 0.01334, "frac_reward_zero_std": 0.25, "grad_norm": 0.5019945502281189, "kl": 0.5521652940660715, "learning_rate": 3.7746811179896937e-06, "loss": -0.0126, "num_tokens": 30592427.0, "reward": 0.3256250023841858, "reward_std": 0.7012691497802734, "rewards/rollout_reward_func/mean": 0.3256250023841858, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.452836513519287, "sampling/importance_sampling_ratio/mean": 1.0405054092407227, "sampling/importance_sampling_ratio/min": 1.0015069246292114, "sampling/sampling_logp_difference/max": 0.37370890378952026, "sampling/sampling_logp_difference/mean": 0.014915871433913708, "step": 667, "step_time": 14.002467446000082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 55.65625, "completions/mean_terminated_length": 55.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10010954935569316, "epoch": 0.01336, "frac_reward_zero_std": 0.0, "grad_norm": 0.11850083619356155, "kl": 0.8326656557619572, "learning_rate": 3.7644665306404726e-06, "loss": -0.0085, "num_tokens": 30638534.0, "reward": -0.3056250214576721, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.431057095527649, "sampling/importance_sampling_ratio/mean": 1.0053755044937134, "sampling/importance_sampling_ratio/min": 0.6130011081695557, "sampling/sampling_logp_difference/max": 0.49108266830444336, "sampling/sampling_logp_difference/mean": 0.023543238639831543, "step": 668, "step_time": 12.054281779004668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 92.71875, "completions/mean_terminated_length": 92.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1343753954861313, "epoch": 0.01338, "frac_reward_zero_std": 0.0, "grad_norm": 0.13086619973182678, "kl": 0.6737979296594858, "learning_rate": 3.7542847291579375e-06, "loss": -0.0164, "num_tokens": 30684084.0, "reward": -0.053125008940696716, "reward_std": 0.9160174131393433, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.940710186958313, "sampling/importance_sampling_ratio/mean": 1.0361615419387817, "sampling/importance_sampling_ratio/min": 0.34577080607414246, "sampling/sampling_logp_difference/max": 0.9195529818534851, "sampling/sampling_logp_difference/mean": 0.04576658084988594, "step": 669, "step_time": 13.265603401996486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 71.59375, "completions/mean_terminated_length": 71.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13502225908450782, "epoch": 0.0134, "frac_reward_zero_std": 0.0, "grad_norm": 0.5187908411026001, "kl": 3.136768637225032, "learning_rate": 3.744135847846937e-06, "loss": -0.0046, "num_tokens": 30729489.0, "reward": -0.11625000834465027, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.594619870185852, "sampling/importance_sampling_ratio/mean": 0.9934428930282593, "sampling/importance_sampling_ratio/min": 0.4230059087276459, "sampling/sampling_logp_difference/max": 0.8602266311645508, "sampling/sampling_logp_difference/mean": 0.04719549044966698, "step": 670, "step_time": 12.07482782900479 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 110.4375, "completions/mean_terminated_length": 110.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12096177227795124, "epoch": 0.01342, "frac_reward_zero_std": 0.0, "grad_norm": 0.18437667191028595, "kl": 0.722720380872488, "learning_rate": 3.7340200205780724e-06, "loss": -0.0335, "num_tokens": 30777142.0, "reward": -0.11750000715255737, "reward_std": 1.0608855485916138, "rewards/rollout_reward_func/mean": -0.11750000715255737, "rewards/rollout_reward_func/std": 1.0166960954666138, "sampling/importance_sampling_ratio/max": 1.836210012435913, "sampling/importance_sampling_ratio/mean": 1.0127614736557007, "sampling/importance_sampling_ratio/min": 0.6052735447883606, "sampling/sampling_logp_difference/max": 0.685293436050415, "sampling/sampling_logp_difference/mean": 0.030277814716100693, "step": 671, "step_time": 11.780848679998599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "completions/clipped_ratio": 0.03125, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 72.375, "completions/mean_terminated_length": 74.19354248046875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14881393732503057, "epoch": 0.01344, "frac_reward_zero_std": 0.0, "grad_norm": 0.39464834332466125, "kl": 1.2403862942010164, "learning_rate": 3.723937380785948e-06, "loss": 0.0013, "num_tokens": 30821221.0, "reward": 0.1993749886751175, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.7858895063400269, "sampling/importance_sampling_ratio/mean": 1.0018305778503418, "sampling/importance_sampling_ratio/min": 0.5191580057144165, "sampling/sampling_logp_difference/max": 0.6553922295570374, "sampling/sampling_logp_difference/mean": 0.04214553162455559, "step": 672, "step_time": 11.768193420000898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 93.15625, "completions/mean_terminated_length": 93.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09352790738921613, "epoch": 0.01346, "frac_reward_zero_std": 0.0, "grad_norm": 0.18383345007896423, "kl": 1.0425384398549795, "learning_rate": 3.7138880614673957e-06, "loss": -0.0085, "num_tokens": 30868142.0, "reward": -0.11625000834465027, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0301027297973633, "sampling/importance_sampling_ratio/mean": 0.9980345964431763, "sampling/importance_sampling_ratio/min": 0.5784663558006287, "sampling/sampling_logp_difference/max": 0.6646723747253418, "sampling/sampling_logp_difference/mean": 0.013897519558668137, "step": 673, "step_time": 13.934331511001801 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 51.4375, "completions/mean_terminated_length": 51.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14172781654633582, "epoch": 0.01348, "frac_reward_zero_std": 0.0, "grad_norm": 0.43524131178855896, "kl": 0.6123971827328205, "learning_rate": 3.7038721951797318e-06, "loss": -0.0048, "num_tokens": 30912996.0, "reward": -0.05437500774860382, "reward_std": 1.0168434381484985, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228264331817627, "sampling/importance_sampling_ratio/max": 1.6275393962860107, "sampling/importance_sampling_ratio/mean": 0.9929172992706299, "sampling/importance_sampling_ratio/min": 0.537533164024353, "sampling/sampling_logp_difference/max": 0.6207244396209717, "sampling/sampling_logp_difference/mean": 0.0345311239361763, "step": 674, "step_time": 11.874425981004606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 99.0625, "completions/mean_terminated_length": 101.74193572998047, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13665772788226604, "epoch": 0.0135, "frac_reward_zero_std": 0.0, "grad_norm": 0.5115253329277039, "kl": 1.3440339136868715, "learning_rate": 3.6938899140390016e-06, "loss": -0.0327, "num_tokens": 30957690.0, "reward": 0.32499998807907104, "reward_std": 0.9706909656524658, "rewards/rollout_reward_func/mean": 0.32499998807907104, "rewards/rollout_reward_func/std": 0.974315345287323, "sampling/importance_sampling_ratio/max": 1.661575198173523, "sampling/importance_sampling_ratio/mean": 1.0178985595703125, "sampling/importance_sampling_ratio/min": 0.37281396985054016, "sampling/sampling_logp_difference/max": 0.986748218536377, "sampling/sampling_logp_difference/mean": 0.024173971265554428, "step": 675, "step_time": 14.291311512984976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 101.59375, "completions/mean_terminated_length": 101.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12004508508834988, "epoch": 0.01352, "frac_reward_zero_std": 0.0, "grad_norm": 0.41611528396606445, "kl": 5.281777383759618, "learning_rate": 3.683941349718244e-06, "loss": -0.005, "num_tokens": 31002620.0, "reward": 0.008749991655349731, "reward_std": 1.0612659454345703, "rewards/rollout_reward_func/mean": 0.008749991655349731, "rewards/rollout_reward_func/std": 1.0249019861221313, "sampling/importance_sampling_ratio/max": 1.8562812805175781, "sampling/importance_sampling_ratio/mean": 0.9795830845832825, "sampling/importance_sampling_ratio/min": 0.37456318736076355, "sampling/sampling_logp_difference/max": 0.9819951057434082, "sampling/sampling_logp_difference/mean": 0.046671196818351746, "step": 676, "step_time": 13.493954971992935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 366.0, "completions/max_terminated_length": 366.0, "completions/mean_length": 19.40625, "completions/mean_terminated_length": 19.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1812206655740738, "epoch": 0.01354, "frac_reward_zero_std": 0.0, "grad_norm": 0.24204647541046143, "kl": 0.616950424388051, "learning_rate": 3.674026633445747e-06, "loss": -0.0053, "num_tokens": 31046506.0, "reward": -0.18000000715255737, "reward_std": 1.0343363285064697, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.4935638904571533, "sampling/importance_sampling_ratio/mean": 0.9691935777664185, "sampling/importance_sampling_ratio/min": 0.29658371210098267, "sampling/sampling_logp_difference/max": 1.2272520065307617, "sampling/sampling_logp_difference/mean": 0.06303051114082336, "step": 677, "step_time": 10.444098069005122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 775.0, "completions/max_terminated_length": 775.0, "completions/mean_length": 72.09375, "completions/mean_terminated_length": 72.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.159437813796103, "epoch": 0.01356, "frac_reward_zero_std": 0.0, "grad_norm": 0.11568842828273773, "kl": 0.6346654333174229, "learning_rate": 3.6641458960033194e-06, "loss": -0.0154, "num_tokens": 31092697.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.3072465658187866, "sampling/importance_sampling_ratio/mean": 1.0164215564727783, "sampling/importance_sampling_ratio/min": 0.46512371301651, "sampling/sampling_logp_difference/max": 0.7655086517333984, "sampling/sampling_logp_difference/mean": 0.024792689830064774, "step": 678, "step_time": 12.260530929990637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 155.9375, "completions/mean_terminated_length": 155.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13621504907496274, "epoch": 0.01358, "frac_reward_zero_std": 0.0, "grad_norm": 0.7723272442817688, "kl": 0.7106228172779083, "learning_rate": 3.654299267724567e-06, "loss": -0.0087, "num_tokens": 31139676.0, "reward": 0.3256250023841858, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": 0.3256250023841858, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 2.4257051944732666, "sampling/importance_sampling_ratio/mean": 1.0288631916046143, "sampling/importance_sampling_ratio/min": 0.3450426161289215, "sampling/sampling_logp_difference/max": 1.0752533674240112, "sampling/sampling_logp_difference/mean": 0.036401793360710144, "step": 679, "step_time": 13.415412539012323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 151.15625, "completions/mean_terminated_length": 151.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14784337487071753, "epoch": 0.0136, "frac_reward_zero_std": 0.0, "grad_norm": 0.37471553683280945, "kl": 0.6499296193942428, "learning_rate": 3.6444868784931752e-06, "loss": -0.0246, "num_tokens": 31188573.0, "reward": 0.13499999046325684, "reward_std": 1.0251907110214233, "rewards/rollout_reward_func/mean": 0.13499999046325684, "rewards/rollout_reward_func/std": 1.0170037746429443, "sampling/importance_sampling_ratio/max": 1.8110679388046265, "sampling/importance_sampling_ratio/mean": 1.0331518650054932, "sampling/importance_sampling_ratio/min": 0.6357179880142212, "sampling/sampling_logp_difference/max": 0.5679205656051636, "sampling/sampling_logp_difference/mean": 0.023177597671747208, "step": 680, "step_time": 14.195319038997695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 393.0, "completions/max_terminated_length": 393.0, "completions/mean_length": 14.21875, "completions/mean_terminated_length": 14.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26321996096521616, "epoch": 0.01362, "frac_reward_zero_std": 0.0, "grad_norm": 0.2731817066669464, "kl": 2.031318884342909, "learning_rate": 3.6347088577411906e-06, "loss": -0.0063, "num_tokens": 31231928.0, "reward": -0.3056250214576721, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.6842448711395264, "sampling/importance_sampling_ratio/mean": 1.0286542177200317, "sampling/importance_sampling_ratio/min": 0.5870248079299927, "sampling/sampling_logp_difference/max": 0.5326054096221924, "sampling/sampling_logp_difference/mean": 0.047558560967445374, "step": 681, "step_time": 10.962363590006134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 147.53125, "completions/mean_terminated_length": 147.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16360846371389925, "epoch": 0.01364, "frac_reward_zero_std": 0.0, "grad_norm": 0.5079512596130371, "kl": 0.6412533055990934, "learning_rate": 3.6249653344473162e-06, "loss": -0.0087, "num_tokens": 31280688.0, "reward": -0.05375000834465027, "reward_std": 0.9518023729324341, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.3774909973144531, "sampling/importance_sampling_ratio/mean": 1.0359479188919067, "sampling/importance_sampling_ratio/min": 0.9023335576057434, "sampling/sampling_logp_difference/max": 0.3043074309825897, "sampling/sampling_logp_difference/mean": 0.015657898038625717, "step": 682, "step_time": 13.50500467402162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 79.375, "completions/mean_terminated_length": 79.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15060565061867237, "epoch": 0.01366, "frac_reward_zero_std": 0.0, "grad_norm": 0.4516844153404236, "kl": 2.117276217788458, "learning_rate": 3.61525643713521e-06, "loss": -0.0199, "num_tokens": 31326939.0, "reward": -0.053125008940696716, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.4531501531600952, "sampling/importance_sampling_ratio/mean": 0.9965353608131409, "sampling/importance_sampling_ratio/min": 0.30772826075553894, "sampling/sampling_logp_difference/max": 1.196272611618042, "sampling/sampling_logp_difference/mean": 0.03557129204273224, "step": 683, "step_time": 11.583511008982896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 39.125, "completions/mean_terminated_length": 39.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14540544128976762, "epoch": 0.01368, "frac_reward_zero_std": 0.0, "grad_norm": 0.7521297335624695, "kl": 0.9476046059280634, "learning_rate": 3.6055822938717933e-06, "loss": -0.0076, "num_tokens": 31370914.0, "reward": -0.053125008940696716, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0893008708953857, "sampling/importance_sampling_ratio/mean": 0.9800143241882324, "sampling/importance_sampling_ratio/min": 0.5174297094345093, "sampling/sampling_logp_difference/max": 0.6591365337371826, "sampling/sampling_logp_difference/mean": 0.029893426224589348, "step": 684, "step_time": 11.17824699099583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 107.3125, "completions/mean_terminated_length": 107.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2142509249970317, "epoch": 0.0137, "frac_reward_zero_std": 0.0, "grad_norm": 0.7137022018432617, "kl": 1.3991131596267223, "learning_rate": 3.595943032265554e-06, "loss": -0.0185, "num_tokens": 31418373.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.4566550254821777, "sampling/importance_sampling_ratio/mean": 0.979577898979187, "sampling/importance_sampling_ratio/min": 0.14302237331867218, "sampling/sampling_logp_difference/max": 1.9636201858520508, "sampling/sampling_logp_difference/mean": 0.056373730301856995, "step": 685, "step_time": 12.201236423992668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 119.15625, "completions/mean_terminated_length": 119.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16216550208628178, "epoch": 0.01372, "frac_reward_zero_std": 0.0, "grad_norm": 0.4480718672275543, "kl": 0.7340421490371227, "learning_rate": 3.5863387794648695e-06, "loss": -0.0299, "num_tokens": 31465086.0, "reward": 0.07312498986721039, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.082250714302063, "sampling/importance_sampling_ratio/mean": 0.9575682878494263, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1413850784301758, "sampling/sampling_logp_difference/mean": 0.03826536238193512, "step": 686, "step_time": 13.553420308024215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 86.28125, "completions/mean_terminated_length": 86.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2068752283230424, "epoch": 0.01374, "frac_reward_zero_std": 0.0, "grad_norm": 0.3226014971733093, "kl": 0.5768997110426426, "learning_rate": 3.5767696621563247e-06, "loss": -0.0135, "num_tokens": 31511475.0, "reward": -0.11812500655651093, "reward_std": 1.0239583253860474, "rewards/rollout_reward_func/mean": -0.11812500655651093, "rewards/rollout_reward_func/std": 1.0159801244735718, "sampling/importance_sampling_ratio/max": 1.199273705482483, "sampling/importance_sampling_ratio/mean": 0.9819602966308594, "sampling/importance_sampling_ratio/min": 0.34179672598838806, "sampling/sampling_logp_difference/max": 1.0877273082733154, "sampling/sampling_logp_difference/mean": 0.039418451488018036, "step": 687, "step_time": 11.94238369201048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 112.8125, "completions/mean_terminated_length": 112.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16174990870058537, "epoch": 0.01376, "frac_reward_zero_std": 0.0, "grad_norm": 0.1609533131122589, "kl": 0.6317895594984293, "learning_rate": 3.5672358065630486e-06, "loss": -0.0089, "num_tokens": 31559525.0, "reward": -0.18062500655651093, "reward_std": 1.052304744720459, "rewards/rollout_reward_func/mean": -0.18062500655651093, "rewards/rollout_reward_func/std": 1.0064371824264526, "sampling/importance_sampling_ratio/max": 1.0654042959213257, "sampling/importance_sampling_ratio/mean": 0.9999574422836304, "sampling/importance_sampling_ratio/min": 0.557712733745575, "sampling/sampling_logp_difference/max": 0.5827839374542236, "sampling/sampling_logp_difference/mean": 0.01789204776287079, "step": 688, "step_time": 11.477654007008823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 114.59375, "completions/mean_terminated_length": 114.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22463700408115983, "epoch": 0.01378, "frac_reward_zero_std": 0.0, "grad_norm": 0.3568863272666931, "kl": 1.5323918107897043, "learning_rate": 3.5577373384430397e-06, "loss": 0.0002, "num_tokens": 31607920.0, "reward": -0.3687500059604645, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.3715251684188843, "sampling/importance_sampling_ratio/mean": 0.9626360535621643, "sampling/importance_sampling_ratio/min": 0.44857901334762573, "sampling/sampling_logp_difference/max": 0.6661484241485596, "sampling/sampling_logp_difference/mean": 0.05414924770593643, "step": 689, "step_time": 12.159775146996253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 69.84375, "completions/mean_terminated_length": 69.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17086252220906317, "epoch": 0.0138, "frac_reward_zero_std": 0.25, "grad_norm": 0.12639665603637695, "kl": 1.6283511258661747, "learning_rate": 3.5482743830875137e-06, "loss": -0.0059, "num_tokens": 31653734.0, "reward": -0.36875003576278687, "reward_std": 0.7926585674285889, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.1263161897659302, "sampling/importance_sampling_ratio/mean": 0.9733279347419739, "sampling/importance_sampling_ratio/min": 0.098998062312603, "sampling/sampling_logp_difference/max": 1.2520506381988525, "sampling/sampling_logp_difference/mean": 0.052238382399082184, "step": 690, "step_time": 15.401322894998884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 54.25, "completions/mean_terminated_length": 54.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17605642625130713, "epoch": 0.01382, "frac_reward_zero_std": 0.0, "grad_norm": 0.3282252550125122, "kl": 1.01265381090343, "learning_rate": 3.538847065319248e-06, "loss": -0.004, "num_tokens": 31698189.0, "reward": 0.009999990463256836, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.3502055406570435, "sampling/importance_sampling_ratio/mean": 1.0100619792938232, "sampling/importance_sampling_ratio/min": 0.7096609473228455, "sampling/sampling_logp_difference/max": 0.3430331349372864, "sampling/sampling_logp_difference/mean": 0.022122740745544434, "step": 691, "step_time": 11.803808961994946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 77.71875, "completions/mean_terminated_length": 77.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17748351907357574, "epoch": 0.01384, "frac_reward_zero_std": 0.0, "grad_norm": 0.2844845652580261, "kl": 1.3627657927572727, "learning_rate": 3.5294555094909387e-06, "loss": 0.0059, "num_tokens": 31743255.0, "reward": 0.009374991059303284, "reward_std": 1.0070273876190186, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.2967448234558105, "sampling/importance_sampling_ratio/mean": 1.0018885135650635, "sampling/importance_sampling_ratio/min": 0.5489605665206909, "sampling/sampling_logp_difference/max": 0.6015787124633789, "sampling/sampling_logp_difference/mean": 0.028946194797754288, "step": 692, "step_time": 14.407676291004464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 187.28125, "completions/mean_terminated_length": 187.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1634428263641894, "epoch": 0.01386, "frac_reward_zero_std": 0.0, "grad_norm": 0.2303048074245453, "kl": 0.6255578193813562, "learning_rate": 3.520099839483554e-06, "loss": -0.026, "num_tokens": 31793959.0, "reward": 0.07249999046325684, "reward_std": 0.9347947835922241, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.4815170764923096, "sampling/importance_sampling_ratio/mean": 1.0450806617736816, "sampling/importance_sampling_ratio/min": 1.002183437347412, "sampling/sampling_logp_difference/max": 0.3661486506462097, "sampling/sampling_logp_difference/mean": 0.014182055369019508, "step": 693, "step_time": 14.58046962798835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 847.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 214.0, "completions/mean_terminated_length": 214.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.295191356446594, "epoch": 0.01388, "frac_reward_zero_std": 0.25, "grad_norm": 1.1014759540557861, "kl": 0.7030089646577835, "learning_rate": 3.5107801787047088e-06, "loss": -0.0508, "num_tokens": 31846357.0, "reward": -0.5587500333786011, "reward_std": 0.6815847158432007, "rewards/rollout_reward_func/mean": -0.5587500333786011, "rewards/rollout_reward_func/std": 0.8472335934638977, "sampling/importance_sampling_ratio/max": 1.972219705581665, "sampling/importance_sampling_ratio/mean": 1.0677869319915771, "sampling/importance_sampling_ratio/min": 0.4178565740585327, "sampling/sampling_logp_difference/max": 0.6696071624755859, "sampling/sampling_logp_difference/mean": 0.05981457233428955, "step": 694, "step_time": 13.073688643009518 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 114.53125, "completions/mean_terminated_length": 114.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22600055299699306, "epoch": 0.0139, "frac_reward_zero_std": 0.0, "grad_norm": 0.45436277985572815, "kl": 1.0406898679211736, "learning_rate": 3.5014966500870244e-06, "loss": -0.0115, "num_tokens": 31893338.0, "reward": -0.3056250214576721, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.6464381217956543, "sampling/importance_sampling_ratio/mean": 1.0155260562896729, "sampling/importance_sampling_ratio/min": 0.282850980758667, "sampling/sampling_logp_difference/max": 1.3159918785095215, "sampling/sampling_logp_difference/mean": 0.05400490760803223, "step": 695, "step_time": 13.900303127011284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 134.1875, "completions/mean_terminated_length": 134.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16332284454256296, "epoch": 0.01392, "frac_reward_zero_std": 0.0, "grad_norm": 0.1948421150445938, "kl": 0.5781102441251278, "learning_rate": 3.492249376086523e-06, "loss": -0.0184, "num_tokens": 31941135.0, "reward": 0.07312498986721039, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.3240848779678345, "sampling/importance_sampling_ratio/mean": 1.0296342372894287, "sampling/importance_sampling_ratio/min": 0.992114782333374, "sampling/sampling_logp_difference/max": 0.28091949224472046, "sampling/sampling_logp_difference/mean": 0.01295323483645916, "step": 696, "step_time": 14.436351928998192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 82.46875, "completions/mean_terminated_length": 82.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2132766032591462, "epoch": 0.01394, "frac_reward_zero_std": 0.0, "grad_norm": 0.29383423924446106, "kl": 0.8131406214088202, "learning_rate": 3.483038478680996e-06, "loss": -0.0287, "num_tokens": 31988315.0, "reward": -0.053125008940696716, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.3041232824325562, "sampling/importance_sampling_ratio/mean": 1.0059680938720703, "sampling/importance_sampling_ratio/min": 0.4482005536556244, "sampling/sampling_logp_difference/max": 0.802377462387085, "sampling/sampling_logp_difference/mean": 0.03714323043823242, "step": 697, "step_time": 13.104477728978964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 838.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 89.375, "completions/mean_terminated_length": 89.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1695735533721745, "epoch": 0.01396, "frac_reward_zero_std": 0.0, "grad_norm": 0.2336946278810501, "kl": 1.1408769469708204, "learning_rate": 3.4738640793684103e-06, "loss": -0.0044, "num_tokens": 32034272.0, "reward": -0.11625000834465027, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.6676591634750366, "sampling/importance_sampling_ratio/mean": 1.0400859117507935, "sampling/importance_sampling_ratio/min": 0.47916874289512634, "sampling/sampling_logp_difference/max": 0.735506534576416, "sampling/sampling_logp_difference/mean": 0.03140759468078613, "step": 698, "step_time": 13.292554821979138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 59.71875, "completions/mean_terminated_length": 59.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20898518292233348, "epoch": 0.01398, "frac_reward_zero_std": 0.0, "grad_norm": 0.2678110897541046, "kl": 1.4171326979994774, "learning_rate": 3.464726299165293e-06, "loss": -0.0121, "num_tokens": 32080699.0, "reward": -0.18000000715255737, "reward_std": 1.0167031288146973, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.0414810180664062, "sampling/importance_sampling_ratio/mean": 0.9476807117462158, "sampling/importance_sampling_ratio/min": 0.5975966453552246, "sampling/sampling_logp_difference/max": 0.514491856098175, "sampling/sampling_logp_difference/mean": 0.03956988826394081, "step": 699, "step_time": 11.290026203998423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 69.96875, "completions/mean_terminated_length": 69.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2382613541558385, "epoch": 0.014, "frac_reward_zero_std": 0.0, "grad_norm": 0.3602612316608429, "kl": 0.8583254832774401, "learning_rate": 3.4556252586051436e-06, "loss": -0.0329, "num_tokens": 32127895.0, "reward": -0.3056250214576721, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685194015503, "sampling/importance_sampling_ratio/max": 1.3766454458236694, "sampling/importance_sampling_ratio/mean": 0.9949767589569092, "sampling/importance_sampling_ratio/min": 0.5579869151115417, "sampling/sampling_logp_difference/max": 0.5868062973022461, "sampling/sampling_logp_difference/mean": 0.038487523794174194, "step": 700, "step_time": 10.913360736012692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 125.1875, "completions/mean_terminated_length": 125.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18967640167102218, "epoch": 0.01402, "frac_reward_zero_std": 0.0, "grad_norm": 0.3239407241344452, "kl": 1.0843753851950169, "learning_rate": 3.4465610777368364e-06, "loss": -0.0076, "num_tokens": 32176490.0, "reward": -0.053125008940696716, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.2044615745544434, "sampling/importance_sampling_ratio/mean": 1.014235019683838, "sampling/importance_sampling_ratio/min": 0.8085627555847168, "sampling/sampling_logp_difference/max": 0.21254348754882812, "sampling/sampling_logp_difference/mean": 0.015614481642842293, "step": 701, "step_time": 13.696482754006865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 420.0, "completions/max_terminated_length": 420.0, "completions/mean_length": 33.59375, "completions/mean_terminated_length": 33.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17411978100426495, "epoch": 0.01404, "frac_reward_zero_std": 0.0, "grad_norm": 0.09197618067264557, "kl": 1.0015601087361574, "learning_rate": 3.4375338761230474e-06, "loss": -0.0123, "num_tokens": 32220994.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1799395084381104, "sampling/importance_sampling_ratio/mean": 0.9907631874084473, "sampling/importance_sampling_ratio/min": 0.6239551901817322, "sampling/sampling_logp_difference/max": 0.4752159118652344, "sampling/sampling_logp_difference/mean": 0.03394069895148277, "step": 702, "step_time": 11.051586611982202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 105.21875, "completions/mean_terminated_length": 105.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17209769459441304, "epoch": 0.01406, "frac_reward_zero_std": 0.0, "grad_norm": 0.23086309432983398, "kl": 1.081784295849502, "learning_rate": 3.428543772838667e-06, "loss": -0.0162, "num_tokens": 32267751.0, "reward": 0.009999990463256836, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.0404692888259888, "sampling/importance_sampling_ratio/mean": 0.9815956354141235, "sampling/importance_sampling_ratio/min": 0.6316559314727783, "sampling/sampling_logp_difference/max": 0.4587574005126953, "sampling/sampling_logp_difference/mean": 0.024874676018953323, "step": 703, "step_time": 11.55369029200665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 79.65625, "completions/mean_terminated_length": 79.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11317208432592452, "epoch": 0.01408, "frac_reward_zero_std": 0.0, "grad_norm": 0.2476935088634491, "kl": 0.7019649501889944, "learning_rate": 3.4195908864692364e-06, "loss": -0.0104, "num_tokens": 32312604.0, "reward": 0.26249998807907104, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": 0.26249998807907104, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.047497034072876, "sampling/importance_sampling_ratio/mean": 1.0080119371414185, "sampling/importance_sampling_ratio/min": 0.7099865674972534, "sampling/sampling_logp_difference/max": 0.34265899658203125, "sampling/sampling_logp_difference/mean": 0.012841342017054558, "step": 704, "step_time": 12.919038304993592 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 85.84375, "completions/mean_terminated_length": 85.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1629097965778783, "epoch": 0.0141, "frac_reward_zero_std": 0.0, "grad_norm": 0.27333688735961914, "kl": 1.600836431607604, "learning_rate": 3.410675335109376e-06, "loss": -0.0061, "num_tokens": 32358740.0, "reward": 0.07124999165534973, "reward_std": 1.016331434249878, "rewards/rollout_reward_func/mean": 0.07124999165534973, "rewards/rollout_reward_func/std": 1.0223935842514038, "sampling/importance_sampling_ratio/max": 1.1858785152435303, "sampling/importance_sampling_ratio/mean": 0.9898945689201355, "sampling/importance_sampling_ratio/min": 0.35172411799430847, "sampling/sampling_logp_difference/max": 1.0790780782699585, "sampling/sampling_logp_difference/mean": 0.03632018715143204, "step": 705, "step_time": 13.790639332983119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 167.3125, "completions/mean_terminated_length": 167.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1337653468362987, "epoch": 0.01412, "frac_reward_zero_std": 0.0, "grad_norm": 0.15220575034618378, "kl": 0.6917563211172819, "learning_rate": 3.4017972363612374e-06, "loss": -0.0392, "num_tokens": 32407382.0, "reward": 0.2618749737739563, "reward_std": 1.0259160995483398, "rewards/rollout_reward_func/mean": 0.2618749737739563, "rewards/rollout_reward_func/std": 0.9930903911590576, "sampling/importance_sampling_ratio/max": 1.789339542388916, "sampling/importance_sampling_ratio/mean": 1.0289795398712158, "sampling/importance_sampling_ratio/min": 0.5508586764335632, "sampling/sampling_logp_difference/max": 0.5967855453491211, "sampling/sampling_logp_difference/mean": 0.020842503756284714, "step": 706, "step_time": 14.295145807001973 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 94.84375, "completions/mean_terminated_length": 94.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14886400592513382, "epoch": 0.01414, "frac_reward_zero_std": 0.0, "grad_norm": 0.3211457133293152, "kl": 0.630776746198535, "learning_rate": 3.392956707332945e-06, "loss": -0.0157, "num_tokens": 32452829.0, "reward": 0.1993749886751175, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.4229915142059326, "sampling/importance_sampling_ratio/mean": 0.9947296380996704, "sampling/importance_sampling_ratio/min": 0.5083930492401123, "sampling/sampling_logp_difference/max": 0.6996866464614868, "sampling/sampling_logp_difference/mean": 0.03697100654244423, "step": 707, "step_time": 13.126506083994173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1742.0, "completions/max_terminated_length": 1742.0, "completions/mean_length": 158.34375, "completions/mean_terminated_length": 158.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1515171485953033, "epoch": 0.01416, "frac_reward_zero_std": 0.0, "grad_norm": 0.25803983211517334, "kl": 1.1806901525706053, "learning_rate": 3.384153864637052e-06, "loss": -0.025, "num_tokens": 32502327.0, "reward": -0.18000000715255737, "reward_std": 1.0173444747924805, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.7023420333862305, "sampling/importance_sampling_ratio/mean": 0.9686287641525269, "sampling/importance_sampling_ratio/min": 0.5494011044502258, "sampling/sampling_logp_difference/max": 0.5984234809875488, "sampling/sampling_logp_difference/mean": 0.04695571959018707, "step": 708, "step_time": 16.419940060004592 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 63.65625, "completions/mean_terminated_length": 63.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15830920659936965, "epoch": 0.01418, "frac_reward_zero_std": 0.25, "grad_norm": 0.21004146337509155, "kl": 2.465100070461631, "learning_rate": 3.3753888243890024e-06, "loss": -0.0105, "num_tokens": 32548808.0, "reward": -0.3056250214576721, "reward_std": 0.6822478771209717, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.1843918561935425, "sampling/importance_sampling_ratio/mean": 0.9875568747520447, "sampling/importance_sampling_ratio/min": 0.4211314022541046, "sampling/sampling_logp_difference/max": 0.864799976348877, "sampling/sampling_logp_difference/mean": 0.03836529701948166, "step": 709, "step_time": 11.934931218012935 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 92.03125, "completions/mean_terminated_length": 92.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.155100999167189, "epoch": 0.0142, "frac_reward_zero_std": 0.0, "grad_norm": 0.7121841311454773, "kl": 2.996211776509881, "learning_rate": 3.3666617022056063e-06, "loss": -0.0051, "num_tokens": 32595178.0, "reward": -0.05375000834465027, "reward_std": 0.9984559416770935, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.1803061962127686, "sampling/importance_sampling_ratio/mean": 0.9618208408355713, "sampling/importance_sampling_ratio/min": 0.3018997013568878, "sampling/sampling_logp_difference/max": 1.2149410247802734, "sampling/sampling_logp_difference/mean": 0.04654354229569435, "step": 710, "step_time": 14.282125862999237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 137.0, "completions/mean_terminated_length": 137.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12913487968035042, "epoch": 0.01422, "frac_reward_zero_std": 0.0, "grad_norm": 0.30756253004074097, "kl": 0.7929653264582157, "learning_rate": 3.357972613203503e-06, "loss": -0.0051, "num_tokens": 32641807.0, "reward": 0.07249999046325684, "reward_std": 1.01747727394104, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.128006100654602, "sampling/importance_sampling_ratio/mean": 0.9980391263961792, "sampling/importance_sampling_ratio/min": 0.5515495538711548, "sampling/sampling_logp_difference/max": 0.5940632820129395, "sampling/sampling_logp_difference/mean": 0.020771553739905357, "step": 711, "step_time": 13.384690185012005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 81.21875, "completions/mean_terminated_length": 81.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18779710703529418, "epoch": 0.01424, "frac_reward_zero_std": 0.0, "grad_norm": 0.2125864028930664, "kl": 1.137658080086112, "learning_rate": 3.3493216719976483e-06, "loss": -0.0032, "num_tokens": 32688322.0, "reward": -0.3687500059604645, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.3687500059604645, "rewards/rollout_reward_func/std": 0.9512767195701599, "sampling/importance_sampling_ratio/max": 1.2290219068527222, "sampling/importance_sampling_ratio/mean": 0.9936469793319702, "sampling/importance_sampling_ratio/min": 0.4767780601978302, "sampling/sampling_logp_difference/max": 0.7405719757080078, "sampling/sampling_logp_difference/mean": 0.027070365846157074, "step": 712, "step_time": 11.900486571998044 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 721.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 93.40625, "completions/mean_terminated_length": 93.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13816195743856952, "epoch": 0.01426, "frac_reward_zero_std": 0.0, "grad_norm": 0.3303317725658417, "kl": 0.7273664865642786, "learning_rate": 3.340708992699808e-06, "loss": -0.0167, "num_tokens": 32734732.0, "reward": 0.07312498986721039, "reward_std": 0.9521816968917847, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.1428992748260498, "sampling/importance_sampling_ratio/mean": 0.9540911912918091, "sampling/importance_sampling_ratio/min": 0.3730590045452118, "sampling/sampling_logp_difference/max": 0.9861268997192383, "sampling/sampling_logp_difference/mean": 0.043263450264930725, "step": 713, "step_time": 12.316659769981925 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 53.6875, "completions/mean_terminated_length": 53.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1613993989303708, "epoch": 0.01428, "frac_reward_zero_std": 0.0, "grad_norm": 0.41703930497169495, "kl": 5.354685187339783, "learning_rate": 3.3321346889170426e-06, "loss": -0.005, "num_tokens": 32778424.0, "reward": 0.07249999046325684, "reward_std": 1.0705007314682007, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.2062053680419922, "sampling/importance_sampling_ratio/mean": 0.9649198651313782, "sampling/importance_sampling_ratio/min": 0.28873756527900696, "sampling/sampling_logp_difference/max": 1.2422809600830078, "sampling/sampling_logp_difference/mean": 0.0458366796374321, "step": 714, "step_time": 10.858904636988882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 97.375, "completions/mean_terminated_length": 97.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13821712345816195, "epoch": 0.0143, "frac_reward_zero_std": 0.0, "grad_norm": 0.107403464615345, "kl": 0.5725794583559036, "learning_rate": 3.323598873750216e-06, "loss": -0.0264, "num_tokens": 32824897.0, "reward": 0.00812499225139618, "reward_std": 1.0602223873138428, "rewards/rollout_reward_func/mean": 0.00812499225139618, "rewards/rollout_reward_func/std": 1.0242838859558105, "sampling/importance_sampling_ratio/max": 1.1780177354812622, "sampling/importance_sampling_ratio/mean": 1.017722249031067, "sampling/importance_sampling_ratio/min": 0.8485734462738037, "sampling/sampling_logp_difference/max": 0.16420972347259521, "sampling/sampling_logp_difference/mean": 0.011542638763785362, "step": 715, "step_time": 14.243891241007077 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 72.78125, "completions/mean_terminated_length": 72.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11981402023229748, "epoch": 0.01432, "frac_reward_zero_std": 0.0, "grad_norm": 0.13097725808620453, "kl": 1.500442260876298, "learning_rate": 3.315101659792499e-06, "loss": -0.0194, "num_tokens": 32869435.0, "reward": 0.1981249898672104, "reward_std": 0.9980893731117249, "rewards/rollout_reward_func/mean": 0.1981249898672104, "rewards/rollout_reward_func/std": 1.0069353580474854, "sampling/importance_sampling_ratio/max": 1.0618557929992676, "sampling/importance_sampling_ratio/mean": 0.9643510580062866, "sampling/importance_sampling_ratio/min": 0.4234074652194977, "sampling/sampling_logp_difference/max": 0.8592791557312012, "sampling/sampling_logp_difference/mean": 0.03900529071688652, "step": 716, "step_time": 13.41491003100964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 44.40625, "completions/mean_terminated_length": 44.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14394288859330118, "epoch": 0.01434, "frac_reward_zero_std": 0.0, "grad_norm": 0.26673054695129395, "kl": 0.6188336554914713, "learning_rate": 3.306643159127889e-06, "loss": -0.0087, "num_tokens": 32913592.0, "reward": -0.11625000834465027, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.4353047609329224, "sampling/importance_sampling_ratio/mean": 1.0164847373962402, "sampling/importance_sampling_ratio/min": 0.6718646883964539, "sampling/sampling_logp_difference/max": 0.3978085517883301, "sampling/sampling_logp_difference/mean": 0.021256640553474426, "step": 717, "step_time": 10.917721152996819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 75.875, "completions/mean_terminated_length": 75.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14846058748662472, "epoch": 0.01436, "frac_reward_zero_std": 0.0, "grad_norm": 0.14858025312423706, "kl": 0.9293948113918304, "learning_rate": 3.2982234833297273e-06, "loss": -0.0176, "num_tokens": 32959179.0, "reward": -0.05375000834465027, "reward_std": 1.0705006122589111, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.2726093530654907, "sampling/importance_sampling_ratio/mean": 0.9824270009994507, "sampling/importance_sampling_ratio/min": 0.3334031105041504, "sampling/sampling_logp_difference/max": 1.1151809692382812, "sampling/sampling_logp_difference/mean": 0.04531756415963173, "step": 718, "step_time": 12.092342098010704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 136.5625, "completions/mean_terminated_length": 136.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14883967849891633, "epoch": 0.01438, "frac_reward_zero_std": 0.0, "grad_norm": 1.0942150354385376, "kl": 0.9090752964839339, "learning_rate": 3.28984274345923e-06, "loss": -0.0059, "num_tokens": 33006812.0, "reward": 0.1981249898672104, "reward_std": 0.9345479607582092, "rewards/rollout_reward_func/mean": 0.1981249898672104, "rewards/rollout_reward_func/std": 1.0069226026535034, "sampling/importance_sampling_ratio/max": 1.5474714040756226, "sampling/importance_sampling_ratio/mean": 1.0128109455108643, "sampling/importance_sampling_ratio/min": 0.5614545941352844, "sampling/sampling_logp_difference/max": 0.5949995517730713, "sampling/sampling_logp_difference/mean": 0.025643043220043182, "step": 719, "step_time": 13.493493956011662 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 75.4375, "completions/mean_terminated_length": 75.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1166238731238991, "epoch": 0.0144, "frac_reward_zero_std": 0.0, "grad_norm": 0.08526962995529175, "kl": 0.7179480642080307, "learning_rate": 3.281501050064022e-06, "loss": -0.0196, "num_tokens": 33052539.0, "reward": 0.1356249898672104, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.1606683731079102, "sampling/importance_sampling_ratio/mean": 0.9932503700256348, "sampling/importance_sampling_ratio/min": 0.22795897722244263, "sampling/sampling_logp_difference/max": 1.520263671875, "sampling/sampling_logp_difference/mean": 0.028402874246239662, "step": 720, "step_time": 11.258468151012494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 71.78125, "completions/mean_terminated_length": 71.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1859710180433467, "epoch": 0.01442, "frac_reward_zero_std": 0.0, "grad_norm": 0.22131910920143127, "kl": 0.7720294836908579, "learning_rate": 3.2731985131766765e-06, "loss": -0.023, "num_tokens": 33099382.0, "reward": -0.11937500536441803, "reward_std": 1.0223926305770874, "rewards/rollout_reward_func/mean": -0.11937500536441803, "rewards/rollout_reward_func/std": 1.0145455598831177, "sampling/importance_sampling_ratio/max": 1.177686095237732, "sampling/importance_sampling_ratio/mean": 1.0064126253128052, "sampling/importance_sampling_ratio/min": 0.46158236265182495, "sampling/sampling_logp_difference/max": 0.793096661567688, "sampling/sampling_logp_difference/mean": 0.03210020437836647, "step": 721, "step_time": 12.03864354097459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 76.03125, "completions/mean_terminated_length": 76.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14281999471131712, "epoch": 0.01444, "frac_reward_zero_std": 0.0, "grad_norm": 0.38064002990722656, "kl": 0.8420963864773512, "learning_rate": 3.2649352423132675e-06, "loss": 0.0001, "num_tokens": 33144214.0, "reward": -0.05375000834465027, "reward_std": 1.0531628131866455, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.2467412948608398, "sampling/importance_sampling_ratio/mean": 1.007596492767334, "sampling/importance_sampling_ratio/min": 0.5495164394378662, "sampling/sampling_logp_difference/max": 0.5987372398376465, "sampling/sampling_logp_difference/mean": 0.02352011576294899, "step": 722, "step_time": 10.967893252010981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 87.6875, "completions/mean_terminated_length": 87.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17477361229248345, "epoch": 0.01446, "frac_reward_zero_std": 0.0, "grad_norm": 0.2289903461933136, "kl": 0.6977330055087805, "learning_rate": 3.256711346471925e-06, "loss": -0.0154, "num_tokens": 33190909.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1206270456314087, "sampling/importance_sampling_ratio/mean": 0.9852433204650879, "sampling/importance_sampling_ratio/min": 0.4918791949748993, "sampling/sampling_logp_difference/max": 0.7316694259643555, "sampling/sampling_logp_difference/mean": 0.028667952865362167, "step": 723, "step_time": 14.432407773012528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 88.6875, "completions/mean_terminated_length": 88.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16279834159649909, "epoch": 0.01448, "frac_reward_zero_std": 0.0, "grad_norm": 0.4267660677433014, "kl": 0.6396651193499565, "learning_rate": 3.2485269341313947e-06, "loss": 0.0064, "num_tokens": 33235897.0, "reward": -0.11687500774860382, "reward_std": 0.9056781530380249, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.645756483078003, "sampling/importance_sampling_ratio/mean": 1.0027374029159546, "sampling/importance_sampling_ratio/min": 0.5364795327186584, "sampling/sampling_logp_difference/max": 0.8090684413909912, "sampling/sampling_logp_difference/mean": 0.047419473528862, "step": 724, "step_time": 13.569100437991438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 157.90625, "completions/mean_terminated_length": 157.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16268973355181515, "epoch": 0.0145, "frac_reward_zero_std": 0.0, "grad_norm": 0.18919654190540314, "kl": 1.2357143387198448, "learning_rate": 3.240382113249607e-06, "loss": -0.0364, "num_tokens": 33285921.0, "reward": -0.11625000834465027, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.7790355682373047, "sampling/importance_sampling_ratio/mean": 1.0701828002929688, "sampling/importance_sampling_ratio/min": 0.7021833658218384, "sampling/sampling_logp_difference/max": 0.5506742000579834, "sampling/sampling_logp_difference/mean": 0.03470796346664429, "step": 725, "step_time": 13.317054607992759 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 81.375, "completions/mean_terminated_length": 81.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15182560007087886, "epoch": 0.01452, "frac_reward_zero_std": 0.0, "grad_norm": 0.3894665837287903, "kl": 0.6373760774731636, "learning_rate": 3.232276991262254e-06, "loss": -0.013, "num_tokens": 33330595.0, "reward": 0.07249999046325684, "reward_std": 1.0535087585449219, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.1707632541656494, "sampling/importance_sampling_ratio/mean": 1.0126795768737793, "sampling/importance_sampling_ratio/min": 0.7869181036949158, "sampling/sampling_logp_difference/max": 0.23882484436035156, "sampling/sampling_logp_difference/mean": 0.012299828231334686, "step": 726, "step_time": 14.066034763010975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 58.75, "completions/mean_terminated_length": 58.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2184732120949775, "epoch": 0.01454, "frac_reward_zero_std": 0.0, "grad_norm": 0.16741076111793518, "kl": 0.7325159627944231, "learning_rate": 3.224211675081378e-06, "loss": -0.0211, "num_tokens": 33377211.0, "reward": -0.24250000715255737, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.2617409229278564, "sampling/importance_sampling_ratio/mean": 0.9879536628723145, "sampling/importance_sampling_ratio/min": 0.5505754351615906, "sampling/sampling_logp_difference/max": 0.5999956130981445, "sampling/sampling_logp_difference/mean": 0.027787774801254272, "step": 727, "step_time": 11.271195398025156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 62.125, "completions/mean_terminated_length": 62.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18613584013655782, "epoch": 0.01456, "frac_reward_zero_std": 0.0, "grad_norm": 0.23009689152240753, "kl": 0.6086405143141747, "learning_rate": 3.216186271093948e-06, "loss": -0.0058, "num_tokens": 33423129.0, "reward": -0.17937502264976501, "reward_std": 0.9160174131393433, "rewards/rollout_reward_func/mean": -0.17937502264976501, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.1702871322631836, "sampling/importance_sampling_ratio/mean": 0.9923120141029358, "sampling/importance_sampling_ratio/min": 0.4861666262149811, "sampling/sampling_logp_difference/max": 0.7239499092102051, "sampling/sampling_logp_difference/mean": 0.031624685972929, "step": 728, "step_time": 11.470629201969132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 60.4375, "completions/mean_terminated_length": 60.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13515427603852004, "epoch": 0.01458, "frac_reward_zero_std": 0.0, "grad_norm": 0.21635636687278748, "kl": 0.9614330288022757, "learning_rate": 3.2082008851604685e-06, "loss": -0.0141, "num_tokens": 33467886.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.070770025253296, "sampling/importance_sampling_ratio/mean": 0.9828662276268005, "sampling/importance_sampling_ratio/min": 0.5762220621109009, "sampling/sampling_logp_difference/max": 0.5519065856933594, "sampling/sampling_logp_difference/mean": 0.02690986543893814, "step": 729, "step_time": 11.929788750006992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 119.5625, "completions/mean_terminated_length": 119.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1503030713647604, "epoch": 0.0146, "frac_reward_zero_std": 0.0, "grad_norm": 0.22626347839832306, "kl": 0.6456222645938396, "learning_rate": 3.200255622613578e-06, "loss": -0.028, "num_tokens": 33514530.0, "reward": 0.009999990463256836, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.207409143447876, "sampling/importance_sampling_ratio/mean": 1.0305095911026, "sampling/importance_sampling_ratio/min": 0.9798312187194824, "sampling/sampling_logp_difference/max": 0.1608644425868988, "sampling/sampling_logp_difference/mean": 0.012167717330157757, "step": 730, "step_time": 13.58395971799473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 847.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 91.03125, "completions/mean_terminated_length": 91.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1614361631218344, "epoch": 0.01462, "frac_reward_zero_std": 0.0, "grad_norm": 0.26759740710258484, "kl": 0.6150404494255781, "learning_rate": 3.192350588256662e-06, "loss": -0.0135, "num_tokens": 33560737.0, "reward": 0.1993749886751175, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.2630398273468018, "sampling/importance_sampling_ratio/mean": 1.0314631462097168, "sampling/importance_sampling_ratio/min": 0.8155295848846436, "sampling/sampling_logp_difference/max": 0.23355364799499512, "sampling/sampling_logp_difference/mean": 0.01666448265314102, "step": 731, "step_time": 12.78027177201875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 87.40625, "completions/mean_terminated_length": 87.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12865178659558296, "epoch": 0.01464, "frac_reward_zero_std": 0.0, "grad_norm": 0.08789172023534775, "kl": 0.6829754877835512, "learning_rate": 3.1844858863624657e-06, "loss": -0.0101, "num_tokens": 33607033.0, "reward": 0.008749991655349731, "reward_std": 1.0614171028137207, "rewards/rollout_reward_func/mean": 0.008749991655349731, "rewards/rollout_reward_func/std": 1.0249019861221313, "sampling/importance_sampling_ratio/max": 1.1476919651031494, "sampling/importance_sampling_ratio/mean": 1.017943263053894, "sampling/importance_sampling_ratio/min": 0.9116555452346802, "sampling/sampling_logp_difference/max": 0.12009881436824799, "sampling/sampling_logp_difference/mean": 0.009842835366725922, "step": 732, "step_time": 11.869923240024946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 53.0625, "completions/mean_terminated_length": 53.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1355877130990848, "epoch": 0.01466, "frac_reward_zero_std": 0.0, "grad_norm": 0.20029404759407043, "kl": 0.7584051378071308, "learning_rate": 3.1766616206717248e-06, "loss": -0.0091, "num_tokens": 33650687.0, "reward": 0.19874998927116394, "reward_std": 0.9622916579246521, "rewards/rollout_reward_func/mean": 0.19874998927116394, "rewards/rollout_reward_func/std": 1.0074424743652344, "sampling/importance_sampling_ratio/max": 1.0363625288009644, "sampling/importance_sampling_ratio/mean": 0.9755740165710449, "sampling/importance_sampling_ratio/min": 0.53013676404953, "sampling/sampling_logp_difference/max": 0.40234947204589844, "sampling/sampling_logp_difference/mean": 0.027362439781427383, "step": 733, "step_time": 14.067476289004844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 75.71875, "completions/mean_terminated_length": 75.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14549578353762627, "epoch": 0.01468, "frac_reward_zero_std": 0.0, "grad_norm": 0.1203358992934227, "kl": 0.5230886694043875, "learning_rate": 3.168877894391792e-06, "loss": -0.002, "num_tokens": 33695589.0, "reward": 0.07312498986721039, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.2283824682235718, "sampling/importance_sampling_ratio/mean": 1.0251840353012085, "sampling/importance_sampling_ratio/min": 0.7267317771911621, "sampling/sampling_logp_difference/max": 0.3226306438446045, "sampling/sampling_logp_difference/mean": 0.01755734346807003, "step": 734, "step_time": 13.88503108300938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 119.8125, "completions/mean_terminated_length": 119.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17948633851483464, "epoch": 0.0147, "frac_reward_zero_std": 0.0, "grad_norm": 0.9817483425140381, "kl": 7.958876840770245, "learning_rate": 3.16113481019528e-06, "loss": -0.0376, "num_tokens": 33743974.0, "reward": 0.009999990463256836, "reward_std": 1.0797353982925415, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.253926157951355, "sampling/importance_sampling_ratio/mean": 0.9998708367347717, "sampling/importance_sampling_ratio/min": 0.37037116289138794, "sampling/sampling_logp_difference/max": 0.9931178092956543, "sampling/sampling_logp_difference/mean": 0.038607288151979446, "step": 735, "step_time": 14.368337495994638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 29.25, "completions/mean_terminated_length": 29.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17533789272420108, "epoch": 0.01472, "frac_reward_zero_std": 0.0, "grad_norm": 0.37199297547340393, "kl": 0.8196496795862913, "learning_rate": 3.1534324702187026e-06, "loss": -0.0003, "num_tokens": 33787659.0, "reward": -0.05375000834465027, "reward_std": 1.01747727394104, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.0234845876693726, "sampling/importance_sampling_ratio/max": 1.4727505445480347, "sampling/importance_sampling_ratio/mean": 1.0164152383804321, "sampling/importance_sampling_ratio/min": 0.619593620300293, "sampling/sampling_logp_difference/max": 0.47855567932128906, "sampling/sampling_logp_difference/mean": 0.026922712102532387, "step": 736, "step_time": 11.773551220991067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 111.0, "completions/mean_terminated_length": 111.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.144456326845102, "epoch": 0.01474, "frac_reward_zero_std": 0.0, "grad_norm": 0.2544025480747223, "kl": 0.9197674682363868, "learning_rate": 3.1457709760611314e-06, "loss": -0.0012, "num_tokens": 33833159.0, "reward": 0.07249999046325684, "reward_std": 1.0705006122589111, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.0942329168319702, "sampling/importance_sampling_ratio/mean": 0.992520272731781, "sampling/importance_sampling_ratio/min": 0.4798503816127777, "sampling/sampling_logp_difference/max": 0.7342333793640137, "sampling/sampling_logp_difference/mean": 0.02190462499856949, "step": 737, "step_time": 13.818842575004965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 122.65625, "completions/mean_terminated_length": 122.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.181602721568197, "epoch": 0.01476, "frac_reward_zero_std": 0.0, "grad_norm": 0.27758485078811646, "kl": 0.6549361143261194, "learning_rate": 3.1381504287828525e-06, "loss": -0.0255, "num_tokens": 33881110.0, "reward": -0.053125008940696716, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.1528457403182983, "sampling/importance_sampling_ratio/mean": 0.9817405939102173, "sampling/importance_sampling_ratio/min": 0.46968403458595276, "sampling/sampling_logp_difference/max": 0.7841489911079407, "sampling/sampling_logp_difference/mean": 0.031148988753557205, "step": 738, "step_time": 12.463115766004194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 703.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 44.40625, "completions/mean_terminated_length": 44.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16640070476569235, "epoch": 0.01478, "frac_reward_zero_std": 0.0, "grad_norm": 0.2256813943386078, "kl": 0.7190607935190201, "learning_rate": 3.130570928904039e-06, "loss": -0.0157, "num_tokens": 33925001.0, "reward": 0.1356249898672104, "reward_std": 1.0257649421691895, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175585746765137, "sampling/importance_sampling_ratio/max": 1.2798430919647217, "sampling/importance_sampling_ratio/mean": 1.016762614250183, "sampling/importance_sampling_ratio/min": 0.7015776038169861, "sampling/sampling_logp_difference/max": 0.3544297218322754, "sampling/sampling_logp_difference/mean": 0.020794548094272614, "step": 739, "step_time": 11.707217381997907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 69.6875, "completions/mean_terminated_length": 69.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1312443856149912, "epoch": 0.0148, "frac_reward_zero_std": 0.25, "grad_norm": 0.3676955997943878, "kl": 0.6146061960607767, "learning_rate": 3.1230325764034134e-06, "loss": 0.0013, "num_tokens": 33968625.0, "reward": 0.3881250023841858, "reward_std": 0.7921464443206787, "rewards/rollout_reward_func/mean": 0.3881250023841858, "rewards/rollout_reward_func/std": 0.9508550763130188, "sampling/importance_sampling_ratio/max": 1.490468978881836, "sampling/importance_sampling_ratio/mean": 1.0393503904342651, "sampling/importance_sampling_ratio/min": 0.9843296408653259, "sampling/sampling_logp_difference/max": 0.37858402729034424, "sampling/sampling_logp_difference/mean": 0.015266237780451775, "step": 740, "step_time": 10.94002911800635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 128.5625, "completions/mean_terminated_length": 128.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1716385455802083, "epoch": 0.01482, "frac_reward_zero_std": 0.0, "grad_norm": 0.37774965167045593, "kl": 0.6114841382950544, "learning_rate": 3.1155354707169467e-06, "loss": -0.0181, "num_tokens": 34016246.0, "reward": -0.11625000834465027, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0870320796966553, "sampling/importance_sampling_ratio/mean": 1.0213004350662231, "sampling/importance_sampling_ratio/min": 0.981286883354187, "sampling/sampling_logp_difference/max": 0.07251912355422974, "sampling/sampling_logp_difference/mean": 0.008616665378212929, "step": 741, "step_time": 14.137024474010104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 438.0, "completions/max_terminated_length": 438.0, "completions/mean_length": 54.9375, "completions/mean_terminated_length": 54.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19304374465718865, "epoch": 0.01484, "frac_reward_zero_std": 0.0, "grad_norm": 0.07507865875959396, "kl": 1.1368140503764153, "learning_rate": 3.108079710736526e-06, "loss": -0.0101, "num_tokens": 34062243.0, "reward": -0.053750038146972656, "reward_std": 0.8519768714904785, "rewards/rollout_reward_func/mean": -0.053750038146972656, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.1685380935668945, "sampling/importance_sampling_ratio/mean": 1.007516622543335, "sampling/importance_sampling_ratio/min": 0.5544774532318115, "sampling/sampling_logp_difference/max": 0.5895042419433594, "sampling/sampling_logp_difference/mean": 0.032854750752449036, "step": 742, "step_time": 10.802277697024692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1697.0, "completions/max_terminated_length": 1697.0, "completions/mean_length": 144.03125, "completions/mean_terminated_length": 144.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21512001939117908, "epoch": 0.01486, "frac_reward_zero_std": 0.25, "grad_norm": 0.18742018938064575, "kl": 1.2583929151296616, "learning_rate": 3.1006653948086677e-06, "loss": -0.0026, "num_tokens": 34111917.0, "reward": -0.43187499046325684, "reward_std": 0.7650657892227173, "rewards/rollout_reward_func/mean": -0.43187499046325684, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.2459522485733032, "sampling/importance_sampling_ratio/mean": 1.0064712762832642, "sampling/importance_sampling_ratio/min": 0.13733196258544922, "sampling/sampling_logp_difference/max": 1.1686210632324219, "sampling/sampling_logp_difference/mean": 0.050798751413822174, "step": 743, "step_time": 16.033262863020354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 93.03125, "completions/mean_terminated_length": 93.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21731155226007104, "epoch": 0.01488, "frac_reward_zero_std": 0.0, "grad_norm": 0.39872458577156067, "kl": 0.9943700339645147, "learning_rate": 3.0932926207332104e-06, "loss": -0.0227, "num_tokens": 34158912.0, "reward": -0.11625000834465027, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.2696343660354614, "sampling/importance_sampling_ratio/mean": 0.9908826947212219, "sampling/importance_sampling_ratio/min": 0.5504287481307983, "sampling/sampling_logp_difference/max": 0.5970416069030762, "sampling/sampling_logp_difference/mean": 0.03658074885606766, "step": 744, "step_time": 13.95903468898905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 108.1875, "completions/mean_terminated_length": 108.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20722419605590403, "epoch": 0.0149, "frac_reward_zero_std": 0.0, "grad_norm": 0.36903393268585205, "kl": 0.6990658491849899, "learning_rate": 3.0859614857620308e-06, "loss": -0.0276, "num_tokens": 34205669.0, "reward": -0.053125008940696716, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.269592046737671, "sampling/importance_sampling_ratio/mean": 0.9780895709991455, "sampling/importance_sampling_ratio/min": 0.5572826266288757, "sampling/sampling_logp_difference/max": 0.5849094390869141, "sampling/sampling_logp_difference/mean": 0.03349117562174797, "step": 745, "step_time": 12.974768844011123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 89.9375, "completions/mean_terminated_length": 89.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1458069123327732, "epoch": 0.01492, "frac_reward_zero_std": 0.0, "grad_norm": 0.11902708560228348, "kl": 0.7548163589090109, "learning_rate": 3.0786720865977503e-06, "loss": -0.0109, "num_tokens": 34250927.0, "reward": 0.1356249898672104, "reward_std": 0.9617735147476196, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.0895419120788574, "sampling/importance_sampling_ratio/mean": 0.9870713949203491, "sampling/importance_sampling_ratio/min": 0.546985924243927, "sampling/sampling_logp_difference/max": 0.629051685333252, "sampling/sampling_logp_difference/mean": 0.02299349755048752, "step": 746, "step_time": 11.50263726599951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 51.15625, "completions/mean_terminated_length": 51.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1808934318833053, "epoch": 0.01494, "frac_reward_zero_std": 0.0, "grad_norm": 0.3669224977493286, "kl": 0.945342056453228, "learning_rate": 3.0714245193924757e-06, "loss": -0.0147, "num_tokens": 34296399.0, "reward": -0.11625000834465027, "reward_std": 0.8798136115074158, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1666275262832642, "sampling/importance_sampling_ratio/mean": 0.9918704032897949, "sampling/importance_sampling_ratio/min": 0.5553978681564331, "sampling/sampling_logp_difference/max": 0.5914652347564697, "sampling/sampling_logp_difference/mean": 0.02682768739759922, "step": 747, "step_time": 11.181690775010793 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 59.59375, "completions/mean_terminated_length": 59.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1296325067523867, "epoch": 0.01496, "frac_reward_zero_std": 0.0, "grad_norm": 0.3629261553287506, "kl": 1.7570812106132507, "learning_rate": 3.064218879746515e-06, "loss": -0.0022, "num_tokens": 34339829.0, "reward": 0.2618749737739563, "reward_std": 1.0449373722076416, "rewards/rollout_reward_func/mean": 0.2618749737739563, "rewards/rollout_reward_func/std": 0.9930903911590576, "sampling/importance_sampling_ratio/max": 1.071682095527649, "sampling/importance_sampling_ratio/mean": 0.9736433625221252, "sampling/importance_sampling_ratio/min": 0.4392562508583069, "sampling/sampling_logp_difference/max": 0.8410381078720093, "sampling/sampling_logp_difference/mean": 0.03275301679968834, "step": 748, "step_time": 11.613720634995843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 811.0, "completions/max_terminated_length": 811.0, "completions/mean_length": 96.25, "completions/mean_terminated_length": 96.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21137594198808074, "epoch": 0.01498, "frac_reward_zero_std": 0.0, "grad_norm": 0.40864482522010803, "kl": 2.07413474842906, "learning_rate": 3.057055262707128e-06, "loss": -0.0348, "num_tokens": 34388565.0, "reward": -0.49562501907348633, "reward_std": 0.878955602645874, "rewards/rollout_reward_func/mean": -0.49562501907348633, "rewards/rollout_reward_func/std": 0.8875880241394043, "sampling/importance_sampling_ratio/max": 1.3698099851608276, "sampling/importance_sampling_ratio/mean": 0.9575692415237427, "sampling/importance_sampling_ratio/min": 0.5548229217529297, "sampling/sampling_logp_difference/max": 0.5893964767456055, "sampling/sampling_logp_difference/mean": 0.04960763454437256, "step": 749, "step_time": 13.329260422986408 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 80.9375, "completions/mean_terminated_length": 80.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1425734788645059, "epoch": 0.015, "frac_reward_zero_std": 0.0, "grad_norm": 0.4473753571510315, "kl": 0.5913108456879854, "learning_rate": 3.0499337627672667e-06, "loss": -0.0096, "num_tokens": 34434040.0, "reward": 0.13624998927116394, "reward_std": 0.9626314640045166, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1882151365280151, "sampling/importance_sampling_ratio/mean": 1.0325796604156494, "sampling/importance_sampling_ratio/min": 1.0052754878997803, "sampling/sampling_logp_difference/max": 0.14585402607917786, "sampling/sampling_logp_difference/mean": 0.012467521242797375, "step": 750, "step_time": 13.590171718977217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 85.4375, "completions/mean_terminated_length": 85.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16220884420908988, "epoch": 0.01502, "frac_reward_zero_std": 0.0, "grad_norm": 0.22292596101760864, "kl": 1.999214991927147, "learning_rate": 3.042854473864327e-06, "loss": -0.0366, "num_tokens": 34479730.0, "reward": -0.05437500774860382, "reward_std": 1.0164865255355835, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228137969970703, "sampling/importance_sampling_ratio/max": 1.3939542770385742, "sampling/importance_sampling_ratio/mean": 0.9843388795852661, "sampling/importance_sampling_ratio/min": 0.5423992872238159, "sampling/sampling_logp_difference/max": 0.6117191314697266, "sampling/sampling_logp_difference/mean": 0.03420737385749817, "step": 751, "step_time": 13.364626694994513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 94.125, "completions/mean_terminated_length": 94.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17650310089811683, "epoch": 0.01504, "frac_reward_zero_std": 0.0, "grad_norm": 0.37033650279045105, "kl": 1.2446265257894993, "learning_rate": 3.0358174893789156e-06, "loss": -0.0261, "num_tokens": 34526387.0, "reward": 0.13499999046325684, "reward_std": 1.0254149436950684, "rewards/rollout_reward_func/mean": 0.13499999046325684, "rewards/rollout_reward_func/std": 1.0170165300369263, "sampling/importance_sampling_ratio/max": 1.3090823888778687, "sampling/importance_sampling_ratio/mean": 0.990433394908905, "sampling/importance_sampling_ratio/min": 0.4719565808773041, "sampling/sampling_logp_difference/max": 0.7509009838104248, "sampling/sampling_logp_difference/mean": 0.03472078591585159, "step": 752, "step_time": 11.943371888999536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 53.0, "completions/mean_terminated_length": 53.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12852917308919132, "epoch": 0.01506, "frac_reward_zero_std": 0.0, "grad_norm": 0.16811661422252655, "kl": 1.8702851887792349, "learning_rate": 3.0288229021336165e-06, "loss": -0.0067, "num_tokens": 34570045.0, "reward": -0.11625000834465027, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1494860649108887, "sampling/importance_sampling_ratio/mean": 0.9882435202598572, "sampling/importance_sampling_ratio/min": 0.4216294586658478, "sampling/sampling_logp_difference/max": 0.8635287284851074, "sampling/sampling_logp_difference/mean": 0.028796372935175896, "step": 753, "step_time": 14.332529298000736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 181.21875, "completions/mean_terminated_length": 181.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1354326270520687, "epoch": 0.01508, "frac_reward_zero_std": 0.0, "grad_norm": 0.8457238674163818, "kl": 1.2927508354187012, "learning_rate": 3.021870804391763e-06, "loss": -0.0143, "num_tokens": 34620252.0, "reward": 0.07312498986721039, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.5418500900268555, "sampling/importance_sampling_ratio/mean": 1.0052841901779175, "sampling/importance_sampling_ratio/min": 0.5485270023345947, "sampling/sampling_logp_difference/max": 0.6006221771240234, "sampling/sampling_logp_difference/mean": 0.03524436801671982, "step": 754, "step_time": 14.185575572017115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 111.0, "completions/mean_terminated_length": 111.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1424095332622528, "epoch": 0.0151, "frac_reward_zero_std": 0.0, "grad_norm": 0.32400646805763245, "kl": 1.4056362612172961, "learning_rate": 3.014961287856224e-06, "loss": -0.012, "num_tokens": 34667067.0, "reward": -0.11625000834465027, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.163379192352295, "sampling/importance_sampling_ratio/mean": 0.979640007019043, "sampling/importance_sampling_ratio/min": 0.3000951111316681, "sampling/sampling_logp_difference/max": 1.3635029792785645, "sampling/sampling_logp_difference/mean": 0.033602550625801086, "step": 755, "step_time": 14.883328515010362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 47.78125, "completions/mean_terminated_length": 47.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17306772363372147, "epoch": 0.01512, "frac_reward_zero_std": 0.0, "grad_norm": 0.16837196052074432, "kl": 0.6239775530993938, "learning_rate": 3.0080944436681924e-06, "loss": -0.0107, "num_tokens": 34712259.0, "reward": -0.11625000834465027, "reward_std": 0.9712425470352173, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.108341097831726, "sampling/importance_sampling_ratio/mean": 0.9878889322280884, "sampling/importance_sampling_ratio/min": 0.5500640273094177, "sampling/sampling_logp_difference/max": 0.5977811813354492, "sampling/sampling_logp_difference/mean": 0.025482449680566788, "step": 756, "step_time": 10.981653941998957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 93.0, "completions/mean_terminated_length": 93.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09503239137120545, "epoch": 0.01514, "frac_reward_zero_std": 0.0, "grad_norm": 0.25213849544525146, "kl": 0.5998181384056807, "learning_rate": 3.001270362405987e-06, "loss": -0.0234, "num_tokens": 34758242.0, "reward": 0.13624998927116394, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1386594772338867, "sampling/importance_sampling_ratio/mean": 1.007965087890625, "sampling/importance_sampling_ratio/min": 0.6652446389198303, "sampling/sampling_logp_difference/max": 0.4209573268890381, "sampling/sampling_logp_difference/mean": 0.012819398194551468, "step": 757, "step_time": 11.59634432400344 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 123.625, "completions/mean_terminated_length": 123.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1558038096409291, "epoch": 0.01516, "frac_reward_zero_std": 0.0, "grad_norm": 0.27249011397361755, "kl": 0.9047639667987823, "learning_rate": 2.9944891340838517e-06, "loss": -0.0279, "num_tokens": 34807260.0, "reward": -0.11687500774860382, "reward_std": 1.0259160995483398, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.0525803565979004, "sampling/importance_sampling_ratio/mean": 0.9803807735443115, "sampling/importance_sampling_ratio/min": 0.3053155839443207, "sampling/sampling_logp_difference/max": 1.1996225118637085, "sampling/sampling_logp_difference/mean": 0.034087661653757095, "step": 758, "step_time": 12.376055987013387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 102.84375, "completions/mean_terminated_length": 102.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10308708879165351, "epoch": 0.01518, "frac_reward_zero_std": 0.0, "grad_norm": 0.24374231696128845, "kl": 0.8418265115469694, "learning_rate": 2.9877508481507723e-06, "loss": -0.012, "num_tokens": 34853927.0, "reward": 0.008749991655349731, "reward_std": 1.07840895652771, "rewards/rollout_reward_func/mean": 0.008749991655349731, "rewards/rollout_reward_func/std": 1.0249019861221313, "sampling/importance_sampling_ratio/max": 1.0537374019622803, "sampling/importance_sampling_ratio/mean": 0.9735949039459229, "sampling/importance_sampling_ratio/min": 0.29296326637268066, "sampling/sampling_logp_difference/max": 1.227684497833252, "sampling/sampling_logp_difference/mean": 0.03304773196578026, "step": 759, "step_time": 11.48661728901061 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 152.71875, "completions/mean_terminated_length": 152.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14332589879631996, "epoch": 0.0152, "frac_reward_zero_std": 0.0, "grad_norm": 0.5452749729156494, "kl": 0.7483772588893771, "learning_rate": 2.981055593489295e-06, "loss": -0.0195, "num_tokens": 34902685.0, "reward": -0.11812500655651093, "reward_std": 1.0242091417312622, "rewards/rollout_reward_func/mean": -0.11812500655651093, "rewards/rollout_reward_func/std": 1.0159801244735718, "sampling/importance_sampling_ratio/max": 1.0974959135055542, "sampling/importance_sampling_ratio/mean": 0.9851375818252563, "sampling/importance_sampling_ratio/min": 0.665368914604187, "sampling/sampling_logp_difference/max": 0.4249610900878906, "sampling/sampling_logp_difference/mean": 0.01956680417060852, "step": 760, "step_time": 13.814496473991312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 204.4375, "completions/mean_terminated_length": 204.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1160535755334422, "epoch": 0.01522, "frac_reward_zero_std": 0.0, "grad_norm": 0.6666576862335205, "kl": 0.5996312350034714, "learning_rate": 2.974403458414355e-06, "loss": -0.0194, "num_tokens": 34952896.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.3293579816818237, "sampling/importance_sampling_ratio/mean": 1.0225872993469238, "sampling/importance_sampling_ratio/min": 0.8341471552848816, "sampling/sampling_logp_difference/max": 0.1979503631591797, "sampling/sampling_logp_difference/mean": 0.01478488091379404, "step": 761, "step_time": 14.935421825008234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1616.0, "completions/max_terminated_length": 1616.0, "completions/mean_length": 119.0, "completions/mean_terminated_length": 119.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11611130088567734, "epoch": 0.01524, "frac_reward_zero_std": 0.0, "grad_norm": 0.18827050924301147, "kl": 2.6091966368258, "learning_rate": 2.9677945306721114e-06, "loss": -0.0194, "num_tokens": 35000990.0, "reward": -0.18000000715255737, "reward_std": 0.9706909656524658, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.3540778160095215, "sampling/importance_sampling_ratio/mean": 0.9509019255638123, "sampling/importance_sampling_ratio/min": 0.34406548738479614, "sampling/sampling_logp_difference/max": 1.0666356086730957, "sampling/sampling_logp_difference/mean": 0.053375244140625, "step": 762, "step_time": 15.5985711149915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 108.625, "completions/mean_terminated_length": 108.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12326249829493463, "epoch": 0.01526, "frac_reward_zero_std": 0.0, "grad_norm": 0.18387861549854279, "kl": 0.5549711920320988, "learning_rate": 2.9612288974387883e-06, "loss": 0.0016, "num_tokens": 35047872.0, "reward": 0.07187499105930328, "reward_std": 1.0529967546463013, "rewards/rollout_reward_func/mean": 0.07187499105930328, "rewards/rollout_reward_func/std": 1.0229729413986206, "sampling/importance_sampling_ratio/max": 1.1029146909713745, "sampling/importance_sampling_ratio/mean": 1.0178875923156738, "sampling/importance_sampling_ratio/min": 0.9579899907112122, "sampling/sampling_logp_difference/max": 0.0847192108631134, "sampling/sampling_logp_difference/mean": 0.00839690025895834, "step": 763, "step_time": 11.628895221008861 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 35.4375, "completions/mean_terminated_length": 35.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16876644734293222, "epoch": 0.01528, "frac_reward_zero_std": 0.0, "grad_norm": 0.5159838199615479, "kl": 1.9214452262967825, "learning_rate": 2.954706645319525e-06, "loss": -0.0138, "num_tokens": 35093010.0, "reward": -0.053125008940696716, "reward_std": 1.0711638927459717, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.1669191122055054, "sampling/importance_sampling_ratio/mean": 1.0042057037353516, "sampling/importance_sampling_ratio/min": 0.6016210317611694, "sampling/sampling_logp_difference/max": 0.5080405473709106, "sampling/sampling_logp_difference/mean": 0.024640340358018875, "step": 764, "step_time": 11.50340950601094 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 71.375, "completions/mean_terminated_length": 71.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07562849717214704, "epoch": 0.0153, "frac_reward_zero_std": 0.0, "grad_norm": 0.08290620893239975, "kl": 2.4090853594243526, "learning_rate": 2.9482278603472393e-06, "loss": -0.0102, "num_tokens": 35138864.0, "reward": 0.07312498986721039, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.383028507232666, "sampling/importance_sampling_ratio/mean": 0.9906103610992432, "sampling/importance_sampling_ratio/min": 0.4814698100090027, "sampling/sampling_logp_difference/max": 0.7308588027954102, "sampling/sampling_logp_difference/mean": 0.026196278631687164, "step": 765, "step_time": 10.842469183000503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 140.65625, "completions/mean_terminated_length": 140.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10460673191118985, "epoch": 0.01532, "frac_reward_zero_std": 0.0, "grad_norm": 0.4340816140174866, "kl": 0.6819293778389692, "learning_rate": 2.9417926279814825e-06, "loss": -0.0285, "num_tokens": 35185366.0, "reward": 0.1993749886751175, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.7562860250473022, "sampling/importance_sampling_ratio/mean": 1.026503324508667, "sampling/importance_sampling_ratio/min": 0.7536391615867615, "sampling/sampling_logp_difference/max": 0.550504207611084, "sampling/sampling_logp_difference/mean": 0.021364042535424232, "step": 766, "step_time": 15.536842868008534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 58.1875, "completions/mean_terminated_length": 58.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11641956283710897, "epoch": 0.01534, "frac_reward_zero_std": 0.0, "grad_norm": 0.14144514501094818, "kl": 1.754040876403451, "learning_rate": 2.935401033107319e-06, "loss": -0.0088, "num_tokens": 35231508.0, "reward": -0.36875003576278687, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.36875003576278687, "rewards/rollout_reward_func/std": 0.9512767791748047, "sampling/importance_sampling_ratio/max": 1.0454422235488892, "sampling/importance_sampling_ratio/mean": 0.9578238725662231, "sampling/importance_sampling_ratio/min": 0.38753119111061096, "sampling/sampling_logp_difference/max": 0.9485843181610107, "sampling/sampling_logp_difference/mean": 0.03937344253063202, "step": 767, "step_time": 12.176324107000255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 393.0, "completions/max_terminated_length": 393.0, "completions/mean_length": 22.21875, "completions/mean_terminated_length": 22.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10277113760821521, "epoch": 0.01536, "frac_reward_zero_std": 0.0, "grad_norm": 0.17075292766094208, "kl": 0.5992859546095133, "learning_rate": 2.929053160034207e-06, "loss": -0.0077, "num_tokens": 35274603.0, "reward": 0.07249999046325684, "reward_std": 1.0343363285064697, "rewards/rollout_reward_func/mean": 0.07249999046325684, "rewards/rollout_reward_func/std": 1.023564338684082, "sampling/importance_sampling_ratio/max": 1.0304012298583984, "sampling/importance_sampling_ratio/mean": 0.994872510433197, "sampling/importance_sampling_ratio/min": 0.5510798096656799, "sampling/sampling_logp_difference/max": 0.5956900119781494, "sampling/sampling_logp_difference/mean": 0.015148920938372612, "step": 768, "step_time": 10.50000675200863 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 793.0, "completions/max_terminated_length": 793.0, "completions/mean_length": 117.46875, "completions/mean_terminated_length": 117.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13729224249254912, "epoch": 0.01538, "frac_reward_zero_std": 0.0, "grad_norm": 0.2723259925842285, "kl": 1.433020919561386, "learning_rate": 2.9227490924948843e-06, "loss": -0.0106, "num_tokens": 35322170.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.600136160850525, "sampling/importance_sampling_ratio/mean": 1.00901460647583, "sampling/importance_sampling_ratio/min": 0.4763661026954651, "sampling/sampling_logp_difference/max": 0.7413091659545898, "sampling/sampling_logp_difference/mean": 0.04210586100816727, "step": 769, "step_time": 12.606345369989867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 59.875, "completions/mean_terminated_length": 59.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.132318873424083, "epoch": 0.0154, "frac_reward_zero_std": 0.0, "grad_norm": 0.1947513371706009, "kl": 2.1334357671439648, "learning_rate": 2.9164889136442616e-06, "loss": -0.0103, "num_tokens": 35367466.0, "reward": -0.11625000834465027, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.2924442291259766, "sampling/importance_sampling_ratio/mean": 1.0116653442382812, "sampling/importance_sampling_ratio/min": 0.613879382610321, "sampling/sampling_logp_difference/max": 0.4880027770996094, "sampling/sampling_logp_difference/mean": 0.019396144896745682, "step": 770, "step_time": 13.362411633999727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 42.59375, "completions/mean_terminated_length": 42.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10461507784202695, "epoch": 0.01542, "frac_reward_zero_std": 0.0, "grad_norm": 0.33793535828590393, "kl": 1.291044618934393, "learning_rate": 2.9102727060583324e-06, "loss": -0.0063, "num_tokens": 35411060.0, "reward": 0.009999990463256836, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.1368184089660645, "sampling/importance_sampling_ratio/mean": 0.9669727683067322, "sampling/importance_sampling_ratio/min": 0.5391504168510437, "sampling/sampling_logp_difference/max": 0.6177549362182617, "sampling/sampling_logp_difference/mean": 0.032959144562482834, "step": 771, "step_time": 11.27827206200891 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 100.59375, "completions/mean_terminated_length": 100.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1273446895647794, "epoch": 0.01544, "frac_reward_zero_std": 0.0, "grad_norm": 0.3002697825431824, "kl": 2.21933582611382, "learning_rate": 2.9041005517330734e-06, "loss": -0.0246, "num_tokens": 35459435.0, "reward": -0.24250000715255737, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.0571231842041016, "sampling/importance_sampling_ratio/mean": 0.9897115230560303, "sampling/importance_sampling_ratio/min": 0.6128349304199219, "sampling/sampling_logp_difference/max": 0.5000348091125488, "sampling/sampling_logp_difference/mean": 0.018456432968378067, "step": 772, "step_time": 14.102499766013352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 97.09375, "completions/mean_terminated_length": 97.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09744138573296368, "epoch": 0.01546, "frac_reward_zero_std": 0.0, "grad_norm": 0.3696766495704651, "kl": 1.4414178179576993, "learning_rate": 2.897972532083375e-06, "loss": -0.0051, "num_tokens": 35506392.0, "reward": -0.18000000715255737, "reward_std": 0.9345266222953796, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.3650716543197632, "sampling/importance_sampling_ratio/mean": 1.0006675720214844, "sampling/importance_sampling_ratio/min": 0.6132739186286926, "sampling/sampling_logp_difference/max": 0.48925161361694336, "sampling/sampling_logp_difference/mean": 0.022276658564805984, "step": 773, "step_time": 11.629224387012073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 102.6875, "completions/mean_terminated_length": 102.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07424198754597455, "epoch": 0.01548, "frac_reward_zero_std": 0.0, "grad_norm": 0.10990285128355026, "kl": 1.2093553906306624, "learning_rate": 2.8918887279419583e-06, "loss": -0.0313, "num_tokens": 35552108.0, "reward": 0.1993749886751175, "reward_std": 1.0349996089935303, "rewards/rollout_reward_func/mean": 0.1993749886751175, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.1972789764404297, "sampling/importance_sampling_ratio/mean": 0.9934014678001404, "sampling/importance_sampling_ratio/min": 0.4816470444202423, "sampling/sampling_logp_difference/max": 0.7304286956787109, "sampling/sampling_logp_difference/mean": 0.01955738663673401, "step": 774, "step_time": 13.08140776601067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 131.375, "completions/mean_terminated_length": 131.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08912300085648894, "epoch": 0.0155, "frac_reward_zero_std": 0.0, "grad_norm": 0.23397399485111237, "kl": 1.6969011519104242, "learning_rate": 2.885849219558313e-06, "loss": -0.0359, "num_tokens": 35600180.0, "reward": 0.009999990463256836, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.3816123008728027, "sampling/importance_sampling_ratio/mean": 1.0159348249435425, "sampling/importance_sampling_ratio/min": 0.69526606798172, "sampling/sampling_logp_difference/max": 0.36282777786254883, "sampling/sampling_logp_difference/mean": 0.016304682940244675, "step": 775, "step_time": 16.364954401018622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 60.84375, "completions/mean_terminated_length": 60.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12080743117257953, "epoch": 0.01552, "frac_reward_zero_std": 0.0, "grad_norm": 0.2912755012512207, "kl": 2.126659097149968, "learning_rate": 2.8798540865976366e-06, "loss": -0.0022, "num_tokens": 35645363.0, "reward": -0.11625000834465027, "reward_std": 0.9626314640045166, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.3965688943862915, "sampling/importance_sampling_ratio/mean": 0.9973700046539307, "sampling/importance_sampling_ratio/min": 0.5419723391532898, "sampling/sampling_logp_difference/max": 0.6125154495239258, "sampling/sampling_logp_difference/mean": 0.0384025052189827, "step": 776, "step_time": 11.033748277979612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 84.4375, "completions/mean_terminated_length": 84.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11947985109873116, "epoch": 0.01554, "frac_reward_zero_std": 0.0, "grad_norm": 0.21191911399364471, "kl": 0.6339747533202171, "learning_rate": 2.8739034081397865e-06, "loss": -0.0101, "num_tokens": 35691468.0, "reward": 0.009999990463256836, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.2458441257476807, "sampling/importance_sampling_ratio/mean": 1.026252269744873, "sampling/importance_sampling_ratio/min": 0.8456206917762756, "sampling/sampling_logp_difference/max": 0.21087855100631714, "sampling/sampling_logp_difference/mean": 0.014342039823532104, "step": 777, "step_time": 11.517459709997638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 420.0, "completions/max_terminated_length": 420.0, "completions/mean_length": 54.9375, "completions/mean_terminated_length": 54.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07103590515907854, "epoch": 0.01556, "frac_reward_zero_std": 0.0, "grad_norm": 0.024198366329073906, "kl": 0.6436098031699657, "learning_rate": 2.8679972626782328e-06, "loss": -0.0188, "num_tokens": 35736760.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.0321706533432007, "sampling/importance_sampling_ratio/mean": 1.0125874280929565, "sampling/importance_sampling_ratio/min": 1.004007339477539, "sampling/sampling_logp_difference/max": 0.03172823786735535, "sampling/sampling_logp_difference/mean": 0.005430983379483223, "step": 778, "step_time": 11.717685731004167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 91.75, "completions/mean_terminated_length": 91.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15358796995133162, "epoch": 0.01558, "frac_reward_zero_std": 0.0, "grad_norm": 0.3648412227630615, "kl": 2.2043138034641743, "learning_rate": 2.8621357281190286e-06, "loss": -0.0208, "num_tokens": 35785217.0, "reward": -0.24250000715255737, "reward_std": 1.0074068307876587, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.254522681236267, "sampling/importance_sampling_ratio/mean": 1.0085737705230713, "sampling/importance_sampling_ratio/min": 0.6972783803939819, "sampling/sampling_logp_difference/max": 0.36421871185302734, "sampling/sampling_logp_difference/mean": 0.021997470408678055, "step": 779, "step_time": 11.617621632998635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 96.25, "completions/mean_terminated_length": 96.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09496095916256309, "epoch": 0.0156, "frac_reward_zero_std": 0.0, "grad_norm": 0.2174472063779831, "kl": 0.6693198941648006, "learning_rate": 2.8563188817797767e-06, "loss": -0.0205, "num_tokens": 35830799.0, "reward": 0.1356249898672104, "reward_std": 1.0259160995483398, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175585746765137, "sampling/importance_sampling_ratio/max": 1.580504298210144, "sampling/importance_sampling_ratio/mean": 1.0225458145141602, "sampling/importance_sampling_ratio/min": 0.6897792220115662, "sampling/sampling_logp_difference/max": 0.4415254592895508, "sampling/sampling_logp_difference/mean": 0.01627233810722828, "step": 780, "step_time": 13.282548313014559 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 25.4375, "completions/mean_terminated_length": 25.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14554975321516395, "epoch": 0.01562, "frac_reward_zero_std": 0.0, "grad_norm": 0.11423420161008835, "kl": 0.699999799951911, "learning_rate": 2.850546800388611e-06, "loss": -0.0154, "num_tokens": 35875628.0, "reward": -0.18000000715255737, "reward_std": 0.9981721639633179, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.2861231565475464, "sampling/importance_sampling_ratio/mean": 1.0015897750854492, "sampling/importance_sampling_ratio/min": 0.3946807086467743, "sampling/sampling_logp_difference/max": 0.9295974969863892, "sampling/sampling_logp_difference/mean": 0.02782175876200199, "step": 781, "step_time": 11.543952158019238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 77.4375, "completions/mean_terminated_length": 77.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10349769808817655, "epoch": 0.01564, "frac_reward_zero_std": 0.25, "grad_norm": 0.4551980197429657, "kl": 0.6803925596177578, "learning_rate": 2.844819560083188e-06, "loss": -0.0001, "num_tokens": 35920189.0, "reward": 0.2618749737739563, "reward_std": 0.6731642484664917, "rewards/rollout_reward_func/mean": 0.2618749737739563, "rewards/rollout_reward_func/std": 0.9930903911590576, "sampling/importance_sampling_ratio/max": 2.3349742889404297, "sampling/importance_sampling_ratio/mean": 1.0618510246276855, "sampling/importance_sampling_ratio/min": 1.0025547742843628, "sampling/sampling_logp_difference/max": 0.7470067739486694, "sampling/sampling_logp_difference/mean": 0.020651722326874733, "step": 782, "step_time": 11.139240273012547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 60.28125, "completions/mean_terminated_length": 60.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09871190786361694, "epoch": 0.01566, "frac_reward_zero_std": 0.0, "grad_norm": 0.1151239350438118, "kl": 0.6273074764758348, "learning_rate": 2.8391372364096794e-06, "loss": -0.0192, "num_tokens": 35965977.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.0248967409133911, "sampling/importance_sampling_ratio/mean": 1.0115065574645996, "sampling/importance_sampling_ratio/min": 0.9675889015197754, "sampling/sampling_logp_difference/max": 0.032855093479156494, "sampling/sampling_logp_difference/mean": 0.005835338495671749, "step": 783, "step_time": 13.450077923997014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 838.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 96.9375, "completions/mean_terminated_length": 96.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09704417560715228, "epoch": 0.01568, "frac_reward_zero_std": 0.0, "grad_norm": 0.37178558111190796, "kl": 0.6273203045129776, "learning_rate": 2.8334999043217752e-06, "loss": -0.0089, "num_tokens": 36011716.0, "reward": 0.07312498986721039, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.0826585292816162, "sampling/importance_sampling_ratio/mean": 1.0091100931167603, "sampling/importance_sampling_ratio/min": 0.7264226078987122, "sampling/sampling_logp_difference/max": 0.32999753952026367, "sampling/sampling_logp_difference/mean": 0.011231537908315659, "step": 784, "step_time": 13.82070243000635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 58.3125, "completions/mean_terminated_length": 58.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18321376340463758, "epoch": 0.0157, "frac_reward_zero_std": 0.25, "grad_norm": 0.38085949420928955, "kl": 1.6172932926565409, "learning_rate": 2.827907638179697e-06, "loss": -0.0199, "num_tokens": 36058393.0, "reward": -0.4318750202655792, "reward_std": 0.7650657892227173, "rewards/rollout_reward_func/mean": -0.4318750202655792, "rewards/rollout_reward_func/std": 0.9227429032325745, "sampling/importance_sampling_ratio/max": 1.2917935848236084, "sampling/importance_sampling_ratio/mean": 0.9947080016136169, "sampling/importance_sampling_ratio/min": 0.4207264184951782, "sampling/sampling_logp_difference/max": 0.8657979965209961, "sampling/sampling_logp_difference/mean": 0.03780566155910492, "step": 785, "step_time": 13.353539744006412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 136.0, "completions/mean_terminated_length": 136.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1106732680927962, "epoch": 0.01572, "frac_reward_zero_std": 0.0, "grad_norm": 0.14951668679714203, "kl": 0.7711734920740128, "learning_rate": 2.8223605117492136e-06, "loss": -0.0141, "num_tokens": 36107653.0, "reward": -0.24250000715255737, "reward_std": 0.9436101913452148, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.9935761094093323, "sampling/importance_sampling_ratio/max": 1.1144739389419556, "sampling/importance_sampling_ratio/mean": 0.9942917823791504, "sampling/importance_sampling_ratio/min": 0.4859227240085602, "sampling/sampling_logp_difference/max": 0.7254171371459961, "sampling/sampling_logp_difference/mean": 0.022316081449389458, "step": 786, "step_time": 14.2601167970206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 128.96875, "completions/mean_terminated_length": 128.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1335961268050596, "epoch": 0.01574, "frac_reward_zero_std": 0.0, "grad_norm": 0.2696618437767029, "kl": 0.710194319486618, "learning_rate": 2.8168585982006753e-06, "loss": -0.025, "num_tokens": 36155553.0, "reward": 0.1356249898672104, "reward_std": 0.9897517561912537, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.1905653476715088, "sampling/importance_sampling_ratio/mean": 0.9922775030136108, "sampling/importance_sampling_ratio/min": 0.6404060125350952, "sampling/sampling_logp_difference/max": 0.49030399322509766, "sampling/sampling_logp_difference/mean": 0.026266401633620262, "step": 787, "step_time": 14.04776944698824 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 145.96875, "completions/mean_terminated_length": 145.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1021765333134681, "epoch": 0.01576, "frac_reward_zero_std": 0.0, "grad_norm": 0.18504181504249573, "kl": 0.8177146222442389, "learning_rate": 2.8114019701080404e-06, "loss": -0.0267, "num_tokens": 36204446.0, "reward": 0.26249998807907104, "reward_std": 1.0264281034469604, "rewards/rollout_reward_func/mean": 0.26249998807907104, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.1408582925796509, "sampling/importance_sampling_ratio/mean": 1.0049476623535156, "sampling/importance_sampling_ratio/min": 0.74017733335495, "sampling/sampling_logp_difference/max": 0.30100536346435547, "sampling/sampling_logp_difference/mean": 0.013990772888064384, "step": 788, "step_time": 13.22786483897653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 72.09375, "completions/mean_terminated_length": 72.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0897712605074048, "epoch": 0.01578, "frac_reward_zero_std": 0.0, "grad_norm": 0.14515942335128784, "kl": 0.9824716709554195, "learning_rate": 2.805990699447921e-06, "loss": -0.0048, "num_tokens": 36248120.0, "reward": 0.2618749737739563, "reward_std": 0.9072020053863525, "rewards/rollout_reward_func/mean": 0.2618749737739563, "rewards/rollout_reward_func/std": 0.9930903911590576, "sampling/importance_sampling_ratio/max": 1.4317117929458618, "sampling/importance_sampling_ratio/mean": 1.0240287780761719, "sampling/importance_sampling_ratio/min": 0.6246073842048645, "sampling/sampling_logp_difference/max": 0.47062110900878906, "sampling/sampling_logp_difference/mean": 0.020437166094779968, "step": 789, "step_time": 11.887627730000531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 59.71875, "completions/mean_terminated_length": 59.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14702836284413934, "epoch": 0.0158, "frac_reward_zero_std": 0.0, "grad_norm": 0.3341042697429657, "kl": 0.6968177631497383, "learning_rate": 2.800624857598635e-06, "loss": -0.0172, "num_tokens": 36294953.0, "reward": -0.3056250214576721, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.3056250214576721, "rewards/rollout_reward_func/std": 0.9747685790061951, "sampling/importance_sampling_ratio/max": 1.2472410202026367, "sampling/importance_sampling_ratio/mean": 1.025960087776184, "sampling/importance_sampling_ratio/min": 0.9724230766296387, "sampling/sampling_logp_difference/max": 0.2210744023323059, "sampling/sampling_logp_difference/mean": 0.011445961892604828, "step": 790, "step_time": 11.647838069991849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 86.6875, "completions/mean_terminated_length": 86.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.101658059284091, "epoch": 0.01582, "frac_reward_zero_std": 0.0, "grad_norm": 0.17924395203590393, "kl": 3.8017486855387688, "learning_rate": 2.7953045153392656e-06, "loss": -0.0096, "num_tokens": 36341013.0, "reward": 0.07312498986721039, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.1558524370193481, "sampling/importance_sampling_ratio/mean": 0.9850864410400391, "sampling/importance_sampling_ratio/min": 0.537398636341095, "sampling/sampling_logp_difference/max": 0.62103271484375, "sampling/sampling_logp_difference/mean": 0.024842195212841034, "step": 791, "step_time": 11.653814779994718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 113.25, "completions/mean_terminated_length": 113.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09918142179958522, "epoch": 0.01584, "frac_reward_zero_std": 0.0, "grad_norm": 0.41425397992134094, "kl": 0.6335231475532055, "learning_rate": 2.790029742848723e-06, "loss": -0.0265, "num_tokens": 36386501.0, "reward": 0.26124995946884155, "reward_std": 0.906048595905304, "rewards/rollout_reward_func/mean": 0.26124995946884155, "rewards/rollout_reward_func/std": 0.99260413646698, "sampling/importance_sampling_ratio/max": 1.1666951179504395, "sampling/importance_sampling_ratio/mean": 1.0093812942504883, "sampling/importance_sampling_ratio/min": 0.7683551907539368, "sampling/sampling_logp_difference/max": 0.1549210548400879, "sampling/sampling_logp_difference/mean": 0.010059423744678497, "step": 792, "step_time": 14.99438700798055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 63.09375, "completions/mean_terminated_length": 63.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13829464511945844, "epoch": 0.01586, "frac_reward_zero_std": 0.0, "grad_norm": 0.10800173878669739, "kl": 0.7014234606176615, "learning_rate": 2.7848006097048208e-06, "loss": -0.0177, "num_tokens": 36432698.0, "reward": -0.11687500774860382, "reward_std": 1.0445914268493652, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.3666774034500122, "sampling/importance_sampling_ratio/mean": 1.0170131921768188, "sampling/importance_sampling_ratio/min": 0.7509689331054688, "sampling/sampling_logp_difference/max": 0.28639912605285645, "sampling/sampling_logp_difference/mean": 0.017597060650587082, "step": 793, "step_time": 10.916686633005156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 84.875, "completions/mean_terminated_length": 84.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09466917405370623, "epoch": 0.01588, "frac_reward_zero_std": 0.0, "grad_norm": 0.0807529017329216, "kl": 0.5809136480093002, "learning_rate": 2.779617184883361e-06, "loss": -0.0122, "num_tokens": 36478475.0, "reward": 0.009999990463256836, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.1240569353103638, "sampling/importance_sampling_ratio/mean": 1.0171831846237183, "sampling/importance_sampling_ratio/min": 1.0045490264892578, "sampling/sampling_logp_difference/max": 0.11685366928577423, "sampling/sampling_logp_difference/mean": 0.007262152153998613, "step": 794, "step_time": 14.112189301995386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 42.71875, "completions/mean_terminated_length": 42.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1162602559197694, "epoch": 0.0159, "frac_reward_zero_std": 0.0, "grad_norm": 0.10663188248872757, "kl": 0.6203050762414932, "learning_rate": 2.7744795367572214e-06, "loss": -0.0185, "num_tokens": 36524254.0, "reward": -0.18000000715255737, "reward_std": 0.9981721639633179, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.0293511152267456, "sampling/importance_sampling_ratio/mean": 1.0121601819992065, "sampling/importance_sampling_ratio/min": 0.922386109828949, "sampling/sampling_logp_difference/max": 0.08084118366241455, "sampling/sampling_logp_difference/mean": 0.007552566006779671, "step": 795, "step_time": 12.270141220011283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 95.96875, "completions/mean_terminated_length": 95.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10093558416701853, "epoch": 0.01592, "frac_reward_zero_std": 0.0, "grad_norm": 0.2379421591758728, "kl": 2.1040908489376307, "learning_rate": 2.7693877330954525e-06, "loss": -0.0146, "num_tokens": 36569797.0, "reward": 0.07312498986721039, "reward_std": 0.9988353252410889, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.028689980506897, "sampling/importance_sampling_ratio/mean": 0.9685333967208862, "sampling/importance_sampling_ratio/min": 0.5474057197570801, "sampling/sampling_logp_difference/max": 0.6022500991821289, "sampling/sampling_logp_difference/mean": 0.027197865769267082, "step": 796, "step_time": 11.629956910997862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 89.21875, "completions/mean_terminated_length": 89.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14851077110506594, "epoch": 0.01594, "frac_reward_zero_std": 0.0, "grad_norm": 0.4905417263507843, "kl": 0.9594859071075916, "learning_rate": 2.7643418410623903e-06, "loss": -0.0191, "num_tokens": 36614603.0, "reward": 0.32499998807907104, "reward_std": 0.9984559416770935, "rewards/rollout_reward_func/mean": 0.32499998807907104, "rewards/rollout_reward_func/std": 0.974315345287323, "sampling/importance_sampling_ratio/max": 1.1219465732574463, "sampling/importance_sampling_ratio/mean": 0.9722847938537598, "sampling/importance_sampling_ratio/min": 0.4212368428707123, "sampling/sampling_logp_difference/max": 0.8797502517700195, "sampling/sampling_logp_difference/mean": 0.02935381419956684, "step": 797, "step_time": 10.931151550008508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 429.0, "completions/max_terminated_length": 429.0, "completions/mean_length": 76.28125, "completions/mean_terminated_length": 76.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12816204223781824, "epoch": 0.01596, "frac_reward_zero_std": 0.0, "grad_norm": 0.09792369604110718, "kl": 0.5730165112763643, "learning_rate": 2.7593419272167567e-06, "loss": -0.0227, "num_tokens": 36661289.0, "reward": -0.053125008940696716, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.412455439567566, "sampling/importance_sampling_ratio/mean": 1.034928321838379, "sampling/importance_sampling_ratio/min": 1.0027951002120972, "sampling/sampling_logp_difference/max": 0.3462435007095337, "sampling/sampling_logp_difference/mean": 0.013277017511427402, "step": 798, "step_time": 12.23473041201214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 73.78125, "completions/mean_terminated_length": 73.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1344852694310248, "epoch": 0.01598, "frac_reward_zero_std": 0.0, "grad_norm": 0.10918737947940826, "kl": 0.7882096264511347, "learning_rate": 2.7543880575107986e-06, "loss": -0.0034, "num_tokens": 36708089.0, "reward": -0.11625000834465027, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1788620948791504, "sampling/importance_sampling_ratio/mean": 1.0139579772949219, "sampling/importance_sampling_ratio/min": 0.6172884106636047, "sampling/sampling_logp_difference/max": 0.48975658416748047, "sampling/sampling_logp_difference/mean": 0.018693167716264725, "step": 799, "step_time": 11.521556702988164 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 176.03125, "completions/mean_terminated_length": 176.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11942547373473644, "epoch": 0.016, "frac_reward_zero_std": 0.0, "grad_norm": 0.29455649852752686, "kl": 1.1619138270616531, "learning_rate": 2.7494802972894046e-06, "loss": -0.0593, "num_tokens": 36758359.0, "reward": 0.009999990463256836, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.612890362739563, "sampling/importance_sampling_ratio/mean": 1.0162410736083984, "sampling/importance_sampling_ratio/min": 0.44594675302505493, "sampling/sampling_logp_difference/max": 0.949559211730957, "sampling/sampling_logp_difference/mean": 0.03225531429052353, "step": 800, "step_time": 14.376639894006075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 72.21875, "completions/mean_terminated_length": 72.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13356717908754945, "epoch": 0.01602, "frac_reward_zero_std": 0.0, "grad_norm": 0.24876528978347778, "kl": 1.872361022979021, "learning_rate": 2.7446187112892518e-06, "loss": -0.0284, "num_tokens": 36804189.0, "reward": -0.11625000834465027, "reward_std": 0.9712425470352173, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.2251863479614258, "sampling/importance_sampling_ratio/mean": 0.9667044878005981, "sampling/importance_sampling_ratio/min": 0.5158522725105286, "sampling/sampling_logp_difference/max": 0.6618704795837402, "sampling/sampling_logp_difference/mean": 0.04014015197753906, "step": 801, "step_time": 15.040991872010636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 123.46875, "completions/mean_terminated_length": 123.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11214319569990039, "epoch": 0.01604, "frac_reward_zero_std": 0.0, "grad_norm": 0.1332731992006302, "kl": 0.6298358514904976, "learning_rate": 2.7398033636379433e-06, "loss": -0.0253, "num_tokens": 36851597.0, "reward": 0.32499998807907104, "reward_std": 0.9347947835922241, "rewards/rollout_reward_func/mean": 0.32499998807907104, "rewards/rollout_reward_func/std": 0.974315345287323, "sampling/importance_sampling_ratio/max": 1.2180887460708618, "sampling/importance_sampling_ratio/mean": 1.0248255729675293, "sampling/importance_sampling_ratio/min": 0.8967770338058472, "sampling/sampling_logp_difference/max": 0.1834130883216858, "sampling/sampling_logp_difference/mean": 0.011106859892606735, "step": 802, "step_time": 13.482970958000806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 17.875, "completions/mean_terminated_length": 17.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12169990537222475, "epoch": 0.01606, "frac_reward_zero_std": 0.0, "grad_norm": 0.11127708852291107, "kl": 0.7449737172573805, "learning_rate": 2.7350343178531718e-06, "loss": 0.0001, "num_tokens": 36894945.0, "reward": -0.17937500774860382, "reward_std": 1.0178565979003906, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.0901178121566772, "sampling/importance_sampling_ratio/mean": 0.9855958223342896, "sampling/importance_sampling_ratio/min": 0.5729053616523743, "sampling/sampling_logp_difference/max": 0.5569660663604736, "sampling/sampling_logp_difference/mean": 0.027402322739362717, "step": 803, "step_time": 11.41277614300634 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 39.34375, "completions/mean_terminated_length": 39.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11035134270787239, "epoch": 0.01608, "frac_reward_zero_std": 0.0, "grad_norm": 0.0725754052400589, "kl": 0.6369364410638809, "learning_rate": 2.7303116368418743e-06, "loss": -0.012, "num_tokens": 36938229.0, "reward": 0.13624998927116394, "reward_std": 1.0625923871994019, "rewards/rollout_reward_func/mean": 0.13624998927116394, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.5565118789672852, "sampling/importance_sampling_ratio/mean": 1.025519847869873, "sampling/importance_sampling_ratio/min": 0.6908935308456421, "sampling/sampling_logp_difference/max": 0.4426766037940979, "sampling/sampling_logp_difference/mean": 0.019163290038704872, "step": 804, "step_time": 11.870437638026488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 784.0, "completions/max_terminated_length": 784.0, "completions/mean_length": 106.09375, "completions/mean_terminated_length": 106.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15367585350759327, "epoch": 0.0161, "frac_reward_zero_std": 0.0, "grad_norm": 0.14672833681106567, "kl": 0.6764717139303684, "learning_rate": 2.7256353828994054e-06, "loss": -0.0132, "num_tokens": 36985877.0, "reward": -0.11750000715255737, "reward_std": 1.02540123462677, "rewards/rollout_reward_func/mean": -0.11750000715255737, "rewards/rollout_reward_func/std": 1.0166834592819214, "sampling/importance_sampling_ratio/max": 1.1778392791748047, "sampling/importance_sampling_ratio/mean": 1.0186593532562256, "sampling/importance_sampling_ratio/min": 0.6142846345901489, "sampling/sampling_logp_difference/max": 0.5033363103866577, "sampling/sampling_logp_difference/mean": 0.02048644796013832, "step": 805, "step_time": 12.09190652698453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 76.59375, "completions/mean_terminated_length": 76.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1110681121936068, "epoch": 0.01612, "frac_reward_zero_std": 0.0, "grad_norm": 0.17731121182441711, "kl": 1.6089581102132797, "learning_rate": 2.7210056177087174e-06, "loss": 0.0039, "num_tokens": 37030614.0, "reward": -0.053125008940696716, "reward_std": 1.054020881652832, "rewards/rollout_reward_func/mean": -0.053125008940696716, "rewards/rollout_reward_func/std": 1.024154782295227, "sampling/importance_sampling_ratio/max": 1.099156141281128, "sampling/importance_sampling_ratio/mean": 0.9545528292655945, "sampling/importance_sampling_ratio/min": 0.45669785141944885, "sampling/sampling_logp_difference/max": 0.7836792469024658, "sampling/sampling_logp_difference/mean": 0.043419335037469864, "step": 806, "step_time": 14.414304624981014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 57.34375, "completions/mean_terminated_length": 57.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15329760522581637, "epoch": 0.01614, "frac_reward_zero_std": 0.0, "grad_norm": 0.28355371952056885, "kl": 0.9473863933235407, "learning_rate": 2.7164224023395413e-06, "loss": -0.0123, "num_tokens": 37075671.0, "reward": -0.24312500655651093, "reward_std": 0.962119460105896, "rewards/rollout_reward_func/mean": -0.24312500655651093, "rewards/rollout_reward_func/std": 0.9927622675895691, "sampling/importance_sampling_ratio/max": 1.1216299533843994, "sampling/importance_sampling_ratio/mean": 0.9739265441894531, "sampling/importance_sampling_ratio/min": 0.4794107973575592, "sampling/sampling_logp_difference/max": 0.735112190246582, "sampling/sampling_logp_difference/mean": 0.03348157927393913, "step": 807, "step_time": 12.199086074011575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 114.5625, "completions/mean_terminated_length": 114.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15605521528050303, "epoch": 0.01616, "frac_reward_zero_std": 0.0, "grad_norm": 0.15669789910316467, "kl": 0.826217208057642, "learning_rate": 2.7118857972475863e-06, "loss": -0.0111, "num_tokens": 37123667.0, "reward": -0.17937500774860382, "reward_std": 0.9712029695510864, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.1176201105117798, "sampling/importance_sampling_ratio/mean": 1.0028010606765747, "sampling/importance_sampling_ratio/min": 0.6134732365608215, "sampling/sampling_logp_difference/max": 0.48825788497924805, "sampling/sampling_logp_difference/mean": 0.013482627458870411, "step": 808, "step_time": 14.02119663899066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 201.1875, "completions/mean_terminated_length": 201.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1763715329580009, "epoch": 0.01618, "frac_reward_zero_std": 0.0, "grad_norm": 0.29117506742477417, "kl": 0.5844429396092892, "learning_rate": 2.7073958622737417e-06, "loss": -0.0358, "num_tokens": 37175355.0, "reward": -0.24250000715255737, "reward_std": 0.9797744750976562, "rewards/rollout_reward_func/mean": -0.24250000715255737, "rewards/rollout_reward_func/std": 0.993576169013977, "sampling/importance_sampling_ratio/max": 1.1445735692977905, "sampling/importance_sampling_ratio/mean": 0.9777872562408447, "sampling/importance_sampling_ratio/min": 0.6265564560890198, "sampling/sampling_logp_difference/max": 0.5417723655700684, "sampling/sampling_logp_difference/mean": 0.03001909703016281, "step": 809, "step_time": 13.848363863995473 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 106.09375, "completions/mean_terminated_length": 106.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12881461181677878, "epoch": 0.0162, "frac_reward_zero_std": 0.0, "grad_norm": 0.26191246509552, "kl": 0.9466448575258255, "learning_rate": 2.702952656643284e-06, "loss": -0.0085, "num_tokens": 37222175.0, "reward": 0.009374991059303284, "reward_std": 1.0445914268493652, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.1374057531356812, "sampling/importance_sampling_ratio/mean": 0.9888870716094971, "sampling/importance_sampling_ratio/min": 0.658008337020874, "sampling/sampling_logp_difference/max": 0.41750168800354004, "sampling/sampling_logp_difference/mean": 0.02348661795258522, "step": 810, "step_time": 12.138843264998286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 62.6875, "completions/mean_terminated_length": 62.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1573568012099713, "epoch": 0.01622, "frac_reward_zero_std": 0.0, "grad_norm": 0.28457874059677124, "kl": 0.6237685475498438, "learning_rate": 2.6985562389650976e-06, "loss": -0.0015, "num_tokens": 37267685.0, "reward": -0.18187500536441803, "reward_std": 0.9316516518592834, "rewards/rollout_reward_func/mean": -0.18187500536441803, "rewards/rollout_reward_func/std": 1.0049086809158325, "sampling/importance_sampling_ratio/max": 1.3031622171401978, "sampling/importance_sampling_ratio/mean": 1.001020908355713, "sampling/importance_sampling_ratio/min": 0.3653353750705719, "sampling/sampling_logp_difference/max": 1.0069918632507324, "sampling/sampling_logp_difference/mean": 0.03835875540971756, "step": 811, "step_time": 11.630767441019998 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 56.34375, "completions/mean_terminated_length": 56.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14203455322422087, "epoch": 0.01624, "frac_reward_zero_std": 0.0, "grad_norm": 0.30022600293159485, "kl": 1.3028197949752212, "learning_rate": 2.6942066672309064e-06, "loss": -0.0139, "num_tokens": 37312706.0, "reward": -0.11687500774860382, "reward_std": 1.0255701541900635, "rewards/rollout_reward_func/mean": -0.11687500774860382, "rewards/rollout_reward_func/std": 1.017398476600647, "sampling/importance_sampling_ratio/max": 1.2013169527053833, "sampling/importance_sampling_ratio/mean": 0.9988287687301636, "sampling/importance_sampling_ratio/min": 0.6954689621925354, "sampling/sampling_logp_difference/max": 0.3645343780517578, "sampling/sampling_logp_difference/mean": 0.026845749467611313, "step": 812, "step_time": 14.123234045997378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 85.71875, "completions/mean_terminated_length": 85.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12467479985207319, "epoch": 0.01626, "frac_reward_zero_std": 0.0, "grad_norm": 0.15934234857559204, "kl": 1.7380877453833818, "learning_rate": 2.6899039988145003e-06, "loss": -0.012, "num_tokens": 37359346.0, "reward": -0.11625000834465027, "reward_std": 0.990263819694519, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.1338483095169067, "sampling/importance_sampling_ratio/mean": 0.9657922983169556, "sampling/importance_sampling_ratio/min": 0.4253333806991577, "sampling/sampling_logp_difference/max": 0.8549556732177734, "sampling/sampling_logp_difference/mean": 0.04184494912624359, "step": 813, "step_time": 11.794981945982727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 103.84375, "completions/mean_terminated_length": 103.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13588043861091137, "epoch": 0.01628, "frac_reward_zero_std": 0.0, "grad_norm": 0.32360294461250305, "kl": 1.7310688570141792, "learning_rate": 2.6856482904709844e-06, "loss": -0.0341, "num_tokens": 37408200.0, "reward": -0.36937499046325684, "reward_std": 0.8793015480041504, "rewards/rollout_reward_func/mean": -0.36937499046325684, "rewards/rollout_reward_func/std": 0.9503409266471863, "sampling/importance_sampling_ratio/max": 2.1621689796447754, "sampling/importance_sampling_ratio/mean": 1.0297026634216309, "sampling/importance_sampling_ratio/min": 0.6361594796180725, "sampling/sampling_logp_difference/max": 0.7562098503112793, "sampling/sampling_logp_difference/mean": 0.034078702330589294, "step": 814, "step_time": 13.897406680996937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 45.8125, "completions/mean_terminated_length": 45.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10510499426163733, "epoch": 0.0163, "frac_reward_zero_std": 0.0, "grad_norm": 0.11889799684286118, "kl": 0.9457233771681786, "learning_rate": 2.6814395983360287e-06, "loss": -0.0071, "num_tokens": 37452686.0, "reward": 0.009999990463256836, "reward_std": 1.0454493761062622, "rewards/rollout_reward_func/mean": 0.009999990463256836, "rewards/rollout_reward_func/std": 1.0261609554290771, "sampling/importance_sampling_ratio/max": 1.146039366722107, "sampling/importance_sampling_ratio/mean": 0.9654601216316223, "sampling/importance_sampling_ratio/min": 0.311491459608078, "sampling/sampling_logp_difference/max": 1.1661040782928467, "sampling/sampling_logp_difference/mean": 0.037884414196014404, "step": 815, "step_time": 11.755991460020596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 76.15625, "completions/mean_terminated_length": 76.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10130244202446193, "epoch": 0.01632, "frac_reward_zero_std": 0.0, "grad_norm": 0.2790387272834778, "kl": 0.8366397842764854, "learning_rate": 2.6772779779251283e-06, "loss": 0.0015, "num_tokens": 37498581.0, "reward": -0.05375000834465027, "reward_std": 1.0343363285064697, "rewards/rollout_reward_func/mean": -0.05375000834465027, "rewards/rollout_reward_func/std": 1.023484706878662, "sampling/importance_sampling_ratio/max": 1.060271978378296, "sampling/importance_sampling_ratio/mean": 1.0011645555496216, "sampling/importance_sampling_ratio/min": 0.6963055729866028, "sampling/sampling_logp_difference/max": 0.3622012138366699, "sampling/sampling_logp_difference/mean": 0.012606384232640266, "step": 816, "step_time": 11.00834137799393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 75.84375, "completions/mean_terminated_length": 75.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0807604540605098, "epoch": 0.01634, "frac_reward_zero_std": 0.0, "grad_norm": 0.5868681073188782, "kl": 0.6301742568612099, "learning_rate": 2.673163484132869e-06, "loss": -0.026, "num_tokens": 37543678.0, "reward": 0.32499998807907104, "reward_std": 0.9519377946853638, "rewards/rollout_reward_func/mean": 0.32499998807907104, "rewards/rollout_reward_func/std": 0.974315345287323, "sampling/importance_sampling_ratio/max": 1.0738693475723267, "sampling/importance_sampling_ratio/mean": 1.0116046667099, "sampling/importance_sampling_ratio/min": 0.899860680103302, "sampling/sampling_logp_difference/max": 0.10553216934204102, "sampling/sampling_logp_difference/mean": 0.007127071265131235, "step": 817, "step_time": 11.19207890398684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1288.0, "completions/max_terminated_length": 1288.0, "completions/mean_length": 97.8125, "completions/mean_terminated_length": 97.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08492266887333244, "epoch": 0.01636, "frac_reward_zero_std": 0.0, "grad_norm": 0.43590644001960754, "kl": 1.1344039365649223, "learning_rate": 2.6690961712322062e-06, "loss": 0.0063, "num_tokens": 37588811.0, "reward": 0.07312498986721039, "reward_std": 1.0711638927459717, "rewards/rollout_reward_func/mean": 0.07312498986721039, "rewards/rollout_reward_func/std": 1.0241549015045166, "sampling/importance_sampling_ratio/max": 1.0433629751205444, "sampling/importance_sampling_ratio/mean": 0.994860053062439, "sampling/importance_sampling_ratio/min": 0.6681485772132874, "sampling/sampling_logp_difference/max": 0.4751051664352417, "sampling/sampling_logp_difference/mean": 0.01541684940457344, "step": 818, "step_time": 14.630174394005735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 59.15625, "completions/mean_terminated_length": 59.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11713664210401475, "epoch": 0.01638, "frac_reward_zero_std": 0.0, "grad_norm": 0.10336649417877197, "kl": 0.5853661820292473, "learning_rate": 2.6650760928737448e-06, "loss": -0.0083, "num_tokens": 37634232.0, "reward": 0.009374991059303284, "reward_std": 1.0619292259216309, "rewards/rollout_reward_func/mean": 0.009374991059303284, "rewards/rollout_reward_func/std": 1.0255318880081177, "sampling/importance_sampling_ratio/max": 1.168674349784851, "sampling/importance_sampling_ratio/mean": 1.0182080268859863, "sampling/importance_sampling_ratio/min": 0.8921026587486267, "sampling/sampling_logp_difference/max": 0.15616285800933838, "sampling/sampling_logp_difference/mean": 0.010624325834214687, "step": 819, "step_time": 10.878604032019211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 137.125, "completions/mean_terminated_length": 137.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09967352962121367, "epoch": 0.0164, "frac_reward_zero_std": 0.0, "grad_norm": 0.5377712845802307, "kl": 2.8267729692161083, "learning_rate": 2.6611033020850367e-06, "loss": -0.0126, "num_tokens": 37682186.0, "reward": -0.17937500774860382, "reward_std": 0.9626710414886475, "rewards/rollout_reward_func/mean": -0.17937500774860382, "rewards/rollout_reward_func/std": 1.007961630821228, "sampling/importance_sampling_ratio/max": 1.269945740699768, "sampling/importance_sampling_ratio/mean": 0.9835832118988037, "sampling/importance_sampling_ratio/min": 0.3801873028278351, "sampling/sampling_logp_difference/max": 0.9714236259460449, "sampling/sampling_logp_difference/mean": 0.031501106917858124, "step": 820, "step_time": 14.532341097990866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 429.0, "completions/max_terminated_length": 429.0, "completions/mean_length": 33.875, "completions/mean_terminated_length": 33.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.120253890985623, "epoch": 0.01642, "frac_reward_zero_std": 0.0, "grad_norm": 0.22161640226840973, "kl": 0.7725724224001169, "learning_rate": 2.6571778512698786e-06, "loss": -0.0119, "num_tokens": 37725744.0, "reward": 0.1356249898672104, "reward_std": 1.0260486602783203, "rewards/rollout_reward_func/mean": 0.1356249898672104, "rewards/rollout_reward_func/std": 1.0175584554672241, "sampling/importance_sampling_ratio/max": 1.170296549797058, "sampling/importance_sampling_ratio/mean": 0.9836922883987427, "sampling/importance_sampling_ratio/min": 0.5230829119682312, "sampling/sampling_logp_difference/max": 0.6476411819458008, "sampling/sampling_logp_difference/mean": 0.027837829664349556, "step": 821, "step_time": 11.101446775013756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 438.0, "completions/max_terminated_length": 438.0, "completions/mean_length": 36.25, "completions/mean_terminated_length": 36.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09504513815045357, "epoch": 0.01644, "frac_reward_zero_std": 0.0, "grad_norm": 0.16908212006092072, "kl": 0.68206294067204, "learning_rate": 2.653299792207621e-06, "loss": -0.0118, "num_tokens": 37769970.0, "reward": -0.11625000834465027, "reward_std": 1.0435711145401, "rewards/rollout_reward_func/mean": -0.11625000834465027, "rewards/rollout_reward_func/std": 1.0181125402450562, "sampling/importance_sampling_ratio/max": 1.0368081331253052, "sampling/importance_sampling_ratio/mean": 1.01221764087677, "sampling/importance_sampling_ratio/min": 0.8632447719573975, "sampling/sampling_logp_difference/max": 0.14821481704711914, "sampling/sampling_logp_difference/mean": 0.009342101402580738, "step": 822, "step_time": 10.784580266030389 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 120.84375, "completions/mean_terminated_length": 120.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11336589383427054, "epoch": 0.01646, "frac_reward_zero_std": 0.0, "grad_norm": 0.19068965315818787, "kl": 1.1529344655573368, "learning_rate": 2.6494691760524815e-06, "loss": -0.017, "num_tokens": 37816606.0, "reward": -0.05437500774860382, "reward_std": 1.0336833000183105, "rewards/rollout_reward_func/mean": -0.05437500774860382, "rewards/rollout_reward_func/std": 1.0228264331817627, "sampling/importance_sampling_ratio/max": 1.1471904516220093, "sampling/importance_sampling_ratio/mean": 0.9807134866714478, "sampling/importance_sampling_ratio/min": 0.18342246115207672, "sampling/sampling_logp_difference/max": 1.7041211128234863, "sampling/sampling_logp_difference/mean": 0.037305328994989395, "step": 823, "step_time": 13.420687569006986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 132.78125, "completions/mean_terminated_length": 132.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12232654145918787, "epoch": 0.01648, "frac_reward_zero_std": 0.0, "grad_norm": 0.3754020631313324, "kl": 3.617659652605653, "learning_rate": 2.6456860533328806e-06, "loss": -0.0061, "num_tokens": 37864951.0, "reward": -0.18000000715255737, "reward_std": 0.9618130326271057, "rewards/rollout_reward_func/mean": -0.18000000715255737, "rewards/rollout_reward_func/std": 1.0071998834609985, "sampling/importance_sampling_ratio/max": 1.1299556493759155, "sampling/importance_sampling_ratio/mean": 0.9770246148109436, "sampling/importance_sampling_ratio/min": 0.539539098739624, "sampling/sampling_logp_difference/max": 0.6169490814208984, "sampling/sampling_logp_difference/mean": 0.028744492679834366, "step": 824, "step_time": 14.038349153990566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 85.15625, "completions/mean_terminated_length": 85.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14089776389300823, "epoch": 0.0165, "frac_reward_zero_std": 0.0, "grad_norm": 0.171576127409935, "kl": 0.9764721393585205, "learning_rate": 2.641950473950762e-06, "loss": -0.0071, "num_tokens": 37912484.0, "reward": -0.3693750202655792, "reward_std": 0.9062925577163696, "rewards/rollout_reward_func/mean": -0.3693750202655792, "rewards/rollout_reward_func/std": 0.950340986251831, "sampling/importance_sampling_ratio/max": 1.2113498449325562, "sampling/importance_sampling_ratio/mean": 0.9824928045272827, "sampling/importance_sampling_ratio/min": 0.4831489026546478, "sampling/sampling_logp_difference/max": 0.7305440902709961, "sampling/sampling_logp_difference/mean": 0.03012598119676113, "step": 825, "step_time": 11.561329699005 } ], "logging_steps": 1.0, "max_steps": 900, "num_input_tokens_seen": 37912484, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }