{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.00031, "eval_steps": 500, "global_step": 31, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1121.0, "completions/max_terminated_length": 1121.0, "completions/mean_length": 89.125, "completions/mean_terminated_length": 89.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8703252337872982, "epoch": 1e-05, "frac_reward_zero_std": 0.0, "grad_norm": 2.819500207901001, "kl": 0.0, "learning_rate": 0.0, "loss": -0.0572, "num_tokens": 20371.0, "reward": -0.7440354824066162, "reward_std": 0.4797802269458771, "rewards/rollout_reward_func/mean": -0.7440354824066162, "rewards/rollout_reward_func/std": 0.6812157034873962, "sampling/importance_sampling_ratio/max": 1.2225033044815063, "sampling/importance_sampling_ratio/mean": 1.0341836214065552, "sampling/importance_sampling_ratio/min": 0.9178955554962158, "sampling/sampling_logp_difference/max": 0.13056421279907227, "sampling/sampling_logp_difference/mean": 0.028775660321116447, "step": 1, "step_time": 10.162960374000022 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 650.0, "completions/max_terminated_length": 650.0, "completions/mean_length": 125.5625, "completions/mean_terminated_length": 125.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9611260890960693, "epoch": 2e-05, "frac_reward_zero_std": 0.0, "grad_norm": 3.7111358642578125, "kl": 0.0, "learning_rate": 2.8571428571428575e-07, "loss": -0.0413, "num_tokens": 42911.0, "reward": -0.5, "reward_std": 0.9258201122283936, "rewards/rollout_reward_func/mean": -0.5, "rewards/rollout_reward_func/std": 0.8944272398948669, "sampling/importance_sampling_ratio/max": 1.2408462762832642, "sampling/importance_sampling_ratio/mean": 1.014411449432373, "sampling/importance_sampling_ratio/min": 0.862123966217041, "sampling/sampling_logp_difference/max": 0.21459484100341797, "sampling/sampling_logp_difference/mean": 0.02787204459309578, "step": 2, "step_time": 8.416886302999842 }, { "clip_ratio/high_max": 0.03645833395421505, "clip_ratio/high_mean": 0.03645833395421505, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03645833395421505, "completions/clipped_ratio": 0.0, "completions/max_length": 1042.0, "completions/max_terminated_length": 1042.0, "completions/mean_length": 101.375, "completions/mean_terminated_length": 101.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9092908762395382, "epoch": 3e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.2625800371170044, "kl": 0.0020873087050858885, "learning_rate": 5.714285714285715e-07, "loss": -0.0354, "num_tokens": 63636.0, "reward": -0.5, "reward_std": 0.8711025714874268, "rewards/rollout_reward_func/mean": -0.5, "rewards/rollout_reward_func/std": 0.8944272398948669, "sampling/importance_sampling_ratio/max": 1.2457796335220337, "sampling/importance_sampling_ratio/mean": 0.9775898456573486, "sampling/importance_sampling_ratio/min": 0.4711868464946747, "sampling/sampling_logp_difference/max": 0.5345618724822998, "sampling/sampling_logp_difference/mean": 0.043653398752212524, "step": 3, "step_time": 8.985772771999791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 569.0, "completions/max_terminated_length": 569.0, "completions/mean_length": 112.0625, "completions/mean_terminated_length": 112.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8898606114089489, "epoch": 4e-05, "frac_reward_zero_std": 0.0, "grad_norm": 2.925949811935425, "kl": 0.0028716400265693665, "learning_rate": 8.571428571428572e-07, "loss": -0.0522, "num_tokens": 85414.0, "reward": -0.6772376298904419, "reward_std": 0.6541639566421509, "rewards/rollout_reward_func/mean": -0.6772376298904419, "rewards/rollout_reward_func/std": 0.7083688974380493, "sampling/importance_sampling_ratio/max": 1.141742467880249, "sampling/importance_sampling_ratio/mean": 1.0114331245422363, "sampling/importance_sampling_ratio/min": 0.7873453497886658, "sampling/sampling_logp_difference/max": 0.2348930835723877, "sampling/sampling_logp_difference/mean": 0.021859874948859215, "step": 4, "step_time": 7.5787519170000905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 659.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 82.4375, "completions/mean_terminated_length": 82.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0198273472487926, "epoch": 5e-05, "frac_reward_zero_std": 0.0, "grad_norm": 2.3419432640075684, "kl": 0.0010417364537715912, "learning_rate": 1.142857142857143e-06, "loss": -0.0133, "num_tokens": 106709.0, "reward": -0.7984567880630493, "reward_std": 0.3939744532108307, "rewards/rollout_reward_func/mean": -0.7984567880630493, "rewards/rollout_reward_func/std": 0.5597541332244873, "sampling/importance_sampling_ratio/max": 1.1149271726608276, "sampling/importance_sampling_ratio/mean": 0.9954546689987183, "sampling/importance_sampling_ratio/min": 0.8653857111930847, "sampling/sampling_logp_difference/max": 0.14299678802490234, "sampling/sampling_logp_difference/mean": 0.02414068393409252, "step": 5, "step_time": 8.253146455000092 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.03125, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9969437904655933, "epoch": 6e-05, "frac_reward_zero_std": 0.5, "grad_norm": 1.3700553178787231, "kl": 0.0019340366125106812, "learning_rate": 1.4285714285714286e-06, "loss": 0.001, "num_tokens": 126317.0, "reward": -0.9375, "reward_std": 0.1767766922712326, "rewards/rollout_reward_func/mean": -0.9375, "rewards/rollout_reward_func/std": 0.25, "sampling/importance_sampling_ratio/max": 1.2087804079055786, "sampling/importance_sampling_ratio/mean": 1.0243126153945923, "sampling/importance_sampling_ratio/min": 0.8649516701698303, "sampling/sampling_logp_difference/max": 0.1895618438720703, "sampling/sampling_logp_difference/mean": 0.027570005506277084, "step": 6, "step_time": 5.888520592000077 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0464323125779629, "epoch": 7e-05, "frac_reward_zero_std": 1.0, "grad_norm": 0.0022114410530775785, "kl": 0.004733920097351074, "learning_rate": 1.7142857142857145e-06, "loss": 0.0, "num_tokens": 145614.0, "reward": -1.0, "reward_std": 0.0, "rewards/rollout_reward_func/mean": -1.0, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.319122076034546, "sampling/importance_sampling_ratio/mean": 1.010606288909912, "sampling/importance_sampling_ratio/min": 0.864284098148346, "sampling/sampling_logp_difference/max": 0.2769632339477539, "sampling/sampling_logp_difference/mean": 0.03655802085995674, "step": 7, "step_time": 5.858400440999844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 518.0, "completions/max_terminated_length": 518.0, "completions/mean_length": 64.375, "completions/mean_terminated_length": 64.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.018570464104414, "epoch": 8e-05, "frac_reward_zero_std": 0.0, "grad_norm": 2.562779188156128, "kl": 0.0009472602978348732, "learning_rate": 2.0000000000000003e-06, "loss": -0.0262, "num_tokens": 167093.0, "reward": -0.375, "reward_std": 0.8880758285522461, "rewards/rollout_reward_func/mean": -0.375, "rewards/rollout_reward_func/std": 0.9574271440505981, "sampling/importance_sampling_ratio/max": 1.1378638744354248, "sampling/importance_sampling_ratio/mean": 0.9837997555732727, "sampling/importance_sampling_ratio/min": 0.725648820400238, "sampling/sampling_logp_difference/max": 0.32072603702545166, "sampling/sampling_logp_difference/mean": 0.03503572568297386, "step": 8, "step_time": 7.824687533999963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.03125, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 475.0, "completions/max_terminated_length": 475.0, "completions/mean_length": 53.8125, "completions/mean_terminated_length": 53.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.041106192395091, "epoch": 9e-05, "frac_reward_zero_std": 0.0, "grad_norm": 2.0428969860076904, "kl": 0.006918169558048248, "learning_rate": 2.285714285714286e-06, "loss": -0.0184, "num_tokens": 186731.0, "reward": -0.7478318214416504, "reward_std": 0.706282913684845, "rewards/rollout_reward_func/mean": -0.7478318214416504, "rewards/rollout_reward_func/std": 0.682338297367096, "sampling/importance_sampling_ratio/max": 1.150418996810913, "sampling/importance_sampling_ratio/mean": 0.9675408601760864, "sampling/importance_sampling_ratio/min": 0.7137911915779114, "sampling/sampling_logp_difference/max": 0.33713316917419434, "sampling/sampling_logp_difference/mean": 0.03557923063635826, "step": 9, "step_time": 7.2318360919999805 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 349.0, "completions/max_terminated_length": 349.0, "completions/mean_length": 40.875, "completions/mean_terminated_length": 40.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.008019097149372, "epoch": 0.0001, "frac_reward_zero_std": 0.0, "grad_norm": 1.9681580066680908, "kl": 0.003238869830965996, "learning_rate": 2.571428571428571e-06, "loss": -0.0182, "num_tokens": 206933.0, "reward": -0.3098890781402588, "reward_std": 0.9037871360778809, "rewards/rollout_reward_func/mean": -0.3098890781402588, "rewards/rollout_reward_func/std": 0.9445173740386963, "sampling/importance_sampling_ratio/max": 1.1492347717285156, "sampling/importance_sampling_ratio/mean": 1.0378103256225586, "sampling/importance_sampling_ratio/min": 0.8731899857521057, "sampling/sampling_logp_difference/max": 0.13912129402160645, "sampling/sampling_logp_difference/mean": 0.03102250210940838, "step": 10, "step_time": 6.932078002999788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 358.0, "completions/max_terminated_length": 358.0, "completions/mean_length": 24.25, "completions/mean_terminated_length": 24.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1186327040195465, "epoch": 0.00011, "frac_reward_zero_std": 0.0, "grad_norm": 1.6200023889541626, "kl": 0.005031676962971687, "learning_rate": 2.8571428571428573e-06, "loss": -0.0307, "num_tokens": 227361.0, "reward": -0.75, "reward_std": 0.7071067690849304, "rewards/rollout_reward_func/mean": -0.75, "rewards/rollout_reward_func/std": 0.6831300854682922, "sampling/importance_sampling_ratio/max": 1.3025950193405151, "sampling/importance_sampling_ratio/mean": 0.9951443076133728, "sampling/importance_sampling_ratio/min": 0.8324748277664185, "sampling/sampling_logp_difference/max": 0.1833159327507019, "sampling/sampling_logp_difference/mean": 0.04638058319687843, "step": 11, "step_time": 7.351286837999851 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 466.0, "completions/max_terminated_length": 466.0, "completions/mean_length": 63.4375, "completions/mean_terminated_length": 66.60000610351562, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0921363905072212, "epoch": 0.00012, "frac_reward_zero_std": 0.0, "grad_norm": 2.049325466156006, "kl": 0.008598579559475183, "learning_rate": 3.142857142857143e-06, "loss": -0.0254, "num_tokens": 248077.0, "reward": -0.546875, "reward_std": 0.8209621906280518, "rewards/rollout_reward_func/mean": -0.546875, "rewards/rollout_reward_func/std": 0.8277416825294495, "sampling/importance_sampling_ratio/max": 1.197043538093567, "sampling/importance_sampling_ratio/mean": 0.9667461514472961, "sampling/importance_sampling_ratio/min": 0.6697664260864258, "sampling/sampling_logp_difference/max": 0.3883552551269531, "sampling/sampling_logp_difference/mean": 0.039610326290130615, "step": 12, "step_time": 7.302310625999894 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 704.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 64.75, "completions/mean_terminated_length": 64.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1115446649491787, "epoch": 0.00013, "frac_reward_zero_std": 0.0, "grad_norm": 2.252307176589966, "kl": 0.02186025306582451, "learning_rate": 3.428571428571429e-06, "loss": -0.048, "num_tokens": 268414.0, "reward": -0.7462500333786011, "reward_std": 0.5920868515968323, "rewards/rollout_reward_func/mean": -0.7462500333786011, "rewards/rollout_reward_func/std": 0.5827163457870483, "sampling/importance_sampling_ratio/max": 1.4250807762145996, "sampling/importance_sampling_ratio/mean": 1.0074121952056885, "sampling/importance_sampling_ratio/min": 0.8410099744796753, "sampling/sampling_logp_difference/max": 0.35880327224731445, "sampling/sampling_logp_difference/mean": 0.03963453322649002, "step": 13, "step_time": 7.840194392000285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2.0, "completions/max_terminated_length": 2.0, "completions/mean_length": 2.0, "completions/mean_terminated_length": 2.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1277522258460522, "epoch": 0.00014, "frac_reward_zero_std": 0.0, "grad_norm": 2.7571370601654053, "kl": 0.05991273373365402, "learning_rate": 3.7142857142857146e-06, "loss": -0.009, "num_tokens": 286792.0, "reward": -0.625, "reward_std": 0.8164634704589844, "rewards/rollout_reward_func/mean": -0.625, "rewards/rollout_reward_func/std": 0.8062257766723633, "sampling/importance_sampling_ratio/max": 1.601056456565857, "sampling/importance_sampling_ratio/mean": 1.031308650970459, "sampling/importance_sampling_ratio/min": 0.7049098014831543, "sampling/sampling_logp_difference/max": 0.4706912040710449, "sampling/sampling_logp_difference/mean": 0.07932695746421814, "step": 14, "step_time": 6.468214063000119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.03125, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 704.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 45.875, "completions/mean_terminated_length": 45.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2666502557694912, "epoch": 0.00015, "frac_reward_zero_std": 0.0, "grad_norm": 2.152557134628296, "kl": 0.06849325262010098, "learning_rate": 4.000000000000001e-06, "loss": -0.03, "num_tokens": 306872.0, "reward": -0.497831791639328, "reward_std": 0.8693404197692871, "rewards/rollout_reward_func/mean": -0.497831791639328, "rewards/rollout_reward_func/std": 0.8931755423545837, "sampling/importance_sampling_ratio/max": 1.4419734477996826, "sampling/importance_sampling_ratio/mean": 0.9915951490402222, "sampling/importance_sampling_ratio/min": 0.6217711567878723, "sampling/sampling_logp_difference/max": 0.4751238226890564, "sampling/sampling_logp_difference/mean": 0.10903055220842361, "step": 15, "step_time": 7.690178571999809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1967.0, "completions/max_terminated_length": 1967.0, "completions/mean_length": 256.3125, "completions/mean_terminated_length": 272.3333435058594, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1599920578300953, "epoch": 0.00016, "frac_reward_zero_std": 0.0, "grad_norm": 2.1746504306793213, "kl": 0.07700802898034453, "learning_rate": 4.2857142857142855e-06, "loss": -0.1384, "num_tokens": 330674.0, "reward": -0.058750003576278687, "reward_std": 1.0126376152038574, "rewards/rollout_reward_func/mean": -0.058750003576278687, "rewards/rollout_reward_func/std": 0.9982777237892151, "sampling/importance_sampling_ratio/max": 1.503495216369629, "sampling/importance_sampling_ratio/mean": 1.054007649421692, "sampling/importance_sampling_ratio/min": 0.6539998650550842, "sampling/sampling_logp_difference/max": 0.42453986406326294, "sampling/sampling_logp_difference/mean": 0.06378096342086792, "step": 16, "step_time": 12.070512280000116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 518.0, "completions/max_terminated_length": 518.0, "completions/mean_length": 52.5625, "completions/mean_terminated_length": 52.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3694819770753384, "epoch": 0.00017, "frac_reward_zero_std": 0.0, "grad_norm": 2.7990901470184326, "kl": 0.304914265871048, "learning_rate": 4.571428571428572e-06, "loss": -0.0654, "num_tokens": 351514.0, "reward": -0.25, "reward_std": 0.9974325299263, "rewards/rollout_reward_func/mean": -0.25, "rewards/rollout_reward_func/std": 1.0, "sampling/importance_sampling_ratio/max": 1.5883798599243164, "sampling/importance_sampling_ratio/mean": 0.9579206705093384, "sampling/importance_sampling_ratio/min": 0.5104092955589294, "sampling/sampling_logp_difference/max": 0.6725739240646362, "sampling/sampling_logp_difference/mean": 0.16684041917324066, "step": 17, "step_time": 7.759517485999709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 596.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 133.9375, "completions/mean_terminated_length": 133.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2054274566471577, "epoch": 0.00018, "frac_reward_zero_std": 0.0, "grad_norm": 2.7592034339904785, "kl": 0.33732227608561516, "learning_rate": 4.857142857142858e-06, "loss": -0.0936, "num_tokens": 372868.0, "reward": -0.30147087574005127, "reward_std": 0.9599188566207886, "rewards/rollout_reward_func/mean": -0.30147087574005127, "rewards/rollout_reward_func/std": 0.9500412344932556, "sampling/importance_sampling_ratio/max": 2.3608694076538086, "sampling/importance_sampling_ratio/mean": 1.1476399898529053, "sampling/importance_sampling_ratio/min": 0.4477699398994446, "sampling/sampling_logp_difference/max": 0.8030790090560913, "sampling/sampling_logp_difference/mean": 0.17698760330677032, "step": 18, "step_time": 7.563858969000194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 686.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 166.625, "completions/mean_terminated_length": 166.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1741414666175842, "epoch": 0.00019, "frac_reward_zero_std": 0.0, "grad_norm": 1.9930346012115479, "kl": 0.5053966641426086, "learning_rate": 5.142857142857142e-06, "loss": -0.0852, "num_tokens": 395205.0, "reward": -0.35473763942718506, "reward_std": 0.7216635346412659, "rewards/rollout_reward_func/mean": -0.35473763942718506, "rewards/rollout_reward_func/std": 0.8181496858596802, "sampling/importance_sampling_ratio/max": 1.9449717998504639, "sampling/importance_sampling_ratio/mean": 1.1056256294250488, "sampling/importance_sampling_ratio/min": 0.26539865136146545, "sampling/sampling_logp_difference/max": 1.3262213468551636, "sampling/sampling_logp_difference/mean": 0.1563001573085785, "step": 19, "step_time": 7.902825313999756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 623.0, "completions/max_terminated_length": 623.0, "completions/mean_length": 205.25, "completions/mean_terminated_length": 205.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9885915629565716, "epoch": 0.0002, "frac_reward_zero_std": 0.0, "grad_norm": 1.3921939134597778, "kl": 1.2418246064335108, "learning_rate": 5.428571428571429e-06, "loss": -0.0776, "num_tokens": 418307.0, "reward": -0.04944445192813873, "reward_std": 0.915343165397644, "rewards/rollout_reward_func/mean": -0.04944445192813873, "rewards/rollout_reward_func/std": 0.9963632822036743, "sampling/importance_sampling_ratio/max": 2.3954970836639404, "sampling/importance_sampling_ratio/mean": 1.1121314764022827, "sampling/importance_sampling_ratio/min": 0.17972268164157867, "sampling/sampling_logp_difference/max": 1.716068148612976, "sampling/sampling_logp_difference/mean": 0.16930413246154785, "step": 20, "step_time": 8.340594768000074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 596.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 165.25, "completions/mean_terminated_length": 165.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9312980249524117, "epoch": 0.00021, "frac_reward_zero_std": 0.0, "grad_norm": 1.646899700164795, "kl": 2.2805014178156853, "learning_rate": 5.7142857142857145e-06, "loss": -0.0563, "num_tokens": 440062.0, "reward": -0.18070214986801147, "reward_std": 0.986350953578949, "rewards/rollout_reward_func/mean": -0.18070214986801147, "rewards/rollout_reward_func/std": 0.975208580493927, "sampling/importance_sampling_ratio/max": 2.236785411834717, "sampling/importance_sampling_ratio/mean": 0.9985036253929138, "sampling/importance_sampling_ratio/min": 0.06830581277608871, "sampling/sampling_logp_difference/max": 2.683295965194702, "sampling/sampling_logp_difference/mean": 0.1988259106874466, "step": 21, "step_time": 7.823992479000026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1123.0, "completions/max_terminated_length": 1123.0, "completions/mean_length": 242.625, "completions/mean_terminated_length": 242.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8350914176553488, "epoch": 0.00022, "frac_reward_zero_std": 0.0, "grad_norm": 1.6486728191375732, "kl": 4.440960839390755, "learning_rate": 6e-06, "loss": -0.0801, "num_tokens": 463789.0, "reward": 0.0625, "reward_std": 0.9756118655204773, "rewards/rollout_reward_func/mean": 0.0625, "rewards/rollout_reward_func/std": 0.9979145526885986, "sampling/importance_sampling_ratio/max": 1.5651099681854248, "sampling/importance_sampling_ratio/mean": 0.844508707523346, "sampling/importance_sampling_ratio/min": 0.0491236187517643, "sampling/sampling_logp_difference/max": 3.0129523277282715, "sampling/sampling_logp_difference/mean": 0.2419586032629013, "step": 22, "step_time": 9.51824111899964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1316.0, "completions/max_terminated_length": 1316.0, "completions/mean_length": 333.1875, "completions/mean_terminated_length": 333.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7513817586004734, "epoch": 0.00023, "frac_reward_zero_std": 0.0, "grad_norm": 1.657171368598938, "kl": 0.43890574388206005, "learning_rate": 6.285714285714286e-06, "loss": -0.0349, "num_tokens": 489083.0, "reward": 0.4499305486679077, "reward_std": 0.6326977610588074, "rewards/rollout_reward_func/mean": 0.4499305486679077, "rewards/rollout_reward_func/std": 0.7994221448898315, "sampling/importance_sampling_ratio/max": 2.3039395809173584, "sampling/importance_sampling_ratio/mean": 1.1215431690216064, "sampling/importance_sampling_ratio/min": 0.29045793414115906, "sampling/sampling_logp_difference/max": 1.2360291481018066, "sampling/sampling_logp_difference/mean": 0.1459958255290985, "step": 23, "step_time": 10.492668100999936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 704.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 186.125, "completions/mean_terminated_length": 186.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6326284091919661, "epoch": 0.00024, "frac_reward_zero_std": 0.0, "grad_norm": 0.816214919090271, "kl": 0.870619298890233, "learning_rate": 6.571428571428572e-06, "loss": -0.0496, "num_tokens": 511257.0, "reward": 0.3787499964237213, "reward_std": 0.864062488079071, "rewards/rollout_reward_func/mean": 0.3787499964237213, "rewards/rollout_reward_func/std": 0.8834921717643738, "sampling/importance_sampling_ratio/max": 2.972301721572876, "sampling/importance_sampling_ratio/mean": 0.9581561088562012, "sampling/importance_sampling_ratio/min": 0.14460575580596924, "sampling/sampling_logp_difference/max": 1.9335517883300781, "sampling/sampling_logp_difference/mean": 0.22764205932617188, "step": 24, "step_time": 8.087724256999877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.03125, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 632.0, "completions/max_terminated_length": 632.0, "completions/mean_length": 180.875, "completions/mean_terminated_length": 180.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6415531504899263, "epoch": 0.00025, "frac_reward_zero_std": 0.0, "grad_norm": 0.9784836769104004, "kl": 3.3794741984456778, "learning_rate": 6.857142857142858e-06, "loss": -0.0801, "num_tokens": 533655.0, "reward": 0.2565123438835144, "reward_std": 0.9484091401100159, "rewards/rollout_reward_func/mean": 0.2565123438835144, "rewards/rollout_reward_func/std": 0.9260807037353516, "sampling/importance_sampling_ratio/max": 2.3607020378112793, "sampling/importance_sampling_ratio/mean": 1.0564424991607666, "sampling/importance_sampling_ratio/min": 0.09664326161146164, "sampling/sampling_logp_difference/max": 2.336583137512207, "sampling/sampling_logp_difference/mean": 0.21205143630504608, "step": 25, "step_time": 7.82037621999973 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 632.0, "completions/max_terminated_length": 632.0, "completions/mean_length": 245.1875, "completions/mean_terminated_length": 245.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5512183737009764, "epoch": 0.00026, "frac_reward_zero_std": 0.0, "grad_norm": 1.1289020776748657, "kl": 2.099851155653596, "learning_rate": 7.1428571428571436e-06, "loss": -0.0686, "num_tokens": 557148.0, "reward": 0.3202931880950928, "reward_std": 0.8082438111305237, "rewards/rollout_reward_func/mean": 0.3202931880950928, "rewards/rollout_reward_func/std": 0.7897863388061523, "sampling/importance_sampling_ratio/max": 2.233189582824707, "sampling/importance_sampling_ratio/mean": 1.1004259586334229, "sampling/importance_sampling_ratio/min": 0.18781760334968567, "sampling/sampling_logp_difference/max": 1.6721630096435547, "sampling/sampling_logp_difference/mean": 0.20380601286888123, "step": 26, "step_time": 8.375234424000041 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.03125, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 695.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 188.1875, "completions/mean_terminated_length": 188.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4561720583587885, "epoch": 0.00027, "frac_reward_zero_std": 0.0, "grad_norm": 1.0175036191940308, "kl": 1.158043622970581, "learning_rate": 7.428571428571429e-06, "loss": -0.0297, "num_tokens": 579090.0, "reward": 0.7049999833106995, "reward_std": 0.5722082257270813, "rewards/rollout_reward_func/mean": 0.7049999833106995, "rewards/rollout_reward_func/std": 0.583072304725647, "sampling/importance_sampling_ratio/max": 1.447882890701294, "sampling/importance_sampling_ratio/mean": 1.0359312295913696, "sampling/importance_sampling_ratio/min": 0.22727327048778534, "sampling/sampling_logp_difference/max": 1.4814252853393555, "sampling/sampling_logp_difference/mean": 0.16597765684127808, "step": 27, "step_time": 8.023200371000144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.03125, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1265.0, "completions/max_terminated_length": 1265.0, "completions/mean_length": 238.1875, "completions/mean_terminated_length": 238.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.38121358351781964, "epoch": 0.00028, "frac_reward_zero_std": 0.0, "grad_norm": 2.2206480503082275, "kl": 2.0255409125238657, "learning_rate": 7.714285714285716e-06, "loss": -0.0131, "num_tokens": 601116.0, "reward": 0.4506250023841858, "reward_std": 0.7092218399047852, "rewards/rollout_reward_func/mean": 0.4506250023841858, "rewards/rollout_reward_func/std": 0.8068991303443909, "sampling/importance_sampling_ratio/max": 1.9049301147460938, "sampling/importance_sampling_ratio/mean": 0.9520484805107117, "sampling/importance_sampling_ratio/min": 0.10445119440555573, "sampling/sampling_logp_difference/max": 2.258340358734131, "sampling/sampling_logp_difference/mean": 0.26176756620407104, "step": 28, "step_time": 9.744828390999942 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 695.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 358.6875, "completions/mean_terminated_length": 358.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3781903591006994, "epoch": 0.00029, "frac_reward_zero_std": 0.0, "grad_norm": 0.5080973505973816, "kl": 2.0241230558604, "learning_rate": 8.000000000000001e-06, "loss": -0.0363, "num_tokens": 627228.0, "reward": 0.26874998211860657, "reward_std": 0.7001081705093384, "rewards/rollout_reward_func/mean": 0.26874998211860657, "rewards/rollout_reward_func/std": 0.6763812899589539, "sampling/importance_sampling_ratio/max": 1.5912672281265259, "sampling/importance_sampling_ratio/mean": 0.860742449760437, "sampling/importance_sampling_ratio/min": 0.10278856754302979, "sampling/sampling_logp_difference/max": 2.27457332611084, "sampling/sampling_logp_difference/mean": 0.21744725108146667, "step": 29, "step_time": 8.69209920499975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 605.0, "completions/max_terminated_length": 605.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 256.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3215527692809701, "epoch": 0.0003, "frac_reward_zero_std": 0.5, "grad_norm": 0.20506170392036438, "kl": 1.6922115366905928, "learning_rate": 8.285714285714287e-06, "loss": -0.0232, "num_tokens": 650846.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/rollout_reward_func/mean": 0.875, "rewards/rollout_reward_func/std": 0.5, "sampling/importance_sampling_ratio/max": 1.4044331312179565, "sampling/importance_sampling_ratio/mean": 0.9546829462051392, "sampling/importance_sampling_ratio/min": 0.13721878826618195, "sampling/sampling_logp_difference/max": 1.9857816696166992, "sampling/sampling_logp_difference/mean": 0.17596650123596191, "step": 30, "step_time": 7.927360212999929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 722.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 272.25, "completions/mean_terminated_length": 272.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3381993127986789, "epoch": 0.00031, "frac_reward_zero_std": 0.0, "grad_norm": 0.9352068305015564, "kl": 3.7475829999893904, "learning_rate": 8.571428571428571e-06, "loss": -0.0734, "num_tokens": 674939.0, "reward": 0.347541481256485, "reward_std": 0.7096856236457825, "rewards/rollout_reward_func/mean": 0.347541481256485, "rewards/rollout_reward_func/std": 0.6897018551826477, "sampling/importance_sampling_ratio/max": 2.054758071899414, "sampling/importance_sampling_ratio/mean": 1.0012009143829346, "sampling/importance_sampling_ratio/min": 0.05817859247326851, "sampling/sampling_logp_difference/max": 2.84386944770813, "sampling/sampling_logp_difference/mean": 0.18952414393424988, "step": 31, "step_time": 8.25228239400019 } ], "logging_steps": 1.0, "max_steps": 1000, "num_input_tokens_seen": 674939, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }