{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.01388, "eval_steps": 500, "global_step": 694, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 31.21875, "completions/mean_terminated_length": 31.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9320596642792225, "epoch": 2e-05, "frac_reward_zero_std": 0.5, "grad_norm": 0.7898357510566711, "kl": 0.0, "learning_rate": 0.0, "loss": -0.0141, "num_tokens": 40962.0, "reward": -0.9375, "reward_std": 0.1767766922712326, "rewards/rollout_reward_func/mean": -0.9375, "rewards/rollout_reward_func/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.2994585037231445, "sampling/importance_sampling_ratio/mean": 0.9931136965751648, "sampling/importance_sampling_ratio/min": 0.6983580589294434, "sampling/sampling_logp_difference/max": 0.36971378326416016, "sampling/sampling_logp_difference/mean": 0.042521096765995026, "step": 1, "step_time": 11.666437403982854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 79.125, "completions/mean_terminated_length": 79.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9518011882901192, "epoch": 4e-05, "frac_reward_zero_std": 0.5, "grad_norm": 1.0171207189559937, "kl": 0.0, "learning_rate": 2.8571428571428575e-07, "loss": 0.0075, "num_tokens": 84359.0, "reward": -1.03125, "reward_std": 0.31963658332824707, "rewards/rollout_reward_func/mean": -1.03125, "rewards/rollout_reward_func/std": 0.5378796458244324, "sampling/importance_sampling_ratio/max": 1.10204017162323, "sampling/importance_sampling_ratio/mean": 0.979063868522644, "sampling/importance_sampling_ratio/min": 0.7147143483161926, "sampling/sampling_logp_difference/max": 0.33587169647216797, "sampling/sampling_logp_difference/mean": 0.03298093006014824, "step": 2, "step_time": 11.910691507990123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 368.0, "completions/max_terminated_length": 368.0, "completions/mean_length": 34.5, "completions/mean_terminated_length": 34.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.919822309166193, "epoch": 6e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.9460965394973755, "kl": 0.0016176775097846985, "learning_rate": 5.714285714285715e-07, "loss": -0.0273, "num_tokens": 124310.0, "reward": -0.84375, "reward_std": 0.6187183856964111, "rewards/rollout_reward_func/mean": -0.84375, "rewards/rollout_reward_func/std": 0.6772512793540955, "sampling/importance_sampling_ratio/max": 1.3399074077606201, "sampling/importance_sampling_ratio/mean": 1.0270624160766602, "sampling/importance_sampling_ratio/min": 0.8663433790206909, "sampling/sampling_logp_difference/max": 0.2960031032562256, "sampling/sampling_logp_difference/mean": 0.032524481415748596, "step": 3, "step_time": 10.455114444994251 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 53.5625, "completions/mean_terminated_length": 53.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9117586705833673, "epoch": 8e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.2431968450546265, "kl": 0.0026596030220389366, "learning_rate": 8.571428571428572e-07, "loss": -0.0069, "num_tokens": 165970.0, "reward": -0.65625, "reward_std": 0.6651592254638672, "rewards/rollout_reward_func/mean": -0.65625, "rewards/rollout_reward_func/std": 0.7873751521110535, "sampling/importance_sampling_ratio/max": 1.0963692665100098, "sampling/importance_sampling_ratio/mean": 0.9545574188232422, "sampling/importance_sampling_ratio/min": 0.5425914525985718, "sampling/sampling_logp_difference/max": 0.4042701721191406, "sampling/sampling_logp_difference/mean": 0.04749114066362381, "step": 4, "step_time": 10.876328090016614 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 73.375, "completions/mean_terminated_length": 73.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9194038733839989, "epoch": 0.0001, "frac_reward_zero_std": 0.5, "grad_norm": 0.7227388024330139, "kl": 0.003440363332629204, "learning_rate": 1.142857142857143e-06, "loss": 0.002, "num_tokens": 208418.0, "reward": -0.875, "reward_std": 0.5055644512176514, "rewards/rollout_reward_func/mean": -0.875, "rewards/rollout_reward_func/std": 0.7071067690849304, "sampling/importance_sampling_ratio/max": 1.2115042209625244, "sampling/importance_sampling_ratio/mean": 1.0080296993255615, "sampling/importance_sampling_ratio/min": 0.7745723724365234, "sampling/sampling_logp_difference/max": 0.25539708137512207, "sampling/sampling_logp_difference/mean": 0.039329174906015396, "step": 5, "step_time": 12.266140054009156 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 92.03125, "completions/mean_terminated_length": 92.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9614051133394241, "epoch": 0.00012, "frac_reward_zero_std": 0.25, "grad_norm": 1.527220368385315, "kl": 0.002038201317191124, "learning_rate": 1.4285714285714286e-06, "loss": -0.0005, "num_tokens": 251488.0, "reward": -0.84375, "reward_std": 0.573716402053833, "rewards/rollout_reward_func/mean": -0.84375, "rewards/rollout_reward_func/std": 0.6772512793540955, "sampling/importance_sampling_ratio/max": 1.413708209991455, "sampling/importance_sampling_ratio/mean": 1.017066478729248, "sampling/importance_sampling_ratio/min": 0.6645265221595764, "sampling/sampling_logp_difference/max": 0.26415079832077026, "sampling/sampling_logp_difference/mean": 0.04878878593444824, "step": 6, "step_time": 11.094186344984337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 19.84375, "completions/mean_terminated_length": 19.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9841081313788891, "epoch": 0.00014, "frac_reward_zero_std": 0.25, "grad_norm": 0.9343042969703674, "kl": 0.002496674656867981, "learning_rate": 1.7142857142857145e-06, "loss": -0.0013, "num_tokens": 291765.0, "reward": -0.71875, "reward_std": 0.5825847387313843, "rewards/rollout_reward_func/mean": -0.71875, "rewards/rollout_reward_func/std": 0.6831791996955872, "sampling/importance_sampling_ratio/max": 1.3241853713989258, "sampling/importance_sampling_ratio/mean": 0.9879853129386902, "sampling/importance_sampling_ratio/min": 0.7658634781837463, "sampling/sampling_logp_difference/max": 0.2809000015258789, "sampling/sampling_logp_difference/mean": 0.03534495830535889, "step": 7, "step_time": 10.827667811958236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 80.09375, "completions/mean_terminated_length": 80.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9165958873927593, "epoch": 0.00016, "frac_reward_zero_std": 0.25, "grad_norm": 1.1118626594543457, "kl": 0.0009985826909542084, "learning_rate": 2.0000000000000003e-06, "loss": -0.0042, "num_tokens": 334791.0, "reward": -0.9375, "reward_std": 0.4972116947174072, "rewards/rollout_reward_func/mean": -0.9375, "rewards/rollout_reward_func/std": 0.6690146923065186, "sampling/importance_sampling_ratio/max": 1.219115972518921, "sampling/importance_sampling_ratio/mean": 1.0235199928283691, "sampling/importance_sampling_ratio/min": 0.8644849061965942, "sampling/sampling_logp_difference/max": 0.19476079940795898, "sampling/sampling_logp_difference/mean": 0.026307199150323868, "step": 8, "step_time": 11.761940058975597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 50.3125, "completions/mean_terminated_length": 50.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9207998104393482, "epoch": 0.00018, "frac_reward_zero_std": 0.0, "grad_norm": 1.1932973861694336, "kl": 0.0016294466331601143, "learning_rate": 2.285714285714286e-06, "loss": 0.0028, "num_tokens": 374773.0, "reward": -0.8125, "reward_std": 0.7087852954864502, "rewards/rollout_reward_func/mean": -0.8125, "rewards/rollout_reward_func/std": 0.692703902721405, "sampling/importance_sampling_ratio/max": 1.1484581232070923, "sampling/importance_sampling_ratio/mean": 0.9969648122787476, "sampling/importance_sampling_ratio/min": 0.7797396779060364, "sampling/sampling_logp_difference/max": 0.24890422821044922, "sampling/sampling_logp_difference/mean": 0.036096103489398956, "step": 9, "step_time": 10.977762839975185 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 384.0, "completions/max_terminated_length": 384.0, "completions/mean_length": 42.5625, "completions/mean_terminated_length": 42.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9118516556918621, "epoch": 0.0002, "frac_reward_zero_std": 0.25, "grad_norm": 1.195055603981018, "kl": 0.0010573174804449081, "learning_rate": 2.571428571428571e-06, "loss": 0.023, "num_tokens": 415405.0, "reward": -1.03125, "reward_std": 0.42571717500686646, "rewards/rollout_reward_func/mean": -1.03125, "rewards/rollout_reward_func/std": 0.5378796458244324, "sampling/importance_sampling_ratio/max": 1.2162063121795654, "sampling/importance_sampling_ratio/mean": 0.9777395725250244, "sampling/importance_sampling_ratio/min": 0.7368171215057373, "sampling/sampling_logp_difference/max": 0.22201776504516602, "sampling/sampling_logp_difference/mean": 0.03685835003852844, "step": 10, "step_time": 10.365177698025946 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017361111240461469, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 87.59375, "completions/mean_terminated_length": 87.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9608894847333431, "epoch": 0.00022, "frac_reward_zero_std": 0.0, "grad_norm": 3.070436954498291, "kl": 0.0027240392519161105, "learning_rate": 2.8571428571428573e-06, "loss": -0.0994, "num_tokens": 458482.0, "reward": -0.5625, "reward_std": 0.8634915947914124, "rewards/rollout_reward_func/mean": -0.5625, "rewards/rollout_reward_func/std": 0.8775883316993713, "sampling/importance_sampling_ratio/max": 1.5278359651565552, "sampling/importance_sampling_ratio/mean": 1.0064283609390259, "sampling/importance_sampling_ratio/min": 0.8402748703956604, "sampling/sampling_logp_difference/max": 0.25031280517578125, "sampling/sampling_logp_difference/mean": 0.03718063607811928, "step": 11, "step_time": 11.891978522980935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 70.125, "completions/mean_terminated_length": 70.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9442263394594193, "epoch": 0.00024, "frac_reward_zero_std": 0.25, "grad_norm": 0.8621652722358704, "kl": 0.0034004333429038525, "learning_rate": 3.142857142857143e-06, "loss": -0.0028, "num_tokens": 499617.0, "reward": -0.96875, "reward_std": 0.4419417381286621, "rewards/rollout_reward_func/mean": -0.96875, "rewards/rollout_reward_func/std": 0.5378796458244324, "sampling/importance_sampling_ratio/max": 1.272804856300354, "sampling/importance_sampling_ratio/mean": 1.0193549394607544, "sampling/importance_sampling_ratio/min": 0.7496445775032043, "sampling/sampling_logp_difference/max": 0.2881956100463867, "sampling/sampling_logp_difference/mean": 0.04022333770990372, "step": 12, "step_time": 11.162492713992833 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 128.40625, "completions/mean_terminated_length": 128.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8857224620878696, "epoch": 0.00026, "frac_reward_zero_std": 0.0, "grad_norm": 0.9342085719108582, "kl": 0.0021988730877637863, "learning_rate": 3.428571428571429e-06, "loss": -0.0009, "num_tokens": 544226.0, "reward": -0.9375, "reward_std": 0.6573334336280823, "rewards/rollout_reward_func/mean": -0.9375, "rewards/rollout_reward_func/std": 0.6690146923065186, "sampling/importance_sampling_ratio/max": 1.2053016424179077, "sampling/importance_sampling_ratio/mean": 0.9945752620697021, "sampling/importance_sampling_ratio/min": 0.6667356491088867, "sampling/sampling_logp_difference/max": 0.3487039804458618, "sampling/sampling_logp_difference/mean": 0.04010609909892082, "step": 13, "step_time": 11.239136504023918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 59.875, "completions/mean_terminated_length": 61.290321350097656, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8813750371336937, "epoch": 0.00028, "frac_reward_zero_std": 0.25, "grad_norm": 0.8487452268600464, "kl": 0.0030243367655202746, "learning_rate": 3.7142857142857146e-06, "loss": 0.0007, "num_tokens": 584692.0, "reward": -0.84375, "reward_std": 0.5621840953826904, "rewards/rollout_reward_func/mean": -0.84375, "rewards/rollout_reward_func/std": 0.7233155965805054, "sampling/importance_sampling_ratio/max": 1.2396714687347412, "sampling/importance_sampling_ratio/mean": 0.9816467761993408, "sampling/importance_sampling_ratio/min": 0.5130836963653564, "sampling/sampling_logp_difference/max": 0.684412956237793, "sampling/sampling_logp_difference/mean": 0.035629384219646454, "step": 14, "step_time": 12.402158952972968 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 115.5, "completions/mean_terminated_length": 115.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9071248434484005, "epoch": 0.0003, "frac_reward_zero_std": 0.0, "grad_norm": 1.2770183086395264, "kl": 0.003599373158067465, "learning_rate": 4.000000000000001e-06, "loss": 0.0093, "num_tokens": 627478.0, "reward": -0.9375, "reward_std": 0.6879827976226807, "rewards/rollout_reward_func/mean": -0.9375, "rewards/rollout_reward_func/std": 0.7156093716621399, "sampling/importance_sampling_ratio/max": 1.257986068725586, "sampling/importance_sampling_ratio/mean": 0.9987573623657227, "sampling/importance_sampling_ratio/min": 0.7830877304077148, "sampling/sampling_logp_difference/max": 0.3451664447784424, "sampling/sampling_logp_difference/mean": 0.040524572134017944, "step": 15, "step_time": 11.145108363009058 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 93.84375, "completions/mean_terminated_length": 93.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.916128795593977, "epoch": 0.00032, "frac_reward_zero_std": 0.0, "grad_norm": 1.0917410850524902, "kl": 0.007343584671616554, "learning_rate": 4.2857142857142855e-06, "loss": 0.0098, "num_tokens": 670949.0, "reward": -0.8125, "reward_std": 0.7436752319335938, "rewards/rollout_reward_func/mean": -0.8125, "rewards/rollout_reward_func/std": 0.7803018093109131, "sampling/importance_sampling_ratio/max": 1.2171841859817505, "sampling/importance_sampling_ratio/mean": 1.0287306308746338, "sampling/importance_sampling_ratio/min": 0.7730493545532227, "sampling/sampling_logp_difference/max": 0.30740928649902344, "sampling/sampling_logp_difference/mean": 0.04861205816268921, "step": 16, "step_time": 11.192658567000763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 63.9375, "completions/mean_terminated_length": 63.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9528503380715847, "epoch": 0.00034, "frac_reward_zero_std": 0.25, "grad_norm": 1.0183051824569702, "kl": 0.0028362837620079517, "learning_rate": 4.571428571428572e-06, "loss": 0.0026, "num_tokens": 712869.0, "reward": -0.96875, "reward_std": 0.5132601261138916, "rewards/rollout_reward_func/mean": -0.96875, "rewards/rollout_reward_func/std": 0.6467972993850708, "sampling/importance_sampling_ratio/max": 1.0882673263549805, "sampling/importance_sampling_ratio/mean": 0.9549283385276794, "sampling/importance_sampling_ratio/min": 0.7942132949829102, "sampling/sampling_logp_difference/max": 0.2304002046585083, "sampling/sampling_logp_difference/mean": 0.03643113374710083, "step": 17, "step_time": 12.581802018961753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.03125, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 89.5, "completions/mean_terminated_length": 79.83870697021484, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8509950712323189, "epoch": 0.00036, "frac_reward_zero_std": 0.0, "grad_norm": 1.6124235391616821, "kl": 0.012776751886121929, "learning_rate": 4.857142857142858e-06, "loss": 0.0232, "num_tokens": 756355.0, "reward": -0.75, "reward_std": 0.7079349160194397, "rewards/rollout_reward_func/mean": -0.75, "rewards/rollout_reward_func/std": 0.7620007395744324, "sampling/importance_sampling_ratio/max": 1.250268816947937, "sampling/importance_sampling_ratio/mean": 1.0029494762420654, "sampling/importance_sampling_ratio/min": 0.7820828557014465, "sampling/sampling_logp_difference/max": 0.22694993019104004, "sampling/sampling_logp_difference/mean": 0.03217899426817894, "step": 18, "step_time": 10.86727241097833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 122.78125, "completions/mean_terminated_length": 122.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8347316533327103, "epoch": 0.00038, "frac_reward_zero_std": 0.0, "grad_norm": 1.2302746772766113, "kl": 0.01652763970196247, "learning_rate": 5.142857142857142e-06, "loss": -0.0171, "num_tokens": 800410.0, "reward": -0.5625, "reward_std": 1.050110936164856, "rewards/rollout_reward_func/mean": -0.5625, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.2601242065429688, "sampling/importance_sampling_ratio/mean": 1.0012304782867432, "sampling/importance_sampling_ratio/min": 0.8176712989807129, "sampling/sampling_logp_difference/max": 0.2490072250366211, "sampling/sampling_logp_difference/mean": 0.03873170167207718, "step": 19, "step_time": 11.91711873699387 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 106.375, "completions/mean_terminated_length": 106.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8444412052631378, "epoch": 0.0004, "frac_reward_zero_std": 0.0, "grad_norm": 1.399171233177185, "kl": 0.03438756428658962, "learning_rate": 5.428571428571429e-06, "loss": -0.0117, "num_tokens": 843565.0, "reward": -0.65625, "reward_std": 0.9135040640830994, "rewards/rollout_reward_func/mean": -0.65625, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 1.5043513774871826, "sampling/importance_sampling_ratio/mean": 1.0462722778320312, "sampling/importance_sampling_ratio/min": 0.7175166606903076, "sampling/sampling_logp_difference/max": 0.7883961200714111, "sampling/sampling_logp_difference/mean": 0.0650564506649971, "step": 20, "step_time": 11.778200944972923 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 32.46875, "completions/mean_terminated_length": 32.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8653258122503757, "epoch": 0.00042, "frac_reward_zero_std": 0.25, "grad_norm": 1.0825802087783813, "kl": 0.033637295477092266, "learning_rate": 5.7142857142857145e-06, "loss": -0.0121, "num_tokens": 884292.0, "reward": -0.65625, "reward_std": 0.662692666053772, "rewards/rollout_reward_func/mean": -0.65625, "rewards/rollout_reward_func/std": 0.7873751521110535, "sampling/importance_sampling_ratio/max": 1.3915132284164429, "sampling/importance_sampling_ratio/mean": 1.0012415647506714, "sampling/importance_sampling_ratio/min": 0.580025315284729, "sampling/sampling_logp_difference/max": 0.5446844100952148, "sampling/sampling_logp_difference/mean": 0.06361827999353409, "step": 21, "step_time": 10.586191855982179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 61.5625, "completions/mean_terminated_length": 61.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8419606909155846, "epoch": 0.00044, "frac_reward_zero_std": 0.0, "grad_norm": 2.0574333667755127, "kl": 0.0628960570320487, "learning_rate": 6e-06, "loss": -0.0307, "num_tokens": 926536.0, "reward": -0.5625, "reward_std": 0.7509444952011108, "rewards/rollout_reward_func/mean": -0.5625, "rewards/rollout_reward_func/std": 0.7593502998352051, "sampling/importance_sampling_ratio/max": 1.802050232887268, "sampling/importance_sampling_ratio/mean": 1.1106624603271484, "sampling/importance_sampling_ratio/min": 0.6562438011169434, "sampling/sampling_logp_difference/max": 0.5891103744506836, "sampling/sampling_logp_difference/mean": 0.08644814789295197, "step": 22, "step_time": 12.466941016027704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 60.28125, "completions/mean_terminated_length": 60.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8583023995161057, "epoch": 0.00046, "frac_reward_zero_std": 0.25, "grad_norm": 0.9862443804740906, "kl": 0.045003571547567844, "learning_rate": 6.285714285714286e-06, "loss": 0.0046, "num_tokens": 968703.0, "reward": -0.96875, "reward_std": 0.47589200735092163, "rewards/rollout_reward_func/mean": -0.96875, "rewards/rollout_reward_func/std": 0.5378796458244324, "sampling/importance_sampling_ratio/max": 1.6494104862213135, "sampling/importance_sampling_ratio/mean": 1.0358936786651611, "sampling/importance_sampling_ratio/min": 0.6717547178268433, "sampling/sampling_logp_difference/max": 0.5004276037216187, "sampling/sampling_logp_difference/mean": 0.0698792040348053, "step": 23, "step_time": 10.88966642699961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 465.0, "completions/max_terminated_length": 465.0, "completions/mean_length": 67.71875, "completions/mean_terminated_length": 67.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8002432510256767, "epoch": 0.00048, "frac_reward_zero_std": 0.0, "grad_norm": 1.9093843698501587, "kl": 0.0992727130651474, "learning_rate": 6.571428571428572e-06, "loss": -0.0187, "num_tokens": 1010887.0, "reward": -0.625, "reward_std": 0.8082882761955261, "rewards/rollout_reward_func/mean": -0.625, "rewards/rollout_reward_func/std": 0.8327955007553101, "sampling/importance_sampling_ratio/max": 1.8569318056106567, "sampling/importance_sampling_ratio/mean": 1.0228205919265747, "sampling/importance_sampling_ratio/min": 0.5314638614654541, "sampling/sampling_logp_difference/max": 0.6190102100372314, "sampling/sampling_logp_difference/mean": 0.08646638691425323, "step": 24, "step_time": 10.776204434980173 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 68.71875, "completions/mean_terminated_length": 68.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7742785513401031, "epoch": 0.0005, "frac_reward_zero_std": 0.0, "grad_norm": 0.9122614860534668, "kl": 0.15454382821917534, "learning_rate": 6.857142857142858e-06, "loss": -0.0271, "num_tokens": 1051833.0, "reward": -0.65625, "reward_std": 0.7814741134643555, "rewards/rollout_reward_func/mean": -0.65625, "rewards/rollout_reward_func/std": 0.8273305296897888, "sampling/importance_sampling_ratio/max": 1.878385305404663, "sampling/importance_sampling_ratio/mean": 0.9446666836738586, "sampling/importance_sampling_ratio/min": 0.495743989944458, "sampling/sampling_logp_difference/max": 0.7016892433166504, "sampling/sampling_logp_difference/mean": 0.1300179809331894, "step": 25, "step_time": 13.374850909982342 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.046875, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 50.59375, "completions/mean_terminated_length": 50.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7700840719044209, "epoch": 0.00052, "frac_reward_zero_std": 0.25, "grad_norm": 0.5478610396385193, "kl": 0.1442856565117836, "learning_rate": 7.1428571428571436e-06, "loss": -0.0111, "num_tokens": 1094290.0, "reward": -0.625, "reward_std": 0.6286503672599792, "rewards/rollout_reward_func/mean": -0.625, "rewards/rollout_reward_func/std": 0.7931155562400818, "sampling/importance_sampling_ratio/max": 2.4175329208374023, "sampling/importance_sampling_ratio/mean": 1.0775666236877441, "sampling/importance_sampling_ratio/min": 0.4036414921283722, "sampling/sampling_logp_difference/max": 0.8828898668289185, "sampling/sampling_logp_difference/mean": 0.11052724719047546, "step": 26, "step_time": 10.967096375941765 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 103.0, "completions/mean_terminated_length": 103.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.704024150967598, "epoch": 0.00054, "frac_reward_zero_std": 0.0, "grad_norm": 1.4181755781173706, "kl": 0.39083763025701046, "learning_rate": 7.428571428571429e-06, "loss": -0.0078, "num_tokens": 1136630.0, "reward": -0.46875, "reward_std": 1.063450574874878, "rewards/rollout_reward_func/mean": -0.46875, "rewards/rollout_reward_func/std": 1.0467884540557861, "sampling/importance_sampling_ratio/max": 2.292524814605713, "sampling/importance_sampling_ratio/mean": 1.088132381439209, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2099802494049072, "sampling/sampling_logp_difference/mean": 0.20815345644950867, "step": 27, "step_time": 11.171268123041955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 348.0, "completions/max_terminated_length": 348.0, "completions/mean_length": 12.8125, "completions/mean_terminated_length": 12.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.684624882414937, "epoch": 0.00056, "frac_reward_zero_std": 0.0, "grad_norm": 0.6015587449073792, "kl": 0.4243163466453552, "learning_rate": 7.714285714285716e-06, "loss": -0.031, "num_tokens": 1175594.0, "reward": -0.59375, "reward_std": 0.9048517942428589, "rewards/rollout_reward_func/mean": -0.59375, "rewards/rollout_reward_func/std": 0.8747119307518005, "sampling/importance_sampling_ratio/max": 1.9673418998718262, "sampling/importance_sampling_ratio/mean": 0.909347414970398, "sampling/importance_sampling_ratio/min": 0.2751339077949524, "sampling/sampling_logp_difference/max": 1.290339469909668, "sampling/sampling_logp_difference/mean": 0.205691397190094, "step": 28, "step_time": 11.282379407057306 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 133.0625, "completions/mean_terminated_length": 133.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6160625051707029, "epoch": 0.00058, "frac_reward_zero_std": 0.0, "grad_norm": 0.936601459980011, "kl": 0.48921788949519396, "learning_rate": 8.000000000000001e-06, "loss": 0.0033, "num_tokens": 1220352.0, "reward": -0.65625, "reward_std": 0.8330508470535278, "rewards/rollout_reward_func/mean": -0.65625, "rewards/rollout_reward_func/std": 0.8273305296897888, "sampling/importance_sampling_ratio/max": 2.27301287651062, "sampling/importance_sampling_ratio/mean": 0.9918374419212341, "sampling/importance_sampling_ratio/min": 0.22528624534606934, "sampling/sampling_logp_difference/max": 1.4902654886245728, "sampling/sampling_logp_difference/mean": 0.19018903374671936, "step": 29, "step_time": 11.21866241801763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 68.5625, "completions/mean_terminated_length": 68.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6394413318485022, "epoch": 0.0006, "frac_reward_zero_std": 0.25, "grad_norm": 0.6389355659484863, "kl": 1.2474178960546851, "learning_rate": 8.285714285714287e-06, "loss": -0.006, "num_tokens": 1262811.0, "reward": -0.375, "reward_std": 0.7529709339141846, "rewards/rollout_reward_func/mean": -0.375, "rewards/rollout_reward_func/std": 0.9755064845085144, "sampling/importance_sampling_ratio/max": 1.9843792915344238, "sampling/importance_sampling_ratio/mean": 0.8709831237792969, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8257040977478027, "sampling/sampling_logp_difference/mean": 0.24921461939811707, "step": 30, "step_time": 10.88124235496798 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 81.21875, "completions/mean_terminated_length": 81.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5992294354364276, "epoch": 0.00062, "frac_reward_zero_std": 0.0, "grad_norm": 1.8153783082962036, "kl": 0.6608234690502286, "learning_rate": 8.571428571428571e-06, "loss": -0.0609, "num_tokens": 1305464.0, "reward": -0.4375, "reward_std": 0.8806005716323853, "rewards/rollout_reward_func/mean": -0.4375, "rewards/rollout_reward_func/std": 0.9482581615447998, "sampling/importance_sampling_ratio/max": 2.14788818359375, "sampling/importance_sampling_ratio/mean": 1.036331295967102, "sampling/importance_sampling_ratio/min": 0.17510269582271576, "sampling/sampling_logp_difference/max": 1.7423536777496338, "sampling/sampling_logp_difference/mean": 0.19263699650764465, "step": 31, "step_time": 12.298982819978846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 53.96875, "completions/mean_terminated_length": 53.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.712474750354886, "epoch": 0.00064, "frac_reward_zero_std": 0.0, "grad_norm": 0.9785094261169434, "kl": 0.8169891908764839, "learning_rate": 8.857142857142858e-06, "loss": -0.007, "num_tokens": 1348173.0, "reward": -0.625, "reward_std": 0.8363723754882812, "rewards/rollout_reward_func/mean": -0.625, "rewards/rollout_reward_func/std": 0.8706690073013306, "sampling/importance_sampling_ratio/max": 1.7954673767089844, "sampling/importance_sampling_ratio/mean": 0.9792878031730652, "sampling/importance_sampling_ratio/min": 0.10586299747228622, "sampling/sampling_logp_difference/max": 2.2456774711608887, "sampling/sampling_logp_difference/mean": 0.15454840660095215, "step": 32, "step_time": 11.200591682005324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 99.5, "completions/mean_terminated_length": 99.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5014531100168824, "epoch": 0.00066, "frac_reward_zero_std": 0.0, "grad_norm": 0.8791465163230896, "kl": 5.375709330663085, "learning_rate": 9.142857142857144e-06, "loss": 0.006, "num_tokens": 1390698.0, "reward": -0.5625, "reward_std": 0.8820203542709351, "rewards/rollout_reward_func/mean": -0.5625, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.657703161239624, "sampling/importance_sampling_ratio/mean": 0.7492748498916626, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3872182369232178, "sampling/sampling_logp_difference/mean": 0.32560622692108154, "step": 33, "step_time": 11.146863386049517 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 105.65625, "completions/mean_terminated_length": 105.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5685148141346872, "epoch": 0.00068, "frac_reward_zero_std": 0.0, "grad_norm": 0.5025176405906677, "kl": 2.250026635825634, "learning_rate": 9.42857142857143e-06, "loss": 0.0075, "num_tokens": 1434335.0, "reward": -0.1875, "reward_std": 0.9932987093925476, "rewards/rollout_reward_func/mean": -0.1875, "rewards/rollout_reward_func/std": 0.9979817867279053, "sampling/importance_sampling_ratio/max": 1.8042256832122803, "sampling/importance_sampling_ratio/mean": 1.0753076076507568, "sampling/importance_sampling_ratio/min": 0.0681660994887352, "sampling/sampling_logp_difference/max": 2.6858246326446533, "sampling/sampling_logp_difference/mean": 0.1499052494764328, "step": 34, "step_time": 12.347806882986333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 186.15625, "completions/mean_terminated_length": 186.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5248282779939473, "epoch": 0.0007, "frac_reward_zero_std": 0.0, "grad_norm": 1.6040096282958984, "kl": 2.6265857629477978, "learning_rate": 9.714285714285715e-06, "loss": 0.0606, "num_tokens": 1481151.0, "reward": -0.40625, "reward_std": 1.0279419422149658, "rewards/rollout_reward_func/mean": -0.40625, "rewards/rollout_reward_func/std": 1.0429292917251587, "sampling/importance_sampling_ratio/max": 2.225717544555664, "sampling/importance_sampling_ratio/mean": 1.1744706630706787, "sampling/importance_sampling_ratio/min": 0.04892890527844429, "sampling/sampling_logp_difference/max": 2.384584903717041, "sampling/sampling_logp_difference/mean": 0.18838027119636536, "step": 35, "step_time": 11.340969707001932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 86.15625, "completions/mean_terminated_length": 86.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5668163103982806, "epoch": 0.00072, "frac_reward_zero_std": 0.0, "grad_norm": 0.6121125221252441, "kl": 2.8937078062444925, "learning_rate": 1e-05, "loss": 0.0018, "num_tokens": 1523406.0, "reward": -0.46875, "reward_std": 1.037715196609497, "rewards/rollout_reward_func/mean": -0.46875, "rewards/rollout_reward_func/std": 1.0155048370361328, "sampling/importance_sampling_ratio/max": 1.965596079826355, "sampling/importance_sampling_ratio/mean": 0.9413849115371704, "sampling/importance_sampling_ratio/min": 0.06682665646076202, "sampling/sampling_logp_difference/max": 2.705662965774536, "sampling/sampling_logp_difference/mean": 0.2623358964920044, "step": 36, "step_time": 11.164119945999118 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 95.375, "completions/mean_terminated_length": 95.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7062911470420659, "epoch": 0.00074, "frac_reward_zero_std": 0.0, "grad_norm": 0.669857919216156, "kl": 0.5864870920777321, "learning_rate": 9.999975267482496e-06, "loss": -0.0238, "num_tokens": 1566774.0, "reward": -0.28125, "reward_std": 0.8704743385314941, "rewards/rollout_reward_func/mean": -0.28125, "rewards/rollout_reward_func/std": 0.9583041071891785, "sampling/importance_sampling_ratio/max": 2.0972633361816406, "sampling/importance_sampling_ratio/mean": 1.04378080368042, "sampling/importance_sampling_ratio/min": 0.32054731249809265, "sampling/sampling_logp_difference/max": 1.4576646089553833, "sampling/sampling_logp_difference/mean": 0.1789175271987915, "step": 37, "step_time": 12.150809079961618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 107.46875, "completions/mean_terminated_length": 107.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7943081762641668, "epoch": 0.00076, "frac_reward_zero_std": 0.0, "grad_norm": 2.0727570056915283, "kl": 5.415150078013539, "learning_rate": 9.99990107025622e-06, "loss": -0.0188, "num_tokens": 1610940.0, "reward": -0.53125, "reward_std": 1.0324573516845703, "rewards/rollout_reward_func/mean": -0.53125, "rewards/rollout_reward_func/std": 1.0467884540557861, "sampling/importance_sampling_ratio/max": 1.6822402477264404, "sampling/importance_sampling_ratio/mean": 0.918075680732727, "sampling/importance_sampling_ratio/min": 0.07511010766029358, "sampling/sampling_logp_difference/max": 2.6937220096588135, "sampling/sampling_logp_difference/mean": 0.2403835654258728, "step": 38, "step_time": 10.930559595988598 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 131.25, "completions/mean_terminated_length": 131.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5162107772193849, "epoch": 0.00078, "frac_reward_zero_std": 0.0, "grad_norm": 0.4657498002052307, "kl": 2.517602464184165, "learning_rate": 9.99977740929988e-06, "loss": -0.0179, "num_tokens": 1654508.0, "reward": -0.3125, "reward_std": 0.9997608661651611, "rewards/rollout_reward_func/mean": -0.3125, "rewards/rollout_reward_func/std": 0.9651173949241638, "sampling/importance_sampling_ratio/max": 1.4146366119384766, "sampling/importance_sampling_ratio/mean": 0.8514896631240845, "sampling/importance_sampling_ratio/min": 0.12617041170597076, "sampling/sampling_logp_difference/max": 2.0699830055236816, "sampling/sampling_logp_difference/mean": 0.2544395923614502, "step": 39, "step_time": 12.483927267981926 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0546875, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 129.53125, "completions/mean_terminated_length": 129.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.541138949804008, "epoch": 0.0008, "frac_reward_zero_std": 0.0, "grad_norm": 1.4927922487258911, "kl": 3.2571986224502325, "learning_rate": 9.999604286244655e-06, "loss": -0.0668, "num_tokens": 1698679.0, "reward": -0.1875, "reward_std": 1.035538673400879, "rewards/rollout_reward_func/mean": -0.1875, "rewards/rollout_reward_func/std": 1.029797911643982, "sampling/importance_sampling_ratio/max": 2.40571928024292, "sampling/importance_sampling_ratio/mean": 1.0399919748306274, "sampling/importance_sampling_ratio/min": 0.11769169569015503, "sampling/sampling_logp_difference/max": 2.1396265029907227, "sampling/sampling_logp_difference/mean": 0.24212923645973206, "step": 40, "step_time": 11.040466864971677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 173.625, "completions/mean_terminated_length": 173.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5250452971085906, "epoch": 0.00082, "frac_reward_zero_std": 0.0, "grad_norm": 0.9037295579910278, "kl": 1.745877175591886, "learning_rate": 9.999381703374151e-06, "loss": -0.0239, "num_tokens": 1743996.0, "reward": -0.125, "reward_std": 0.7969473600387573, "rewards/rollout_reward_func/mean": -0.125, "rewards/rollout_reward_func/std": 0.8327955007553101, "sampling/importance_sampling_ratio/max": 1.5475584268569946, "sampling/importance_sampling_ratio/mean": 0.8600172400474548, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9671783447265625, "sampling/sampling_logp_difference/mean": 0.19052167236804962, "step": 41, "step_time": 11.397497419980937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 95.6875, "completions/mean_terminated_length": 95.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32482594763860106, "epoch": 0.00084, "frac_reward_zero_std": 0.0, "grad_norm": 0.7636421322822571, "kl": 3.318049158900976, "learning_rate": 9.999109663624387e-06, "loss": -0.005, "num_tokens": 1783989.0, "reward": 0.03125, "reward_std": 1.029504656791687, "rewards/rollout_reward_func/mean": 0.03125, "rewards/rollout_reward_func/std": 1.0312652587890625, "sampling/importance_sampling_ratio/max": 1.621386170387268, "sampling/importance_sampling_ratio/mean": 0.9144099354743958, "sampling/importance_sampling_ratio/min": 0.1311909407377243, "sampling/sampling_logp_difference/max": 2.031087875366211, "sampling/sampling_logp_difference/mean": 0.23584392666816711, "step": 42, "step_time": 12.90829740803747 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 228.125, "completions/mean_terminated_length": 228.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.496860945597291, "epoch": 0.00086, "frac_reward_zero_std": 0.0, "grad_norm": 0.7204203605651855, "kl": 2.488615872338414, "learning_rate": 9.99878817058375e-06, "loss": -0.0249, "num_tokens": 1831801.0, "reward": -0.0625, "reward_std": 1.3097376823425293, "rewards/rollout_reward_func/mean": -0.0625, "rewards/rollout_reward_func/std": 1.268412709236145, "sampling/importance_sampling_ratio/max": 1.9912376403808594, "sampling/importance_sampling_ratio/mean": 0.8437385559082031, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9810681343078613, "sampling/sampling_logp_difference/mean": 0.21304546296596527, "step": 43, "step_time": 11.47902102500666 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 217.3125, "completions/mean_terminated_length": 217.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5059409588575363, "epoch": 0.00088, "frac_reward_zero_std": 0.0, "grad_norm": 2.433419942855835, "kl": 3.324080215767026, "learning_rate": 9.998417228492952e-06, "loss": -0.081, "num_tokens": 1879814.0, "reward": -0.21875, "reward_std": 1.201997995376587, "rewards/rollout_reward_func/mean": -0.21875, "rewards/rollout_reward_func/std": 1.2110878229141235, "sampling/importance_sampling_ratio/max": 2.956212043762207, "sampling/importance_sampling_ratio/mean": 1.0323933362960815, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5173940658569336, "sampling/sampling_logp_difference/mean": 0.22386214137077332, "step": 44, "step_time": 11.379070247028721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 243.3125, "completions/mean_terminated_length": 243.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3927864283323288, "epoch": 0.0009, "frac_reward_zero_std": 0.0, "grad_norm": 1.454461932182312, "kl": 3.9159104684367776, "learning_rate": 9.99799684224497e-06, "loss": -0.0318, "num_tokens": 1927601.0, "reward": 0.0625, "reward_std": 1.2900443077087402, "rewards/rollout_reward_func/mean": 0.0625, "rewards/rollout_reward_func/std": 1.268412709236145, "sampling/importance_sampling_ratio/max": 2.5400803089141846, "sampling/importance_sampling_ratio/mean": 1.017512559890747, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2049496173858643, "sampling/sampling_logp_difference/mean": 0.20012876391410828, "step": 45, "step_time": 11.917574603969115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 194.6875, "completions/mean_terminated_length": 194.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3917444758117199, "epoch": 0.00092, "frac_reward_zero_std": 0.0, "grad_norm": 0.6260811686515808, "kl": 0.8299789717420936, "learning_rate": 9.997527017384983e-06, "loss": -0.0557, "num_tokens": 1973505.0, "reward": 0.25, "reward_std": 1.0789108276367188, "rewards/rollout_reward_func/mean": 0.25, "rewards/rollout_reward_func/std": 1.1071614027023315, "sampling/importance_sampling_ratio/max": 2.001307249069214, "sampling/importance_sampling_ratio/mean": 1.0107555389404297, "sampling/importance_sampling_ratio/min": 0.21665838360786438, "sampling/sampling_logp_difference/max": 1.5294818878173828, "sampling/sampling_logp_difference/mean": 0.16527986526489258, "step": 46, "step_time": 11.282765980038675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 164.90625, "completions/mean_terminated_length": 164.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3640204444527626, "epoch": 0.00094, "frac_reward_zero_std": 0.0, "grad_norm": 1.0556226968765259, "kl": 0.7647094307467341, "learning_rate": 9.997007760110298e-06, "loss": -0.0172, "num_tokens": 2017452.0, "reward": 0.3125, "reward_std": 1.037058711051941, "rewards/rollout_reward_func/mean": 0.3125, "rewards/rollout_reward_func/std": 1.0606601238250732, "sampling/importance_sampling_ratio/max": 1.8514070510864258, "sampling/importance_sampling_ratio/mean": 1.0085704326629639, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3892521858215332, "sampling/sampling_logp_difference/mean": 0.1763553023338318, "step": 47, "step_time": 11.203057540027658 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 184.28125, "completions/mean_terminated_length": 184.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32736432272940874, "epoch": 0.00096, "frac_reward_zero_std": 0.0, "grad_norm": 0.8721429705619812, "kl": 0.720185068435967, "learning_rate": 9.996439077270269e-06, "loss": -0.0703, "num_tokens": 2063096.0, "reward": 0.375, "reward_std": 1.1617975234985352, "rewards/rollout_reward_func/mean": 0.375, "rewards/rollout_reward_func/std": 1.1288018226623535, "sampling/importance_sampling_ratio/max": 2.7482876777648926, "sampling/importance_sampling_ratio/mean": 1.0630409717559814, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3428795337677002, "sampling/sampling_logp_difference/mean": 0.17995157837867737, "step": 48, "step_time": 12.27966232097242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 222.0625, "completions/mean_terminated_length": 222.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29998447466641665, "epoch": 0.00098, "frac_reward_zero_std": 0.0, "grad_norm": 0.4916776716709137, "kl": 1.4262684024870396, "learning_rate": 9.995820976366208e-06, "loss": -0.0814, "num_tokens": 2111285.0, "reward": 0.53125, "reward_std": 1.3476016521453857, "rewards/rollout_reward_func/mean": 0.53125, "rewards/rollout_reward_func/std": 1.343667984008789, "sampling/importance_sampling_ratio/max": 2.3584651947021484, "sampling/importance_sampling_ratio/mean": 1.0179400444030762, "sampling/importance_sampling_ratio/min": 0.07456348091363907, "sampling/sampling_logp_difference/max": 2.595933437347412, "sampling/sampling_logp_difference/mean": 0.18933385610580444, "step": 49, "step_time": 11.537118092019227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 271.96875, "completions/mean_terminated_length": 271.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25075762066990137, "epoch": 0.001, "frac_reward_zero_std": 0.0, "grad_norm": 0.5755541324615479, "kl": 1.8406738163903356, "learning_rate": 9.995153465551283e-06, "loss": -0.0778, "num_tokens": 2160232.0, "reward": 0.5, "reward_std": 1.4450817108154297, "rewards/rollout_reward_func/mean": 0.5, "rewards/rollout_reward_func/std": 1.3912166357040405, "sampling/importance_sampling_ratio/max": 1.6897252798080444, "sampling/importance_sampling_ratio/mean": 0.8860001564025879, "sampling/importance_sampling_ratio/min": 0.07953189313411713, "sampling/sampling_logp_difference/max": 2.5422987937927246, "sampling/sampling_logp_difference/mean": 0.16909587383270264, "step": 50, "step_time": 11.45669094400364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 213.6875, "completions/mean_terminated_length": 213.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23833983996883035, "epoch": 0.00102, "frac_reward_zero_std": 0.0, "grad_norm": 0.406845360994339, "kl": 1.3570541720837355, "learning_rate": 9.994436553630411e-06, "loss": -0.0635, "num_tokens": 2206583.0, "reward": 0.53125, "reward_std": 1.1574089527130127, "rewards/rollout_reward_func/mean": 0.53125, "rewards/rollout_reward_func/std": 1.1354798078536987, "sampling/importance_sampling_ratio/max": 1.5882447957992554, "sampling/importance_sampling_ratio/mean": 0.8459539413452148, "sampling/importance_sampling_ratio/min": 0.04926040396094322, "sampling/sampling_logp_difference/max": 2.40498423576355, "sampling/sampling_logp_difference/mean": 0.1845960021018982, "step": 51, "step_time": 12.20510281805764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 180.34375, "completions/mean_terminated_length": 180.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2550461054779589, "epoch": 0.00104, "frac_reward_zero_std": 0.0, "grad_norm": 0.3576003611087799, "kl": 0.7773090861737728, "learning_rate": 9.993670250060152e-06, "loss": -0.0472, "num_tokens": 2251957.0, "reward": 0.5, "reward_std": 1.0944453477859497, "rewards/rollout_reward_func/mean": 0.5, "rewards/rollout_reward_func/std": 1.0776318311691284, "sampling/importance_sampling_ratio/max": 1.9495834112167358, "sampling/importance_sampling_ratio/mean": 0.9972184896469116, "sampling/importance_sampling_ratio/min": 0.16274717450141907, "sampling/sampling_logp_difference/max": 1.5296885967254639, "sampling/sampling_logp_difference/mean": 0.1387980878353119, "step": 52, "step_time": 11.274986464981339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 265.90625, "completions/mean_terminated_length": 265.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22390712006017566, "epoch": 0.00106, "frac_reward_zero_std": 0.0, "grad_norm": 0.47191229462623596, "kl": 0.9176817573606968, "learning_rate": 9.99285456494856e-06, "loss": -0.028, "num_tokens": 2301362.0, "reward": 0.90625, "reward_std": 1.007149338722229, "rewards/rollout_reward_func/mean": 0.90625, "rewards/rollout_reward_func/std": 1.0273478031158447, "sampling/importance_sampling_ratio/max": 1.6747417449951172, "sampling/importance_sampling_ratio/mean": 1.049408197402954, "sampling/importance_sampling_ratio/min": 0.3464587926864624, "sampling/sampling_logp_difference/max": 1.0784142017364502, "sampling/sampling_logp_difference/mean": 0.0823807418346405, "step": 53, "step_time": 11.359327576006763 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 169.875, "completions/mean_terminated_length": 169.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.311746085062623, "epoch": 0.00108, "frac_reward_zero_std": 0.0, "grad_norm": 1.3613451719284058, "kl": 1.2133773304522038, "learning_rate": 9.991989509055075e-06, "loss": -0.037, "num_tokens": 2344409.0, "reward": 0.75, "reward_std": 1.0345653295516968, "rewards/rollout_reward_func/mean": 0.75, "rewards/rollout_reward_func/std": 1.0776318311691284, "sampling/importance_sampling_ratio/max": 2.255628824234009, "sampling/importance_sampling_ratio/mean": 0.9827680587768555, "sampling/importance_sampling_ratio/min": 0.18077298998832703, "sampling/sampling_logp_difference/max": 1.7105125188827515, "sampling/sampling_logp_difference/mean": 0.13865616917610168, "step": 54, "step_time": 12.046740047022467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 244.125, "completions/mean_terminated_length": 244.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23533852584660053, "epoch": 0.0011, "frac_reward_zero_std": 0.0, "grad_norm": 1.2687926292419434, "kl": 0.8530482836067677, "learning_rate": 9.991075093790372e-06, "loss": -0.0493, "num_tokens": 2392784.0, "reward": 1.1875, "reward_std": 1.0595791339874268, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.0606601238250732, "sampling/importance_sampling_ratio/max": 1.501755714416504, "sampling/importance_sampling_ratio/mean": 1.0562961101531982, "sampling/importance_sampling_ratio/min": 0.37685710191726685, "sampling/sampling_logp_difference/max": 1.3653464317321777, "sampling/sampling_logp_difference/mean": 0.05706201121211052, "step": 55, "step_time": 11.352730427970528 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 264.15625, "completions/mean_terminated_length": 264.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2505794698372483, "epoch": 0.00112, "frac_reward_zero_std": 0.0, "grad_norm": 1.0631753206253052, "kl": 2.563420470803976, "learning_rate": 9.990111331216204e-06, "loss": -0.0485, "num_tokens": 2441421.0, "reward": 1.0, "reward_std": 1.0518605709075928, "rewards/rollout_reward_func/mean": 1.0, "rewards/rollout_reward_func/std": 1.0776318311691284, "sampling/importance_sampling_ratio/max": 1.8085652589797974, "sampling/importance_sampling_ratio/mean": 0.9267995357513428, "sampling/importance_sampling_ratio/min": 0.17390626668930054, "sampling/sampling_logp_difference/max": 1.7491989135742188, "sampling/sampling_logp_difference/mean": 0.10832004249095917, "step": 56, "step_time": 11.912780995029607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 304.5625, "completions/mean_terminated_length": 304.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2508107628673315, "epoch": 0.00114, "frac_reward_zero_std": 0.0, "grad_norm": 0.6311008930206299, "kl": 1.5253621228039265, "learning_rate": 9.989098234045254e-06, "loss": -0.0417, "num_tokens": 2491816.0, "reward": 0.6875, "reward_std": 1.1992462873458862, "rewards/rollout_reward_func/mean": 0.6875, "rewards/rollout_reward_func/std": 1.1760376691818237, "sampling/importance_sampling_ratio/max": 1.9641000032424927, "sampling/importance_sampling_ratio/mean": 0.9425516724586487, "sampling/importance_sampling_ratio/min": 0.20561183989048004, "sampling/sampling_logp_difference/max": 1.8712592124938965, "sampling/sampling_logp_difference/mean": 0.09314735978841782, "step": 57, "step_time": 11.972163331040065 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 243.84375, "completions/mean_terminated_length": 243.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22545340517535806, "epoch": 0.00116, "frac_reward_zero_std": 0.0, "grad_norm": 0.6751827001571655, "kl": 1.153871901333332, "learning_rate": 9.988035815640953e-06, "loss": -0.0538, "num_tokens": 2539974.0, "reward": 0.875, "reward_std": 1.0994656085968018, "rewards/rollout_reward_func/mean": 0.875, "rewards/rollout_reward_func/std": 1.0701221227645874, "sampling/importance_sampling_ratio/max": 2.1724822521209717, "sampling/importance_sampling_ratio/mean": 0.8622069358825684, "sampling/importance_sampling_ratio/min": 0.23442766070365906, "sampling/sampling_logp_difference/max": 1.4498639106750488, "sampling/sampling_logp_difference/mean": 0.11690501868724823, "step": 58, "step_time": 11.342359606016544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.046875, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 331.65625, "completions/mean_terminated_length": 331.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2523074056953192, "epoch": 0.00118, "frac_reward_zero_std": 0.0, "grad_norm": 1.2779186964035034, "kl": 4.496506713330746, "learning_rate": 9.986924090017322e-06, "loss": -0.0449, "num_tokens": 2592623.0, "reward": 1.03125, "reward_std": 1.1243396997451782, "rewards/rollout_reward_func/mean": 1.03125, "rewards/rollout_reward_func/std": 1.1495966911315918, "sampling/importance_sampling_ratio/max": 2.012497663497925, "sampling/importance_sampling_ratio/mean": 0.994400143623352, "sampling/importance_sampling_ratio/min": 0.04033363237977028, "sampling/sampling_logp_difference/max": 3.3006155490875244, "sampling/sampling_logp_difference/mean": 0.12554790079593658, "step": 59, "step_time": 13.151727148026112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 231.90625, "completions/mean_terminated_length": 231.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31803171487990767, "epoch": 0.0012, "frac_reward_zero_std": 0.0, "grad_norm": 2.2517879009246826, "kl": 1.3760347040370107, "learning_rate": 9.985763071838774e-06, "loss": -0.0352, "num_tokens": 2639579.0, "reward": 0.90625, "reward_std": 1.2430509328842163, "rewards/rollout_reward_func/mean": 0.90625, "rewards/rollout_reward_func/std": 1.2276222705841064, "sampling/importance_sampling_ratio/max": 2.871330499649048, "sampling/importance_sampling_ratio/mean": 0.937651515007019, "sampling/importance_sampling_ratio/min": 0.11985822767019272, "sampling/sampling_logp_difference/max": 2.1210336685180664, "sampling/sampling_logp_difference/mean": 0.1352756768465042, "step": 60, "step_time": 11.771704801969463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 264.65625, "completions/mean_terminated_length": 264.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25549063133075833, "epoch": 0.00122, "frac_reward_zero_std": 0.0, "grad_norm": 1.1953744888305664, "kl": 1.0506798774003983, "learning_rate": 9.98455277641992e-06, "loss": -0.0309, "num_tokens": 2688224.0, "reward": 1.0, "reward_std": 1.038672924041748, "rewards/rollout_reward_func/mean": 1.0, "rewards/rollout_reward_func/std": 1.0160009860992432, "sampling/importance_sampling_ratio/max": 2.8034634590148926, "sampling/importance_sampling_ratio/mean": 0.9786415100097656, "sampling/importance_sampling_ratio/min": 0.023068277165293694, "sampling/sampling_logp_difference/max": 3.769007444381714, "sampling/sampling_logp_difference/mean": 0.1225055605173111, "step": 61, "step_time": 11.416885740021826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 220.6875, "completions/mean_terminated_length": 220.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.34351445361971855, "epoch": 0.00124, "frac_reward_zero_std": 0.0, "grad_norm": 1.6104074716567993, "kl": 0.6596160810440779, "learning_rate": 9.983293219725379e-06, "loss": -0.0428, "num_tokens": 2735072.0, "reward": 0.46875, "reward_std": 1.1810407638549805, "rewards/rollout_reward_func/mean": 0.46875, "rewards/rollout_reward_func/std": 1.1909435987472534, "sampling/importance_sampling_ratio/max": 2.48545503616333, "sampling/importance_sampling_ratio/mean": 0.9570181369781494, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6737163066864014, "sampling/sampling_logp_difference/mean": 0.09554591774940491, "step": 62, "step_time": 12.105101279026712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 263.46875, "completions/mean_terminated_length": 263.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21513652347493917, "epoch": 0.00126, "frac_reward_zero_std": 0.0, "grad_norm": 3.1774353981018066, "kl": 8.563187420368195, "learning_rate": 9.981984418369552e-06, "loss": -0.0301, "num_tokens": 2784386.0, "reward": 1.0625, "reward_std": 1.1046645641326904, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.1341474056243896, "sampling/importance_sampling_ratio/max": 2.881258010864258, "sampling/importance_sampling_ratio/mean": 0.9623677730560303, "sampling/importance_sampling_ratio/min": 0.07400012761354446, "sampling/sampling_logp_difference/max": 2.6035776138305664, "sampling/sampling_logp_difference/mean": 0.09384943544864655, "step": 63, "step_time": 11.567457363024005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 223.34375, "completions/mean_terminated_length": 223.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23865353292785585, "epoch": 0.00128, "frac_reward_zero_std": 0.0, "grad_norm": 0.8613842725753784, "kl": 1.0374977588653564, "learning_rate": 9.980626389616414e-06, "loss": -0.0499, "num_tokens": 2831267.0, "reward": 0.75, "reward_std": 1.219443440437317, "rewards/rollout_reward_func/mean": 0.75, "rewards/rollout_reward_func/std": 1.1913667917251587, "sampling/importance_sampling_ratio/max": 1.630966305732727, "sampling/importance_sampling_ratio/mean": 0.9062402844429016, "sampling/importance_sampling_ratio/min": 0.11962966620922089, "sampling/sampling_logp_difference/max": 2.123232841491699, "sampling/sampling_logp_difference/mean": 0.11629284918308258, "step": 64, "step_time": 11.290368011963437 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.03125, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 326.53125, "completions/mean_terminated_length": 318.8064270019531, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20423890324309468, "epoch": 0.0013, "frac_reward_zero_std": 0.0, "grad_norm": 2.1429941654205322, "kl": 0.997785298153758, "learning_rate": 9.979219151379285e-06, "loss": -0.0548, "num_tokens": 2883116.0, "reward": 1.3125, "reward_std": 1.0883184671401978, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.0906493663787842, "sampling/importance_sampling_ratio/max": 1.6378145217895508, "sampling/importance_sampling_ratio/mean": 0.9880042672157288, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6758522987365723, "sampling/sampling_logp_difference/mean": 0.0766817182302475, "step": 65, "step_time": 12.949794946951442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 282.9375, "completions/mean_terminated_length": 282.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1988681850489229, "epoch": 0.00132, "frac_reward_zero_std": 0.0, "grad_norm": 0.7908368706703186, "kl": 1.7130778981372714, "learning_rate": 9.977762722220585e-06, "loss": -0.0394, "num_tokens": 2932737.0, "reward": 0.8125, "reward_std": 0.983958899974823, "rewards/rollout_reward_func/mean": 0.8125, "rewards/rollout_reward_func/std": 1.1198358535766602, "sampling/importance_sampling_ratio/max": 1.7433757781982422, "sampling/importance_sampling_ratio/mean": 0.9519436955451965, "sampling/importance_sampling_ratio/min": 0.2962948977947235, "sampling/sampling_logp_difference/max": 1.2161684036254883, "sampling/sampling_logp_difference/mean": 0.0792214572429657, "step": 66, "step_time": 11.412843257014174 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 175.71875, "completions/mean_terminated_length": 175.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13997940928675234, "epoch": 0.00134, "frac_reward_zero_std": 0.0, "grad_norm": 1.6619311571121216, "kl": 5.998074978590012, "learning_rate": 9.976257121351602e-06, "loss": -0.0592, "num_tokens": 2977233.0, "reward": 1.09375, "reward_std": 0.9554383754730225, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 0.9954533576965332, "sampling/importance_sampling_ratio/max": 1.7542433738708496, "sampling/importance_sampling_ratio/mean": 0.9644668102264404, "sampling/importance_sampling_ratio/min": 0.029159268364310265, "sampling/sampling_logp_difference/max": 3.534813165664673, "sampling/sampling_logp_difference/mean": 0.12494733184576035, "step": 67, "step_time": 11.630185200978303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 195.4375, "completions/mean_terminated_length": 195.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16040176060050726, "epoch": 0.00136, "frac_reward_zero_std": 0.0, "grad_norm": 0.626408040523529, "kl": 0.9022160228341818, "learning_rate": 9.974702368632228e-06, "loss": -0.0488, "num_tokens": 3022643.0, "reward": 0.96875, "reward_std": 0.9308042526245117, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 0.9666828513145447, "sampling/importance_sampling_ratio/max": 1.7982441186904907, "sampling/importance_sampling_ratio/mean": 0.9963648915290833, "sampling/importance_sampling_ratio/min": 0.3037925958633423, "sampling/sampling_logp_difference/max": 1.1886656284332275, "sampling/sampling_logp_difference/mean": 0.06637202948331833, "step": 68, "step_time": 11.583390460975352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 264.65625, "completions/mean_terminated_length": 264.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16527789598330855, "epoch": 0.00138, "frac_reward_zero_std": 0.0, "grad_norm": 0.5700135827064514, "kl": 2.4761747140437365, "learning_rate": 9.973098484570702e-06, "loss": -0.0325, "num_tokens": 3071616.0, "reward": 1.0625, "reward_std": 1.0073541402816772, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.4364933967590332, "sampling/importance_sampling_ratio/mean": 0.8604850172996521, "sampling/importance_sampling_ratio/min": 0.058823082596063614, "sampling/sampling_logp_difference/max": 2.833117961883545, "sampling/sampling_logp_difference/mean": 0.11738038063049316, "step": 69, "step_time": 11.402018775013858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 321.84375, "completions/mean_terminated_length": 321.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13781232619658113, "epoch": 0.0014, "frac_reward_zero_std": 0.0, "grad_norm": 0.7338686585426331, "kl": 1.292791312560439, "learning_rate": 9.97144549032334e-06, "loss": -0.0484, "num_tokens": 3122680.0, "reward": 1.09375, "reward_std": 1.106506109237671, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.1738927364349365, "sampling/importance_sampling_ratio/max": 2.1809706687927246, "sampling/importance_sampling_ratio/mean": 1.004666805267334, "sampling/importance_sampling_ratio/min": 0.04265930876135826, "sampling/sampling_logp_difference/max": 1.8892066478729248, "sampling/sampling_logp_difference/mean": 0.09892367571592331, "step": 70, "step_time": 12.01346462201036 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 225.75, "completions/mean_terminated_length": 225.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12212204607203603, "epoch": 0.00142, "frac_reward_zero_std": 0.0, "grad_norm": 1.0600917339324951, "kl": 4.885354918427765, "learning_rate": 9.969743407694254e-06, "loss": -0.0259, "num_tokens": 3169282.0, "reward": 1.28125, "reward_std": 0.7097088098526001, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.7718588709831238, "sampling/importance_sampling_ratio/max": 1.2103098630905151, "sampling/importance_sampling_ratio/mean": 0.9580530524253845, "sampling/importance_sampling_ratio/min": 0.026294805109500885, "sampling/sampling_logp_difference/max": 3.6381583213806152, "sampling/sampling_logp_difference/mean": 0.0645737498998642, "step": 71, "step_time": 11.551435373010463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 321.71875, "completions/mean_terminated_length": 321.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1584914915729314, "epoch": 0.00144, "frac_reward_zero_std": 0.0, "grad_norm": 1.0732065439224243, "kl": 0.8940995708107948, "learning_rate": 9.967992259135063e-06, "loss": -0.0394, "num_tokens": 3220383.0, "reward": 1.1875, "reward_std": 0.9210742115974426, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 0.8957785964012146, "sampling/importance_sampling_ratio/max": 1.6308703422546387, "sampling/importance_sampling_ratio/mean": 0.9564945697784424, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.502565383911133, "sampling/sampling_logp_difference/mean": 0.08294906467199326, "step": 72, "step_time": 11.620703086940921 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 261.4375, "completions/mean_terminated_length": 261.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09936718340031803, "epoch": 0.00146, "frac_reward_zero_std": 0.0, "grad_norm": 4.2813825607299805, "kl": 21.793556328862906, "learning_rate": 9.9661920677446e-06, "loss": -0.0087, "num_tokens": 3268528.0, "reward": 0.96875, "reward_std": 1.084357500076294, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 1.121185302734375, "sampling/importance_sampling_ratio/max": 1.7068644762039185, "sampling/importance_sampling_ratio/mean": 0.8584051728248596, "sampling/importance_sampling_ratio/min": 0.035112399607896805, "sampling/sampling_logp_difference/max": 3.3491058349609375, "sampling/sampling_logp_difference/mean": 0.13202719390392303, "step": 73, "step_time": 12.829841696977383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 299.34375, "completions/mean_terminated_length": 299.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13296697288751602, "epoch": 0.00148, "frac_reward_zero_std": 0.0, "grad_norm": 0.9020382761955261, "kl": 1.5102986749261618, "learning_rate": 9.964342857268609e-06, "loss": -0.0272, "num_tokens": 3318966.0, "reward": 1.34375, "reward_std": 0.8184924721717834, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.8654431700706482, "sampling/importance_sampling_ratio/max": 1.9613794088363647, "sampling/importance_sampling_ratio/mean": 0.9886914491653442, "sampling/importance_sampling_ratio/min": 0.04327766224741936, "sampling/sampling_logp_difference/max": 3.3183345794677734, "sampling/sampling_logp_difference/mean": 0.08532620966434479, "step": 74, "step_time": 11.581191210978432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 287.53125, "completions/mean_terminated_length": 287.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15034426166675985, "epoch": 0.0015, "frac_reward_zero_std": 0.0, "grad_norm": 0.8684410452842712, "kl": 1.7290330082178116, "learning_rate": 9.962444652099422e-06, "loss": -0.058, "num_tokens": 3369280.0, "reward": 1.21875, "reward_std": 1.0798513889312744, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.0993949174880981, "sampling/importance_sampling_ratio/max": 1.6313560009002686, "sampling/importance_sampling_ratio/mean": 0.9344650506973267, "sampling/importance_sampling_ratio/min": 0.05653540790081024, "sampling/sampling_logp_difference/max": 2.8727593421936035, "sampling/sampling_logp_difference/mean": 0.0933370590209961, "step": 75, "step_time": 11.564885397063335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 272.21875, "completions/mean_terminated_length": 272.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13900031056255102, "epoch": 0.00152, "frac_reward_zero_std": 0.0, "grad_norm": 0.9704777598381042, "kl": 1.3484667632728815, "learning_rate": 9.960497477275653e-06, "loss": -0.0362, "num_tokens": 3418756.0, "reward": 1.1875, "reward_std": 0.900712251663208, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 0.8957785964012146, "sampling/importance_sampling_ratio/max": 1.780420184135437, "sampling/importance_sampling_ratio/mean": 0.984249472618103, "sampling/importance_sampling_ratio/min": 0.11867707967758179, "sampling/sampling_logp_difference/max": 2.139289379119873, "sampling/sampling_logp_difference/mean": 0.06108468770980835, "step": 76, "step_time": 12.830781276948983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 231.90625, "completions/mean_terminated_length": 231.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18062162608839571, "epoch": 0.00154, "frac_reward_zero_std": 0.0, "grad_norm": 1.1658787727355957, "kl": 3.2830700278282166, "learning_rate": 9.95850135848185e-06, "loss": -0.0468, "num_tokens": 3466469.0, "reward": 0.84375, "reward_std": 1.1458055973052979, "rewards/rollout_reward_func/mean": 0.84375, "rewards/rollout_reward_func/std": 1.1103435754776, "sampling/importance_sampling_ratio/max": 1.5454765558242798, "sampling/importance_sampling_ratio/mean": 0.8513920903205872, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.260195255279541, "sampling/sampling_logp_difference/mean": 0.1379757523536682, "step": 77, "step_time": 11.85138143999211 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 258.0, "completions/mean_terminated_length": 258.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12013657600618899, "epoch": 0.00156, "frac_reward_zero_std": 0.25, "grad_norm": 0.41953355073928833, "kl": 2.168890221044421, "learning_rate": 9.956456322048174e-06, "loss": -0.0364, "num_tokens": 3515524.0, "reward": 1.34375, "reward_std": 0.7218577265739441, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.0658745765686035, "sampling/importance_sampling_ratio/max": 1.0944042205810547, "sampling/importance_sampling_ratio/mean": 0.9101173877716064, "sampling/importance_sampling_ratio/min": 0.10899689048528671, "sampling/sampling_logp_difference/max": 2.216186761856079, "sampling/sampling_logp_difference/mean": 0.06783867627382278, "step": 78, "step_time": 11.5368067399977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 329.125, "completions/mean_terminated_length": 329.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20095332176424563, "epoch": 0.00158, "frac_reward_zero_std": 0.0, "grad_norm": 1.1228458881378174, "kl": 1.4139137901365757, "learning_rate": 9.954362394950035e-06, "loss": -0.0475, "num_tokens": 3567208.0, "reward": 1.09375, "reward_std": 1.1470638513565063, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.201058030128479, "sampling/importance_sampling_ratio/max": 1.2632485628128052, "sampling/importance_sampling_ratio/mean": 0.8483125567436218, "sampling/importance_sampling_ratio/min": 0.09566221386194229, "sampling/sampling_logp_difference/max": 2.3466005325317383, "sampling/sampling_logp_difference/mean": 0.1154029592871666, "step": 79, "step_time": 12.859868795989314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 217.78125, "completions/mean_terminated_length": 217.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2741812968160957, "epoch": 0.0016, "frac_reward_zero_std": 0.0, "grad_norm": 0.675167977809906, "kl": 5.697039952501655, "learning_rate": 9.952219604807746e-06, "loss": -0.0883, "num_tokens": 3613604.0, "reward": 1.09375, "reward_std": 1.1347887516021729, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.1460838317871094, "sampling/importance_sampling_ratio/max": 1.6844247579574585, "sampling/importance_sampling_ratio/mean": 0.7855990529060364, "sampling/importance_sampling_ratio/min": 0.11579425632953644, "sampling/sampling_logp_difference/max": 2.1559529304504395, "sampling/sampling_logp_difference/mean": 0.18667854368686676, "step": 80, "step_time": 11.460505888971966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 196.21875, "completions/mean_terminated_length": 196.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28592561231926084, "epoch": 0.00162, "frac_reward_zero_std": 0.0, "grad_norm": 1.1092965602874756, "kl": 1.1189161781221628, "learning_rate": 9.950027979886159e-06, "loss": -0.0618, "num_tokens": 3659337.0, "reward": 0.90625, "reward_std": 1.1558747291564941, "rewards/rollout_reward_func/mean": 0.90625, "rewards/rollout_reward_func/std": 1.1460838317871094, "sampling/importance_sampling_ratio/max": 1.5059292316436768, "sampling/importance_sampling_ratio/mean": 0.8182501792907715, "sampling/importance_sampling_ratio/min": 0.13447163999080658, "sampling/sampling_logp_difference/max": 2.006371021270752, "sampling/sampling_logp_difference/mean": 0.13647563755512238, "step": 81, "step_time": 11.31048478599405 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 237.96875, "completions/mean_terminated_length": 237.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29075616877526045, "epoch": 0.00164, "frac_reward_zero_std": 0.0, "grad_norm": 0.436435729265213, "kl": 1.3324220515787601, "learning_rate": 9.947787549094288e-06, "loss": -0.0428, "num_tokens": 3705330.0, "reward": 1.0, "reward_std": 1.230472207069397, "rewards/rollout_reward_func/mean": 1.0, "rewards/rollout_reward_func/std": 1.2700011730194092, "sampling/importance_sampling_ratio/max": 1.9398707151412964, "sampling/importance_sampling_ratio/mean": 0.8319602012634277, "sampling/importance_sampling_ratio/min": 0.2161659598350525, "sampling/sampling_logp_difference/max": 1.5315495729446411, "sampling/sampling_logp_difference/mean": 0.16198177635669708, "step": 82, "step_time": 12.85476766100328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 235.71875, "completions/mean_terminated_length": 235.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32235735561698675, "epoch": 0.00166, "frac_reward_zero_std": 0.0, "grad_norm": 0.4511769413948059, "kl": 3.0084350556135178, "learning_rate": 9.945498341984929e-06, "loss": -0.0469, "num_tokens": 3751957.0, "reward": 1.0, "reward_std": 1.058655023574829, "rewards/rollout_reward_func/mean": 1.0, "rewards/rollout_reward_func/std": 1.0472698211669922, "sampling/importance_sampling_ratio/max": 1.8046413660049438, "sampling/importance_sampling_ratio/mean": 0.8079686164855957, "sampling/importance_sampling_ratio/min": 0.11039608716964722, "sampling/sampling_logp_difference/max": 2.203646183013916, "sampling/sampling_logp_difference/mean": 0.17553269863128662, "step": 83, "step_time": 11.357482589010033 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 279.4375, "completions/mean_terminated_length": 279.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2988183037377894, "epoch": 0.00168, "frac_reward_zero_std": 0.0, "grad_norm": 1.7082585096359253, "kl": 3.2760359421372414, "learning_rate": 9.943160388754274e-06, "loss": -0.076, "num_tokens": 3801777.0, "reward": 1.25, "reward_std": 0.961219072341919, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.0776318311691284, "sampling/importance_sampling_ratio/max": 2.30365252494812, "sampling/importance_sampling_ratio/mean": 0.8170902729034424, "sampling/importance_sampling_ratio/min": 0.026935581117868423, "sampling/sampling_logp_difference/max": 3.6100401878356934, "sampling/sampling_logp_difference/mean": 0.184578537940979, "step": 84, "step_time": 11.438660500047263 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.017578125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.033203125, "completions/clipped_ratio": 0.03125, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 227.28125, "completions/mean_terminated_length": 234.09677124023438, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22724101739004254, "epoch": 0.0017, "frac_reward_zero_std": 0.0, "grad_norm": 0.34271278977394104, "kl": 1.105888869613409, "learning_rate": 9.940773720241503e-06, "loss": -0.0715, "num_tokens": 3850032.0, "reward": 1.34375, "reward_std": 0.948811411857605, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 2.096190929412842, "sampling/importance_sampling_ratio/mean": 0.8424466848373413, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3155715465545654, "sampling/sampling_logp_difference/mean": 0.15178421139717102, "step": 85, "step_time": 12.978846625017468 }, { "clip_ratio/high_max": 0.026041666977107525, "clip_ratio/high_mean": 0.013020833488553762, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03645833348855376, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 255.8125, "completions/mean_terminated_length": 255.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29682247154414654, "epoch": 0.00172, "frac_reward_zero_std": 0.0, "grad_norm": 0.6635299921035767, "kl": 2.7029092833399773, "learning_rate": 9.938338367928391e-06, "loss": -0.0498, "num_tokens": 3897938.0, "reward": 1.03125, "reward_std": 1.0703709125518799, "rewards/rollout_reward_func/mean": 1.03125, "rewards/rollout_reward_func/std": 1.121185302734375, "sampling/importance_sampling_ratio/max": 2.9373552799224854, "sampling/importance_sampling_ratio/mean": 0.8815596103668213, "sampling/importance_sampling_ratio/min": 5.593795204816843e-09, "sampling/sampling_logp_difference/max": 18.431718826293945, "sampling/sampling_logp_difference/mean": 0.3719289302825928, "step": 86, "step_time": 11.750265939015662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.03125, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 313.28125, "completions/mean_terminated_length": 322.8709716796875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2681382931768894, "epoch": 0.00174, "frac_reward_zero_std": 0.0, "grad_norm": 5.396854400634766, "kl": 7.390463003888726, "learning_rate": 9.935854363938876e-06, "loss": -0.0181, "num_tokens": 3948394.0, "reward": 0.84375, "reward_std": 1.3448522090911865, "rewards/rollout_reward_func/mean": 0.84375, "rewards/rollout_reward_func/std": 1.3224945068359375, "sampling/importance_sampling_ratio/max": 2.423076868057251, "sampling/importance_sampling_ratio/mean": 0.8453531265258789, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.676208257675171, "sampling/sampling_logp_difference/mean": 0.19792482256889343, "step": 87, "step_time": 11.981117413015454 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0546875, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 284.875, "completions/mean_terminated_length": 284.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22405766509473324, "epoch": 0.00176, "frac_reward_zero_std": 0.0, "grad_norm": 4.24516487121582, "kl": 9.487529492005706, "learning_rate": 9.933321741038655e-06, "loss": -0.0394, "num_tokens": 3998680.0, "reward": 0.9375, "reward_std": 1.0765184164047241, "rewards/rollout_reward_func/mean": 0.9375, "rewards/rollout_reward_func/std": 1.1053389310836792, "sampling/importance_sampling_ratio/max": 1.7575771808624268, "sampling/importance_sampling_ratio/mean": 0.6912931203842163, "sampling/importance_sampling_ratio/min": 0.028209341689944267, "sampling/sampling_logp_difference/max": 2.7894208431243896, "sampling/sampling_logp_difference/mean": 0.24522249400615692, "step": 88, "step_time": 11.960086607999983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 254.125, "completions/mean_terminated_length": 254.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27438862272538245, "epoch": 0.00178, "frac_reward_zero_std": 0.0, "grad_norm": 0.6965829133987427, "kl": 5.652095748111606, "learning_rate": 9.930740532634733e-06, "loss": -0.0739, "num_tokens": 4046635.0, "reward": 0.78125, "reward_std": 1.4365880489349365, "rewards/rollout_reward_func/mean": 0.78125, "rewards/rollout_reward_func/std": 1.4081416130065918, "sampling/importance_sampling_ratio/max": 1.8677366971969604, "sampling/importance_sampling_ratio/mean": 0.8026033639907837, "sampling/importance_sampling_ratio/min": 0.06247870624065399, "sampling/sampling_logp_difference/max": 2.772242784500122, "sampling/sampling_logp_difference/mean": 0.2531365752220154, "step": 89, "step_time": 13.599878925990197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 267.3125, "completions/mean_terminated_length": 267.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28442200273275375, "epoch": 0.0018, "frac_reward_zero_std": 0.0, "grad_norm": 0.655674159526825, "kl": 2.8140620961785316, "learning_rate": 9.928110772774995e-06, "loss": -0.0837, "num_tokens": 4094748.0, "reward": 0.78125, "reward_std": 1.138383388519287, "rewards/rollout_reward_func/mean": 0.78125, "rewards/rollout_reward_func/std": 1.2374368906021118, "sampling/importance_sampling_ratio/max": 1.7086113691329956, "sampling/importance_sampling_ratio/mean": 0.7258652448654175, "sampling/importance_sampling_ratio/min": 0.06997895985841751, "sampling/sampling_logp_difference/max": 2.659541606903076, "sampling/sampling_logp_difference/mean": 0.2072906494140625, "step": 90, "step_time": 14.866481214950909 }, { "clip_ratio/high_max": 0.0625, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.03125, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 258.0625, "completions/mean_terminated_length": 265.8709716796875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3242306923493743, "epoch": 0.00182, "frac_reward_zero_std": 0.0, "grad_norm": 0.3057500422000885, "kl": 2.1801970321685076, "learning_rate": 9.92543249614775e-06, "loss": -0.0595, "num_tokens": 4142992.0, "reward": 0.75, "reward_std": 1.2317585945129395, "rewards/rollout_reward_func/mean": 0.75, "rewards/rollout_reward_func/std": 1.2181423902511597, "sampling/importance_sampling_ratio/max": 2.195753812789917, "sampling/importance_sampling_ratio/mean": 0.7545443773269653, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3468799591064453, "sampling/sampling_logp_difference/mean": 0.19587546586990356, "step": 91, "step_time": 13.839472674997523 }, { "clip_ratio/high_max": 0.0625, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 282.1875, "completions/mean_terminated_length": 282.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24518498685210943, "epoch": 0.00184, "frac_reward_zero_std": 0.0, "grad_norm": 0.5407370328903198, "kl": 1.0705399587750435, "learning_rate": 9.922705738081279e-06, "loss": -0.0395, "num_tokens": 4193044.0, "reward": 1.0625, "reward_std": 1.1043577194213867, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.0757592916488647, "sampling/importance_sampling_ratio/max": 1.7658262252807617, "sampling/importance_sampling_ratio/mean": 0.7871720194816589, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.257204532623291, "sampling/sampling_logp_difference/mean": 0.1855379194021225, "step": 92, "step_time": 13.615127239041612 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017361111124046147, "completions/clipped_ratio": 0.03125, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 251.90625, "completions/mean_terminated_length": 250.6774139404297, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.46685940120369196, "epoch": 0.00186, "frac_reward_zero_std": 0.0, "grad_norm": 1.273982286453247, "kl": 2.9461166486144066, "learning_rate": 9.919930534543365e-06, "loss": -0.0249, "num_tokens": 4240619.0, "reward": 0.6875, "reward_std": 1.2202157974243164, "rewards/rollout_reward_func/mean": 0.6875, "rewards/rollout_reward_func/std": 1.306003451347351, "sampling/importance_sampling_ratio/max": 1.5020077228546143, "sampling/importance_sampling_ratio/mean": 0.6535407304763794, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.987065315246582, "sampling/sampling_logp_difference/mean": 0.18315604329109192, "step": 93, "step_time": 14.329339338029968 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 268.84375, "completions/mean_terminated_length": 268.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31569950515404344, "epoch": 0.00188, "frac_reward_zero_std": 0.0, "grad_norm": 0.42849844694137573, "kl": 1.2293035797774792, "learning_rate": 9.917106922140814e-06, "loss": -0.0616, "num_tokens": 4288735.0, "reward": 1.0625, "reward_std": 1.1315523386001587, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.1896733045578003, "sampling/importance_sampling_ratio/max": 2.100494861602783, "sampling/importance_sampling_ratio/mean": 0.8006396889686584, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.069516897201538, "sampling/sampling_logp_difference/mean": 0.1671556532382965, "step": 94, "step_time": 13.849829273021896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 303.5, "completions/mean_terminated_length": 303.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3844900978729129, "epoch": 0.0019, "frac_reward_zero_std": 0.0, "grad_norm": 0.9655054211616516, "kl": 1.5059290062636137, "learning_rate": 9.914234938118988e-06, "loss": -0.0467, "num_tokens": 4338181.0, "reward": 0.9375, "reward_std": 1.0164296627044678, "rewards/rollout_reward_func/mean": 0.9375, "rewards/rollout_reward_func/std": 1.0757592916488647, "sampling/importance_sampling_ratio/max": 1.3846509456634521, "sampling/importance_sampling_ratio/mean": 0.6473355889320374, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7942121028900146, "sampling/sampling_logp_difference/mean": 0.23950277268886566, "step": 95, "step_time": 14.216661832018872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 307.34375, "completions/mean_terminated_length": 307.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2691213018260896, "epoch": 0.00192, "frac_reward_zero_std": 0.0, "grad_norm": 0.5178808569908142, "kl": 1.1331802532076836, "learning_rate": 9.911314620361294e-06, "loss": -0.0712, "num_tokens": 4387736.0, "reward": 0.875, "reward_std": 1.439156413078308, "rewards/rollout_reward_func/mean": 0.875, "rewards/rollout_reward_func/std": 1.38540780544281, "sampling/importance_sampling_ratio/max": 1.8776792287826538, "sampling/importance_sampling_ratio/mean": 0.8164584636688232, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.567051887512207, "sampling/sampling_logp_difference/mean": 0.1963246464729309, "step": 96, "step_time": 14.233796298954985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 296.96875, "completions/mean_terminated_length": 296.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3244233909063041, "epoch": 0.00194, "frac_reward_zero_std": 0.0, "grad_norm": 1.0547850131988525, "kl": 3.60640642978251, "learning_rate": 9.908346007388698e-06, "loss": -0.0681, "num_tokens": 4437183.0, "reward": 0.4375, "reward_std": 1.5999572277069092, "rewards/rollout_reward_func/mean": 0.4375, "rewards/rollout_reward_func/std": 1.5644745826721191, "sampling/importance_sampling_ratio/max": 2.3757717609405518, "sampling/importance_sampling_ratio/mean": 0.7622094750404358, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.601517915725708, "sampling/sampling_logp_difference/mean": 0.2694668173789978, "step": 97, "step_time": 13.617985813034466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 270.125, "completions/mean_terminated_length": 270.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27305965893901885, "epoch": 0.00196, "frac_reward_zero_std": 0.0, "grad_norm": 8.37128734588623, "kl": 22.282493229955435, "learning_rate": 9.905329138359213e-06, "loss": -0.0463, "num_tokens": 4486452.0, "reward": 0.96875, "reward_std": 1.1179354190826416, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 1.2044105529785156, "sampling/importance_sampling_ratio/max": 1.2155061960220337, "sampling/importance_sampling_ratio/mean": 0.6622193455696106, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.706277370452881, "sampling/sampling_logp_difference/mean": 0.2237791121006012, "step": 98, "step_time": 14.377591560012661 }, { "clip_ratio/high_max": 0.04513888922519982, "clip_ratio/high_mean": 0.02256944461259991, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02256944461259991, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 272.6875, "completions/mean_terminated_length": 272.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2743502545636147, "epoch": 0.00198, "frac_reward_zero_std": 0.0, "grad_norm": 0.6745516061782837, "kl": 1.2847364861518145, "learning_rate": 9.902264053067381e-06, "loss": -0.1087, "num_tokens": 4534584.0, "reward": 0.9375, "reward_std": 1.1867362260818481, "rewards/rollout_reward_func/mean": 0.9375, "rewards/rollout_reward_func/std": 1.1622419357299805, "sampling/importance_sampling_ratio/max": 2.4441423416137695, "sampling/importance_sampling_ratio/mean": 0.7655668258666992, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4372406005859375, "sampling/sampling_logp_difference/mean": 0.15401461720466614, "step": 99, "step_time": 13.872322146984516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0322265625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0322265625, "completions/clipped_ratio": 0.03125, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 273.6875, "completions/mean_terminated_length": 261.3548278808594, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2537284654099494, "epoch": 0.002, "frac_reward_zero_std": 0.0, "grad_norm": 0.5095395445823669, "kl": 1.5012515522539616, "learning_rate": 9.899150791943747e-06, "loss": -0.0202, "num_tokens": 4583279.0, "reward": 0.9375, "reward_std": 1.4227921962738037, "rewards/rollout_reward_func/mean": 0.9375, "rewards/rollout_reward_func/std": 1.3897666931152344, "sampling/importance_sampling_ratio/max": 1.1976944208145142, "sampling/importance_sampling_ratio/mean": 0.7104281187057495, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2792985439300537, "sampling/sampling_logp_difference/mean": 0.1571941077709198, "step": 100, "step_time": 13.547125598008279 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 298.5, "completions/mean_terminated_length": 298.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25784868374466896, "epoch": 0.00202, "frac_reward_zero_std": 0.0, "grad_norm": 0.7853564620018005, "kl": 2.195194707252085, "learning_rate": 9.895989396054334e-06, "loss": -0.0626, "num_tokens": 4633508.0, "reward": 0.71875, "reward_std": 1.2138206958770752, "rewards/rollout_reward_func/mean": 0.71875, "rewards/rollout_reward_func/std": 1.3009766340255737, "sampling/importance_sampling_ratio/max": 1.8202441930770874, "sampling/importance_sampling_ratio/mean": 0.8282261490821838, "sampling/importance_sampling_ratio/min": 0.10892469435930252, "sampling/sampling_logp_difference/max": 1.7346787452697754, "sampling/sampling_logp_difference/mean": 0.18933400511741638, "step": 101, "step_time": 14.01855112798512 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 404.3125, "completions/mean_terminated_length": 404.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2152117567602545, "epoch": 0.00204, "frac_reward_zero_std": 0.0, "grad_norm": 0.36234670877456665, "kl": 1.2044182308018208, "learning_rate": 9.892779907100084e-06, "loss": -0.0209, "num_tokens": 4687464.0, "reward": 1.125, "reward_std": 1.0851054191589355, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 1.1570261716842651, "sampling/importance_sampling_ratio/max": 2.868651866912842, "sampling/importance_sampling_ratio/mean": 0.8917727470397949, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.243793249130249, "sampling/sampling_logp_difference/mean": 0.13851961493492126, "step": 102, "step_time": 14.96300620297552 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 204.9375, "completions/mean_terminated_length": 211.03225708007812, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27744756161700934, "epoch": 0.00206, "frac_reward_zero_std": 0.0, "grad_norm": 0.8647305369377136, "kl": 1.8588872123509645, "learning_rate": 9.889522367416332e-06, "loss": -0.0502, "num_tokens": 4732802.0, "reward": 1.0625, "reward_std": 1.0313780307769775, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.0757592916488647, "sampling/importance_sampling_ratio/max": 1.776564598083496, "sampling/importance_sampling_ratio/mean": 0.8397999405860901, "sampling/importance_sampling_ratio/min": 0.13154380023479462, "sampling/sampling_logp_difference/max": 2.0283617973327637, "sampling/sampling_logp_difference/mean": 0.14575853943824768, "step": 103, "step_time": 13.704288512992207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 266.90625, "completions/mean_terminated_length": 266.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2680952233495191, "epoch": 0.00208, "frac_reward_zero_std": 0.0, "grad_norm": 0.40665203332901, "kl": 2.5941703654825687, "learning_rate": 9.886216819972226e-06, "loss": -0.0727, "num_tokens": 4781329.0, "reward": 0.96875, "reward_std": 1.1630816459655762, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 1.2309024333953857, "sampling/importance_sampling_ratio/max": 1.22427499294281, "sampling/importance_sampling_ratio/mean": 0.7545508146286011, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1706929206848145, "sampling/sampling_logp_difference/mean": 0.14932318031787872, "step": 104, "step_time": 14.30169549700804 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 290.625, "completions/mean_terminated_length": 290.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22253448120318353, "epoch": 0.0021, "frac_reward_zero_std": 0.0, "grad_norm": 0.4547457993030548, "kl": 1.2097172029316425, "learning_rate": 9.882863308370175e-06, "loss": -0.0935, "num_tokens": 4831254.0, "reward": 1.09375, "reward_std": 1.3458253145217896, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.3762823343276978, "sampling/importance_sampling_ratio/max": 1.701600193977356, "sampling/importance_sampling_ratio/mean": 0.8532682657241821, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3638139963150024, "sampling/sampling_logp_difference/mean": 0.1263984739780426, "step": 105, "step_time": 14.425741204016958 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.028645833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 273.875, "completions/mean_terminated_length": 273.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20992138143628836, "epoch": 0.00212, "frac_reward_zero_std": 0.0, "grad_norm": 0.6968414783477783, "kl": 1.5795001350343227, "learning_rate": 9.879461876845255e-06, "loss": -0.0625, "num_tokens": 4880275.0, "reward": 1.0625, "reward_std": 1.3266968727111816, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.268412709236145, "sampling/importance_sampling_ratio/max": 2.245370388031006, "sampling/importance_sampling_ratio/mean": 0.8836613297462463, "sampling/importance_sampling_ratio/min": 0.13906681537628174, "sampling/sampling_logp_difference/max": 1.9726738929748535, "sampling/sampling_logp_difference/mean": 0.13299354910850525, "step": 106, "step_time": 13.564993905019946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 414.0, "completions/mean_terminated_length": 414.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2227070895023644, "epoch": 0.00214, "frac_reward_zero_std": 0.0, "grad_norm": 0.8418347835540771, "kl": 0.8917684555053711, "learning_rate": 9.876012570264658e-06, "loss": -0.0619, "num_tokens": 4933585.0, "reward": 1.03125, "reward_std": 1.332922101020813, "rewards/rollout_reward_func/mean": 1.03125, "rewards/rollout_reward_func/std": 1.3556184768676758, "sampling/importance_sampling_ratio/max": 1.5475668907165527, "sampling/importance_sampling_ratio/mean": 0.8132284879684448, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.641331911087036, "sampling/sampling_logp_difference/mean": 0.13806256651878357, "step": 107, "step_time": 15.439721513990662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 260.125, "completions/mean_terminated_length": 260.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18002237007021904, "epoch": 0.00216, "frac_reward_zero_std": 0.0, "grad_norm": 0.6981293559074402, "kl": 1.0277733113616705, "learning_rate": 9.872515434127064e-06, "loss": -0.0345, "num_tokens": 4982144.0, "reward": 1.0, "reward_std": 1.0846971273422241, "rewards/rollout_reward_func/mean": 1.0, "rewards/rollout_reward_func/std": 1.1071614027023315, "sampling/importance_sampling_ratio/max": 2.9626662731170654, "sampling/importance_sampling_ratio/mean": 0.9411440491676331, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3984453678131104, "sampling/sampling_logp_difference/mean": 0.11907397210597992, "step": 108, "step_time": 13.287218255980406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 317.03125, "completions/mean_terminated_length": 317.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18486209795810282, "epoch": 0.00218, "frac_reward_zero_std": 0.0, "grad_norm": 0.38580596446990967, "kl": 1.2858061753213406, "learning_rate": 9.86897051456206e-06, "loss": -0.0712, "num_tokens": 5032784.0, "reward": 0.96875, "reward_std": 1.159074306488037, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 1.1495966911315918, "sampling/importance_sampling_ratio/max": 1.4293805360794067, "sampling/importance_sampling_ratio/mean": 0.7758607864379883, "sampling/importance_sampling_ratio/min": 0.07078838348388672, "sampling/sampling_logp_difference/max": 2.349926233291626, "sampling/sampling_logp_difference/mean": 0.16027337312698364, "step": 109, "step_time": 13.604344855964882 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 388.53125, "completions/mean_terminated_length": 388.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15890948730520904, "epoch": 0.0022, "frac_reward_zero_std": 0.0, "grad_norm": 1.1176029443740845, "kl": 1.8361767027527094, "learning_rate": 9.865377858329539e-06, "loss": -0.0461, "num_tokens": 5086592.0, "reward": 1.3125, "reward_std": 0.9103090763092041, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.1198358535766602, "sampling/importance_sampling_ratio/max": 2.563629150390625, "sampling/importance_sampling_ratio/mean": 0.8609116077423096, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.62979793548584, "sampling/sampling_logp_difference/mean": 0.12823638319969177, "step": 110, "step_time": 15.30264747502224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.046875, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 209.96875, "completions/mean_terminated_length": 209.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22185231861658394, "epoch": 0.00222, "frac_reward_zero_std": 0.0, "grad_norm": 0.36788514256477356, "kl": 1.2183266039937735, "learning_rate": 9.861737512819058e-06, "loss": -0.0766, "num_tokens": 5132535.0, "reward": 1.40625, "reward_std": 0.878544270992279, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.8747119307518005, "sampling/importance_sampling_ratio/max": 1.3334965705871582, "sampling/importance_sampling_ratio/mean": 0.7319439649581909, "sampling/importance_sampling_ratio/min": 0.14938592910766602, "sampling/sampling_logp_difference/max": 1.9011436700820923, "sampling/sampling_logp_difference/mean": 0.185620978474617, "step": 111, "step_time": 13.90263126300124 }, { "clip_ratio/high_max": 0.0625, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.046875, "completions/clipped_ratio": 0.0, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 186.65625, "completions/mean_terminated_length": 186.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25037059281021357, "epoch": 0.00224, "frac_reward_zero_std": 0.0, "grad_norm": 0.7863025069236755, "kl": 1.6412594895809889, "learning_rate": 9.858049526049241e-06, "loss": -0.0486, "num_tokens": 5177065.0, "reward": 1.09375, "reward_std": 1.128882884979248, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.0883362293243408, "sampling/importance_sampling_ratio/max": 2.807896614074707, "sampling/importance_sampling_ratio/mean": 0.8424193859100342, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.209484815597534, "sampling/sampling_logp_difference/mean": 0.19884026050567627, "step": 112, "step_time": 12.225312836977537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 376.1875, "completions/mean_terminated_length": 376.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17044509260449558, "epoch": 0.00226, "frac_reward_zero_std": 0.0, "grad_norm": 0.40476951003074646, "kl": 2.12968981731683, "learning_rate": 9.85431394666712e-06, "loss": -0.116, "num_tokens": 5230318.0, "reward": 1.3125, "reward_std": 1.3148307800292969, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.281065583229065, "sampling/importance_sampling_ratio/max": 1.4483850002288818, "sampling/importance_sampling_ratio/mean": 0.7357762455940247, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.510573387145996, "sampling/sampling_logp_difference/mean": 0.1968384087085724, "step": 113, "step_time": 14.792214386994601 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 218.0, "completions/mean_terminated_length": 218.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17695420212112367, "epoch": 0.00228, "frac_reward_zero_std": 0.0, "grad_norm": 0.33294233679771423, "kl": 1.947101267054677, "learning_rate": 9.850530823947519e-06, "loss": -0.0664, "num_tokens": 5276794.0, "reward": 1.1875, "reward_std": 1.1327643394470215, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.0906493663787842, "sampling/importance_sampling_ratio/max": 1.5687897205352783, "sampling/importance_sampling_ratio/mean": 0.8533026576042175, "sampling/importance_sampling_ratio/min": 0.09430717676877975, "sampling/sampling_logp_difference/max": 2.3611721992492676, "sampling/sampling_logp_difference/mean": 0.11788403987884521, "step": 114, "step_time": 13.451171651948243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0546875, "clip_ratio/low_min": 0.03125, "clip_ratio/region_mean": 0.0546875, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 251.71875, "completions/mean_terminated_length": 251.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1972943702712655, "epoch": 0.0023, "frac_reward_zero_std": 0.0, "grad_norm": 0.8882143497467041, "kl": 3.594126718118787, "learning_rate": 9.846700207792381e-06, "loss": -0.0343, "num_tokens": 5324559.0, "reward": 1.0625, "reward_std": 0.987578809261322, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.0757592916488647, "sampling/importance_sampling_ratio/max": 2.9105629920959473, "sampling/importance_sampling_ratio/mean": 0.8142977952957153, "sampling/importance_sampling_ratio/min": 0.06742209196090698, "sampling/sampling_logp_difference/max": 2.5405795574188232, "sampling/sampling_logp_difference/mean": 0.24153809249401093, "step": 115, "step_time": 14.45718931700685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 276.5625, "completions/mean_terminated_length": 276.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2017975776689127, "epoch": 0.00232, "frac_reward_zero_std": 0.0, "grad_norm": 1.4184907674789429, "kl": 3.7554089426994324, "learning_rate": 9.842822148730122e-06, "loss": -0.0642, "num_tokens": 5373011.0, "reward": 0.96875, "reward_std": 1.1179356575012207, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 1.1773226261138916, "sampling/importance_sampling_ratio/max": 1.237683892250061, "sampling/importance_sampling_ratio/mean": 0.6761186122894287, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1075029373168945, "sampling/sampling_logp_difference/mean": 0.20240998268127441, "step": 116, "step_time": 14.724679832987022 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 242.84375, "completions/mean_terminated_length": 242.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17774101451504976, "epoch": 0.00234, "frac_reward_zero_std": 0.0, "grad_norm": 0.5377053022384644, "kl": 1.5079663414508104, "learning_rate": 9.838896697914964e-06, "loss": -0.0682, "num_tokens": 5419449.0, "reward": 1.34375, "reward_std": 0.8599852323532104, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.9708451628684998, "sampling/importance_sampling_ratio/max": 1.561344861984253, "sampling/importance_sampling_ratio/mean": 0.8027511835098267, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0671348571777344, "sampling/sampling_logp_difference/mean": 0.16049212217330933, "step": 117, "step_time": 13.566635757000768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 425.5, "completions/mean_terminated_length": 425.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13599598850123584, "epoch": 0.00236, "frac_reward_zero_std": 0.0, "grad_norm": 0.5442306995391846, "kl": 2.625730223953724, "learning_rate": 9.834923907126256e-06, "loss": -0.06, "num_tokens": 5473736.0, "reward": 1.28125, "reward_std": 1.0547839403152466, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.1425601243972778, "sampling/importance_sampling_ratio/max": 1.4133540391921997, "sampling/importance_sampling_ratio/mean": 0.7891436815261841, "sampling/importance_sampling_ratio/min": 0.020936764776706696, "sampling/sampling_logp_difference/max": 2.5864148139953613, "sampling/sampling_logp_difference/mean": 0.1681758463382721, "step": 118, "step_time": 15.0033856099908 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 318.90625, "completions/mean_terminated_length": 318.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1874263547360897, "epoch": 0.00238, "frac_reward_zero_std": 0.0, "grad_norm": 0.6721803545951843, "kl": 2.6203718474134803, "learning_rate": 9.830903828767796e-06, "loss": -0.057, "num_tokens": 5522508.0, "reward": 1.3125, "reward_std": 0.9544942378997803, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.9979817867279053, "sampling/importance_sampling_ratio/max": 1.8634008169174194, "sampling/importance_sampling_ratio/mean": 0.7564166784286499, "sampling/importance_sampling_ratio/min": 0.10493848472833633, "sampling/sampling_logp_difference/max": 2.254395008087158, "sampling/sampling_logp_difference/mean": 0.18060891330242157, "step": 119, "step_time": 14.402309173005051 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 320.28125, "completions/mean_terminated_length": 320.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16126347112003714, "epoch": 0.0024, "frac_reward_zero_std": 0.0, "grad_norm": 0.2266867458820343, "kl": 1.4748082999140024, "learning_rate": 9.826836515867131e-06, "loss": -0.0629, "num_tokens": 5573382.0, "reward": 1.3125, "reward_std": 0.9287548065185547, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.9651173949241638, "sampling/importance_sampling_ratio/max": 2.638827085494995, "sampling/importance_sampling_ratio/mean": 0.7688011527061462, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0059237480163574, "sampling/sampling_logp_difference/mean": 0.17213714122772217, "step": 120, "step_time": 13.88061939898762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 235.8125, "completions/mean_terminated_length": 235.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11438240623101592, "epoch": 0.00242, "frac_reward_zero_std": 0.0, "grad_norm": 2.2210230827331543, "kl": 6.325484870001674, "learning_rate": 9.822722022074871e-06, "loss": -0.0491, "num_tokens": 5621050.0, "reward": 1.34375, "reward_std": 0.9551509618759155, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.0035220384597778, "sampling/importance_sampling_ratio/max": 2.1139485836029053, "sampling/importance_sampling_ratio/mean": 0.8948121070861816, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.407421350479126, "sampling/sampling_logp_difference/mean": 0.13261175155639648, "step": 121, "step_time": 13.850175498984754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 384.78125, "completions/mean_terminated_length": 384.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1481839888729155, "epoch": 0.00244, "frac_reward_zero_std": 0.0, "grad_norm": 0.8597337007522583, "kl": 1.0615279665216804, "learning_rate": 9.818560401663972e-06, "loss": -0.0829, "num_tokens": 5673939.0, "reward": 1.15625, "reward_std": 1.296135663986206, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.2727763652801514, "sampling/importance_sampling_ratio/max": 2.0567219257354736, "sampling/importance_sampling_ratio/mean": 0.9168287515640259, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.272522449493408, "sampling/sampling_logp_difference/mean": 0.13784050941467285, "step": 122, "step_time": 14.985034360026475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.046875, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 261.125, "completions/mean_terminated_length": 261.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17314486170653254, "epoch": 0.00246, "frac_reward_zero_std": 0.0, "grad_norm": 0.3472461998462677, "kl": 1.5742682982236147, "learning_rate": 9.814351709529018e-06, "loss": -0.0813, "num_tokens": 5721780.0, "reward": 1.15625, "reward_std": 1.1471906900405884, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.167002558708191, "sampling/importance_sampling_ratio/max": 2.0025839805603027, "sampling/importance_sampling_ratio/mean": 0.7789444923400879, "sampling/importance_sampling_ratio/min": 0.05767301842570305, "sampling/sampling_logp_difference/max": 2.156113624572754, "sampling/sampling_logp_difference/mean": 0.2358439862728119, "step": 123, "step_time": 13.230079361004755 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1270.0, "completions/max_terminated_length": 1270.0, "completions/mean_length": 341.78125, "completions/mean_terminated_length": 341.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15489079128019512, "epoch": 0.00248, "frac_reward_zero_std": 0.0, "grad_norm": 0.6214333176612854, "kl": 3.5453504286706448, "learning_rate": 9.8100960011855e-06, "loss": -0.0688, "num_tokens": 5772352.0, "reward": 1.15625, "reward_std": 1.333357572555542, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.3224945068359375, "sampling/importance_sampling_ratio/max": 1.647971749305725, "sampling/importance_sampling_ratio/mean": 0.6532666683197021, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0146241188049316, "sampling/sampling_logp_difference/mean": 0.24523651599884033, "step": 124, "step_time": 15.546281347022159 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 305.375, "completions/mean_terminated_length": 305.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15906932996585965, "epoch": 0.0025, "frac_reward_zero_std": 0.0, "grad_norm": 0.6180495023727417, "kl": 0.8607366234064102, "learning_rate": 9.805793332769095e-06, "loss": -0.0471, "num_tokens": 5822160.0, "reward": 1.375, "reward_std": 0.9391831159591675, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.941858172416687, "sampling/importance_sampling_ratio/max": 2.815729856491089, "sampling/importance_sampling_ratio/mean": 0.8729480504989624, "sampling/importance_sampling_ratio/min": 0.085760697722435, "sampling/sampling_logp_difference/max": 2.456094741821289, "sampling/sampling_logp_difference/mean": 0.14068126678466797, "step": 125, "step_time": 14.289117105989135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 784.0, "completions/max_terminated_length": 784.0, "completions/mean_length": 251.96875, "completions/mean_terminated_length": 251.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1240817024372518, "epoch": 0.00252, "frac_reward_zero_std": 0.0, "grad_norm": 0.3374398648738861, "kl": 1.8512703916057944, "learning_rate": 9.801443761034903e-06, "loss": -0.0091, "num_tokens": 5870038.0, "reward": 0.9375, "reward_std": 1.2243475914001465, "rewards/rollout_reward_func/mean": 0.9375, "rewards/rollout_reward_func/std": 1.2427207231521606, "sampling/importance_sampling_ratio/max": 2.3698177337646484, "sampling/importance_sampling_ratio/mean": 0.9345589280128479, "sampling/importance_sampling_ratio/min": 0.07700422406196594, "sampling/sampling_logp_difference/max": 2.572364568710327, "sampling/sampling_logp_difference/mean": 0.15856684744358063, "step": 126, "step_time": 11.977175460997387 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 283.1875, "completions/mean_terminated_length": 283.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17827959510032088, "epoch": 0.00254, "frac_reward_zero_std": 0.0, "grad_norm": 0.3603960871696472, "kl": 1.7320336010307074, "learning_rate": 9.797047343356718e-06, "loss": -0.0728, "num_tokens": 5919040.0, "reward": 1.1875, "reward_std": 1.2985808849334717, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.3781123161315918, "sampling/importance_sampling_ratio/max": 2.4201645851135254, "sampling/importance_sampling_ratio/mean": 0.8766059875488281, "sampling/importance_sampling_ratio/min": 0.09021345525979996, "sampling/sampling_logp_difference/max": 2.405421495437622, "sampling/sampling_logp_difference/mean": 0.17211870849132538, "step": 127, "step_time": 15.077540861966554 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 366.625, "completions/mean_terminated_length": 366.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15426228707656264, "epoch": 0.00256, "frac_reward_zero_std": 0.0, "grad_norm": 0.6722431778907776, "kl": 1.8802095092833042, "learning_rate": 9.792604137726259e-06, "loss": -0.0436, "num_tokens": 5971822.0, "reward": 1.0625, "reward_std": 1.2731267213821411, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.2427207231521606, "sampling/importance_sampling_ratio/max": 2.323535680770874, "sampling/importance_sampling_ratio/mean": 0.9477406740188599, "sampling/importance_sampling_ratio/min": 0.05419948324561119, "sampling/sampling_logp_difference/max": 2.914781093597412, "sampling/sampling_logp_difference/mean": 0.11999005079269409, "step": 128, "step_time": 13.447650452028029 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 302.84375, "completions/mean_terminated_length": 302.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20137143041938543, "epoch": 0.00258, "frac_reward_zero_std": 0.0, "grad_norm": 0.601019024848938, "kl": 1.9521203003823757, "learning_rate": 9.788114202752415e-06, "loss": -0.0981, "num_tokens": 6020927.0, "reward": 0.875, "reward_std": 1.4042547941207886, "rewards/rollout_reward_func/mean": 0.875, "rewards/rollout_reward_func/std": 1.38540780544281, "sampling/importance_sampling_ratio/max": 1.6969562768936157, "sampling/importance_sampling_ratio/mean": 0.7849422693252563, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.02504825592041, "sampling/sampling_logp_difference/mean": 0.20325344800949097, "step": 129, "step_time": 13.744189225995797 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 241.1875, "completions/mean_terminated_length": 241.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.213963208720088, "epoch": 0.0026, "frac_reward_zero_std": 0.0, "grad_norm": 0.44000834226608276, "kl": 2.0944806709885597, "learning_rate": 9.78357759766046e-06, "loss": -0.0859, "num_tokens": 6067631.0, "reward": 0.78125, "reward_std": 1.2739992141723633, "rewards/rollout_reward_func/mean": 0.78125, "rewards/rollout_reward_func/std": 1.263236403465271, "sampling/importance_sampling_ratio/max": 1.2499077320098877, "sampling/importance_sampling_ratio/mean": 0.659430980682373, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9470508098602295, "sampling/sampling_logp_difference/mean": 0.20708227157592773, "step": 130, "step_time": 14.037352448984166 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 548.125, "completions/mean_terminated_length": 548.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11975072650238872, "epoch": 0.00262, "frac_reward_zero_std": 0.0, "grad_norm": 0.5161893963813782, "kl": 2.2450691796839237, "learning_rate": 9.778994382291283e-06, "loss": -0.0412, "num_tokens": 6128032.0, "reward": 1.5625, "reward_std": 1.039553165435791, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 1.1053389310836792, "sampling/importance_sampling_ratio/max": 1.3651703596115112, "sampling/importance_sampling_ratio/mean": 0.9003458619117737, "sampling/importance_sampling_ratio/min": 0.19690868258476257, "sampling/sampling_logp_difference/max": 1.7630552053451538, "sampling/sampling_logp_difference/mean": 0.0797187089920044, "step": 131, "step_time": 14.88791667597252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 233.03125, "completions/mean_terminated_length": 233.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1920871522743255, "epoch": 0.00264, "frac_reward_zero_std": 0.0, "grad_norm": 0.14959612488746643, "kl": 1.129742108285427, "learning_rate": 9.774364617100596e-06, "loss": -0.0464, "num_tokens": 6175119.0, "reward": 0.59375, "reward_std": 1.2623143196105957, "rewards/rollout_reward_func/mean": 0.59375, "rewards/rollout_reward_func/std": 1.266424298286438, "sampling/importance_sampling_ratio/max": 1.5154683589935303, "sampling/importance_sampling_ratio/mean": 0.7826578617095947, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3525792360305786, "sampling/sampling_logp_difference/mean": 0.14975200593471527, "step": 132, "step_time": 11.668955407949397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 266.3125, "completions/mean_terminated_length": 266.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16706520144362003, "epoch": 0.00266, "frac_reward_zero_std": 0.0, "grad_norm": 0.29701992869377136, "kl": 1.3321392517536879, "learning_rate": 9.769688363158127e-06, "loss": -0.0248, "num_tokens": 6223528.0, "reward": 0.875, "reward_std": 1.2419415712356567, "rewards/rollout_reward_func/mean": 0.875, "rewards/rollout_reward_func/std": 1.2115039825439453, "sampling/importance_sampling_ratio/max": 2.3380279541015625, "sampling/importance_sampling_ratio/mean": 0.8778375387191772, "sampling/importance_sampling_ratio/min": 0.11521721631288528, "sampling/sampling_logp_difference/max": 2.1608638763427734, "sampling/sampling_logp_difference/mean": 0.1798931509256363, "step": 133, "step_time": 13.889362932022777 }, { "clip_ratio/high_max": 0.0625, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 313.09375, "completions/mean_terminated_length": 313.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18445693724788725, "epoch": 0.00268, "frac_reward_zero_std": 0.0, "grad_norm": 0.3889417350292206, "kl": 4.001335929147899, "learning_rate": 9.76496568214683e-06, "loss": -0.0837, "num_tokens": 6273131.0, "reward": 1.09375, "reward_std": 1.3549834489822388, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.3526407480239868, "sampling/importance_sampling_ratio/max": 1.655823826789856, "sampling/importance_sampling_ratio/mean": 0.7519404888153076, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.563716411590576, "sampling/sampling_logp_difference/mean": 0.18903721868991852, "step": 134, "step_time": 13.93148724999628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 307.75, "completions/mean_terminated_length": 307.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1660536378622055, "epoch": 0.0027, "frac_reward_zero_std": 0.0, "grad_norm": 0.32876741886138916, "kl": 1.0296348296105862, "learning_rate": 9.760196636362058e-06, "loss": -0.0716, "num_tokens": 6323714.0, "reward": 1.46875, "reward_std": 1.1774959564208984, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.1635422706604004, "sampling/importance_sampling_ratio/max": 2.311647891998291, "sampling/importance_sampling_ratio/mean": 0.8481613993644714, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7935080528259277, "sampling/sampling_logp_difference/mean": 0.11996470391750336, "step": 135, "step_time": 13.826161960998434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 284.8125, "completions/mean_terminated_length": 284.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22113141929730773, "epoch": 0.00272, "frac_reward_zero_std": 0.0, "grad_norm": 0.408471018075943, "kl": 2.0748042687773705, "learning_rate": 9.755381288710751e-06, "loss": -0.0612, "num_tokens": 6372053.0, "reward": 0.71875, "reward_std": 1.2967110872268677, "rewards/rollout_reward_func/mean": 0.71875, "rewards/rollout_reward_func/std": 1.3496564626693726, "sampling/importance_sampling_ratio/max": 1.7738945484161377, "sampling/importance_sampling_ratio/mean": 0.851650595664978, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5794353485107422, "sampling/sampling_logp_difference/mean": 0.14824968576431274, "step": 136, "step_time": 13.885015921026934 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 256.5, "completions/mean_terminated_length": 256.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19809468917082995, "epoch": 0.00274, "frac_reward_zero_std": 0.0, "grad_norm": 0.36826297640800476, "kl": 1.3085883958265185, "learning_rate": 9.750519702710598e-06, "loss": -0.0615, "num_tokens": 6419909.0, "reward": 1.0625, "reward_std": 1.320794701576233, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.2935945987701416, "sampling/importance_sampling_ratio/max": 1.8246791362762451, "sampling/importance_sampling_ratio/mean": 0.8179781436920166, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.584231376647949, "sampling/sampling_logp_difference/mean": 0.160110741853714, "step": 137, "step_time": 14.238315248003346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 374.09375, "completions/mean_terminated_length": 374.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2123808212345466, "epoch": 0.00276, "frac_reward_zero_std": 0.0, "grad_norm": 0.9399845004081726, "kl": 3.509156296029687, "learning_rate": 9.745611942489202e-06, "loss": -0.0679, "num_tokens": 6471718.0, "reward": 1.03125, "reward_std": 1.3297159671783447, "rewards/rollout_reward_func/mean": 1.03125, "rewards/rollout_reward_func/std": 1.4252192974090576, "sampling/importance_sampling_ratio/max": 2.293229579925537, "sampling/importance_sampling_ratio/mean": 0.8330119848251343, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.292904853820801, "sampling/sampling_logp_difference/mean": 0.17925134301185608, "step": 138, "step_time": 14.895737515995279 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 838.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 224.71875, "completions/mean_terminated_length": 224.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20427117228973657, "epoch": 0.00278, "frac_reward_zero_std": 0.0, "grad_norm": 0.2862147390842438, "kl": 2.299131779000163, "learning_rate": 9.740658072783244e-06, "loss": -0.0498, "num_tokens": 6518980.0, "reward": 1.125, "reward_std": 1.1993839740753174, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 1.263635277748108, "sampling/importance_sampling_ratio/max": 2.177474021911621, "sampling/importance_sampling_ratio/mean": 0.8810761570930481, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.198241949081421, "sampling/sampling_logp_difference/mean": 0.14731958508491516, "step": 139, "step_time": 12.492955234993133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 248.3125, "completions/mean_terminated_length": 248.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19338116270955652, "epoch": 0.0028, "frac_reward_zero_std": 0.0, "grad_norm": 0.44646069407463074, "kl": 1.0222832206636667, "learning_rate": 9.73565815893761e-06, "loss": -0.04, "num_tokens": 6567080.0, "reward": 1.1875, "reward_std": 0.8602820038795471, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.1482806205749512, "sampling/importance_sampling_ratio/max": 1.432073712348938, "sampling/importance_sampling_ratio/mean": 0.897666335105896, "sampling/importance_sampling_ratio/min": 0.26819977164268494, "sampling/sampling_logp_difference/max": 1.0815095901489258, "sampling/sampling_logp_difference/mean": 0.07340474426746368, "step": 140, "step_time": 13.875526928997715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 288.9375, "completions/mean_terminated_length": 288.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22114048642106354, "epoch": 0.00282, "frac_reward_zero_std": 0.0, "grad_norm": 0.4551962614059448, "kl": 0.9591837413609028, "learning_rate": 9.730612266904548e-06, "loss": -0.0469, "num_tokens": 6616859.0, "reward": 0.96875, "reward_std": 1.151381254196167, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 1.121185302734375, "sampling/importance_sampling_ratio/max": 1.2538485527038574, "sampling/importance_sampling_ratio/mean": 0.8550385236740112, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4532852172851562, "sampling/sampling_logp_difference/mean": 0.10415095835924149, "step": 141, "step_time": 14.116877781008952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1288.0, "completions/max_terminated_length": 1288.0, "completions/mean_length": 365.375, "completions/mean_terminated_length": 365.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1889871247112751, "epoch": 0.00284, "frac_reward_zero_std": 0.0, "grad_norm": 0.2870098352432251, "kl": 1.097646875306964, "learning_rate": 9.72552046324278e-06, "loss": -0.0834, "num_tokens": 6667789.0, "reward": 1.53125, "reward_std": 0.8627535104751587, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.9832262396812439, "sampling/importance_sampling_ratio/max": 1.4892014265060425, "sampling/importance_sampling_ratio/mean": 0.8983615636825562, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2728917598724365, "sampling/sampling_logp_difference/mean": 0.08627241849899292, "step": 142, "step_time": 14.592704096954549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 249.3125, "completions/mean_terminated_length": 249.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17449068173300475, "epoch": 0.00286, "frac_reward_zero_std": 0.0, "grad_norm": 1.307930588722229, "kl": 0.7698394069448113, "learning_rate": 9.72038281511664e-06, "loss": -0.0419, "num_tokens": 6716150.0, "reward": 1.21875, "reward_std": 0.9039911031723022, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 0.9064064025878906, "sampling/importance_sampling_ratio/max": 1.2562305927276611, "sampling/importance_sampling_ratio/mean": 0.9188930988311768, "sampling/importance_sampling_ratio/min": 0.5112020969390869, "sampling/sampling_logp_difference/max": 0.6708117127418518, "sampling/sampling_logp_difference/mean": 0.04882374405860901, "step": 143, "step_time": 13.857101939996937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 311.28125, "completions/mean_terminated_length": 311.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23418543010484427, "epoch": 0.00288, "frac_reward_zero_std": 0.0, "grad_norm": 0.6690128445625305, "kl": 0.9600532706826925, "learning_rate": 9.715199390295181e-06, "loss": -0.0888, "num_tokens": 6766149.0, "reward": 1.25, "reward_std": 1.2696597576141357, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.295152187347412, "sampling/importance_sampling_ratio/max": 1.5444436073303223, "sampling/importance_sampling_ratio/mean": 0.9053760766983032, "sampling/importance_sampling_ratio/min": 0.37153372168540955, "sampling/sampling_logp_difference/max": 0.9899716377258301, "sampling/sampling_logp_difference/mean": 0.0747428610920906, "step": 144, "step_time": 14.316153897991171 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 249.875, "completions/mean_terminated_length": 249.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2099364373134449, "epoch": 0.0029, "frac_reward_zero_std": 0.0, "grad_norm": 0.7563704252243042, "kl": 1.129881869070232, "learning_rate": 9.70997025715128e-06, "loss": -0.0658, "num_tokens": 6815066.0, "reward": 1.15625, "reward_std": 1.0302629470825195, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.050633430480957, "sampling/importance_sampling_ratio/max": 1.2043788433074951, "sampling/importance_sampling_ratio/mean": 0.8180960416793823, "sampling/importance_sampling_ratio/min": 0.3126044273376465, "sampling/sampling_logp_difference/max": 1.1606061458587646, "sampling/sampling_logp_difference/mean": 0.09385167062282562, "step": 145, "step_time": 13.658980200969381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 341.75, "completions/mean_terminated_length": 341.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1695311317453161, "epoch": 0.00292, "frac_reward_zero_std": 0.0, "grad_norm": 0.7111180424690247, "kl": 1.6455589681863785, "learning_rate": 9.704695484660736e-06, "loss": -0.073, "num_tokens": 6866769.0, "reward": 1.40625, "reward_std": 1.078390121459961, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.1319231986999512, "sampling/importance_sampling_ratio/max": 1.308408260345459, "sampling/importance_sampling_ratio/mean": 0.8429861068725586, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5514869689941406, "sampling/sampling_logp_difference/mean": 0.10012258589267731, "step": 146, "step_time": 14.30287343901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 205.21875, "completions/mean_terminated_length": 205.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2338483437197283, "epoch": 0.00294, "frac_reward_zero_std": 0.0, "grad_norm": 0.3799244165420532, "kl": 1.884797865524888, "learning_rate": 9.699375142401365e-06, "loss": -0.0504, "num_tokens": 6911885.0, "reward": 1.03125, "reward_std": 1.295470118522644, "rewards/rollout_reward_func/mean": 1.03125, "rewards/rollout_reward_func/std": 1.256836175918579, "sampling/importance_sampling_ratio/max": 2.707463026046753, "sampling/importance_sampling_ratio/mean": 0.9390182495117188, "sampling/importance_sampling_ratio/min": 0.22109456360340118, "sampling/sampling_logp_difference/max": 1.5084130764007568, "sampling/sampling_logp_difference/mean": 0.12976568937301636, "step": 147, "step_time": 14.63131550597609 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 224.4375, "completions/mean_terminated_length": 224.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2000522018643096, "epoch": 0.00296, "frac_reward_zero_std": 0.0, "grad_norm": 1.2028510570526123, "kl": 6.4571885373443365, "learning_rate": 9.694009300552081e-06, "loss": -0.0355, "num_tokens": 6959372.0, "reward": 0.875, "reward_std": 1.348803997039795, "rewards/rollout_reward_func/mean": 0.875, "rewards/rollout_reward_func/std": 1.338029384613037, "sampling/importance_sampling_ratio/max": 2.760803461074829, "sampling/importance_sampling_ratio/mean": 0.8745015859603882, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.082157611846924, "sampling/sampling_logp_difference/mean": 0.1808915138244629, "step": 148, "step_time": 13.70771933895594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1288.0, "completions/max_terminated_length": 1288.0, "completions/mean_length": 275.4375, "completions/mean_terminated_length": 275.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20681680785492063, "epoch": 0.00298, "frac_reward_zero_std": 0.0, "grad_norm": 0.6584192514419556, "kl": 0.844359845854342, "learning_rate": 9.68859802989196e-06, "loss": -0.0711, "num_tokens": 7008104.0, "reward": 1.40625, "reward_std": 1.1555156707763672, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.1319231986999512, "sampling/importance_sampling_ratio/max": 1.283434271812439, "sampling/importance_sampling_ratio/mean": 0.8245251774787903, "sampling/importance_sampling_ratio/min": 0.21684080362319946, "sampling/sampling_logp_difference/max": 1.5277409553527832, "sampling/sampling_logp_difference/mean": 0.1112825870513916, "step": 149, "step_time": 14.983133889021701 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 372.125, "completions/mean_terminated_length": 372.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15372822992503643, "epoch": 0.003, "frac_reward_zero_std": 0.0, "grad_norm": 0.6745824813842773, "kl": 1.223582106642425, "learning_rate": 9.683141401799326e-06, "loss": -0.0332, "num_tokens": 7061021.0, "reward": 1.34375, "reward_std": 1.0535039901733398, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.180742621421814, "sampling/importance_sampling_ratio/max": 2.468240737915039, "sampling/importance_sampling_ratio/mean": 0.9683204889297485, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3863036632537842, "sampling/sampling_logp_difference/mean": 0.09529823064804077, "step": 150, "step_time": 14.479191737991641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 290.625, "completions/mean_terminated_length": 290.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23257667431607842, "epoch": 0.00302, "frac_reward_zero_std": 0.0, "grad_norm": 0.5663034915924072, "kl": 1.056601855903864, "learning_rate": 9.677639488250787e-06, "loss": -0.0324, "num_tokens": 7109098.0, "reward": 1.15625, "reward_std": 1.1999757289886475, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.1943246126174927, "sampling/importance_sampling_ratio/max": 2.8148343563079834, "sampling/importance_sampling_ratio/mean": 0.9836826920509338, "sampling/importance_sampling_ratio/min": 0.23474809527397156, "sampling/sampling_logp_difference/max": 1.4491827487945557, "sampling/sampling_logp_difference/mean": 0.15070267021656036, "step": 151, "step_time": 13.596281572026783 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 252.125, "completions/mean_terminated_length": 252.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2262503345264122, "epoch": 0.00304, "frac_reward_zero_std": 0.0, "grad_norm": 0.26775503158569336, "kl": 1.6905091721564531, "learning_rate": 9.672092361820306e-06, "loss": -0.0803, "num_tokens": 7156796.0, "reward": 1.09375, "reward_std": 1.3031929731369019, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.3285784721374512, "sampling/importance_sampling_ratio/max": 1.8267841339111328, "sampling/importance_sampling_ratio/mean": 0.7990831136703491, "sampling/importance_sampling_ratio/min": 0.040731094777584076, "sampling/sampling_logp_difference/max": 1.6881003379821777, "sampling/sampling_logp_difference/mean": 0.15996326506137848, "step": 152, "step_time": 14.13111290197412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 301.84375, "completions/mean_terminated_length": 301.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1700148235540837, "epoch": 0.00306, "frac_reward_zero_std": 0.0, "grad_norm": 0.37110546231269836, "kl": 0.882345387712121, "learning_rate": 9.666500095678226e-06, "loss": -0.0576, "num_tokens": 7207302.0, "reward": 1.21875, "reward_std": 1.1913070678710938, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.156590461730957, "sampling/importance_sampling_ratio/max": 2.70015025138855, "sampling/importance_sampling_ratio/mean": 0.9818962216377258, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.603177785873413, "sampling/sampling_logp_difference/mean": 0.08297109603881836, "step": 153, "step_time": 13.713550230007968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 336.96875, "completions/mean_terminated_length": 336.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16135327972006053, "epoch": 0.00308, "frac_reward_zero_std": 0.0, "grad_norm": 0.275499165058136, "kl": 1.5272481888532639, "learning_rate": 9.660862763590322e-06, "loss": -0.0555, "num_tokens": 7258000.0, "reward": 1.28125, "reward_std": 1.2956287860870361, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.2759404182434082, "sampling/importance_sampling_ratio/max": 1.5687167644500732, "sampling/importance_sampling_ratio/mean": 0.9393272399902344, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6409235000610352, "sampling/sampling_logp_difference/mean": 0.07315731048583984, "step": 154, "step_time": 14.748264763999032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 192.75, "completions/mean_terminated_length": 197.9354705810547, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2248875016812235, "epoch": 0.0031, "frac_reward_zero_std": 0.0, "grad_norm": 0.47834065556526184, "kl": 1.5227338299155235, "learning_rate": 9.655180439916814e-06, "loss": -0.0348, "num_tokens": 7304272.0, "reward": 1.15625, "reward_std": 1.2384815216064453, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.1943246126174927, "sampling/importance_sampling_ratio/max": 2.315746307373047, "sampling/importance_sampling_ratio/mean": 1.0376530885696411, "sampling/importance_sampling_ratio/min": 2.1429184031553513e-23, "sampling/sampling_logp_difference/max": 26.683996200561523, "sampling/sampling_logp_difference/mean": 0.40860599279403687, "step": 155, "step_time": 11.754180272953818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 312.9375, "completions/mean_terminated_length": 312.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17029064474627376, "epoch": 0.00312, "frac_reward_zero_std": 0.0, "grad_norm": 1.7377171516418457, "kl": 4.850027801468968, "learning_rate": 9.64945319961139e-06, "loss": -0.0399, "num_tokens": 7355547.0, "reward": 1.1875, "reward_std": 1.131500005722046, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.1198358535766602, "sampling/importance_sampling_ratio/max": 2.137042999267578, "sampling/importance_sampling_ratio/mean": 0.915464460849762, "sampling/importance_sampling_ratio/min": 0.2898354232311249, "sampling/sampling_logp_difference/max": 1.2394299507141113, "sampling/sampling_logp_difference/mean": 0.08941207826137543, "step": 156, "step_time": 13.949641831990448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 249.1875, "completions/mean_terminated_length": 249.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2350542660569772, "epoch": 0.00314, "frac_reward_zero_std": 0.0, "grad_norm": 0.3832235336303711, "kl": 0.9457587283104658, "learning_rate": 9.643681118220224e-06, "loss": -0.0681, "num_tokens": 7402114.0, "reward": 1.125, "reward_std": 1.2414333820343018, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 1.31369948387146, "sampling/importance_sampling_ratio/max": 2.300368070602417, "sampling/importance_sampling_ratio/mean": 0.9687473177909851, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2586066722869873, "sampling/sampling_logp_difference/mean": 0.08579417318105698, "step": 157, "step_time": 13.69429440300155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 190.0625, "completions/mean_terminated_length": 190.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24992230138741434, "epoch": 0.00316, "frac_reward_zero_std": 0.0, "grad_norm": 1.1671841144561768, "kl": 2.255382936447859, "learning_rate": 9.637864271880972e-06, "loss": -0.0392, "num_tokens": 7446070.0, "reward": 0.90625, "reward_std": 1.054424524307251, "rewards/rollout_reward_func/mean": 0.90625, "rewards/rollout_reward_func/std": 1.058281421661377, "sampling/importance_sampling_ratio/max": 2.8599319458007812, "sampling/importance_sampling_ratio/mean": 0.9034602642059326, "sampling/importance_sampling_ratio/min": 0.28169408440589905, "sampling/sampling_logp_difference/max": 1.266477108001709, "sampling/sampling_logp_difference/mean": 0.1166858822107315, "step": 158, "step_time": 13.802159664017381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 248.59375, "completions/mean_terminated_length": 248.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22115793253760785, "epoch": 0.00318, "frac_reward_zero_std": 0.0, "grad_norm": 0.6137709617614746, "kl": 1.864316202700138, "learning_rate": 9.632002737321768e-06, "loss": -0.0513, "num_tokens": 7493637.0, "reward": 1.25, "reward_std": 0.9730371236801147, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9837387204170227, "sampling/importance_sampling_ratio/max": 1.9693505764007568, "sampling/importance_sampling_ratio/mean": 0.9112547636032104, "sampling/importance_sampling_ratio/min": 0.2539611756801605, "sampling/sampling_logp_difference/max": 1.3719000816345215, "sampling/sampling_logp_difference/mean": 0.07083098590373993, "step": 159, "step_time": 14.089623506006319 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1270.0, "completions/max_terminated_length": 1270.0, "completions/mean_length": 308.0625, "completions/mean_terminated_length": 308.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20744509832002223, "epoch": 0.0032, "frac_reward_zero_std": 0.0, "grad_norm": 0.42464494705200195, "kl": 3.1837764736264944, "learning_rate": 9.626096591860215e-06, "loss": -0.0766, "num_tokens": 7542981.0, "reward": 0.96875, "reward_std": 1.3929297924041748, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 1.4024028778076172, "sampling/importance_sampling_ratio/max": 2.33221173286438, "sampling/importance_sampling_ratio/mean": 0.9076904654502869, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.401538610458374, "sampling/sampling_logp_difference/mean": 0.10393822193145752, "step": 160, "step_time": 14.718676014002995 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 296.96875, "completions/mean_terminated_length": 296.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1727471719495952, "epoch": 0.00322, "frac_reward_zero_std": 0.0, "grad_norm": 0.5552732348442078, "kl": 0.8587866835296154, "learning_rate": 9.620145913402366e-06, "loss": -0.0513, "num_tokens": 7592295.0, "reward": 1.4375, "reward_std": 0.7410570383071899, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.7593502998352051, "sampling/importance_sampling_ratio/max": 1.2572555541992188, "sampling/importance_sampling_ratio/mean": 0.9272380471229553, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4972434043884277, "sampling/sampling_logp_difference/mean": 0.05386638641357422, "step": 161, "step_time": 14.043268853987684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 291.59375, "completions/mean_terminated_length": 291.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1681077032117173, "epoch": 0.00324, "frac_reward_zero_std": 0.0, "grad_norm": 0.5011897087097168, "kl": 1.1701193936169147, "learning_rate": 9.614150780441687e-06, "loss": -0.0473, "num_tokens": 7641113.0, "reward": 1.1875, "reward_std": 1.08481764793396, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.0606601238250732, "sampling/importance_sampling_ratio/max": 1.3320823907852173, "sampling/importance_sampling_ratio/mean": 0.9172073602676392, "sampling/importance_sampling_ratio/min": 0.2760615944862366, "sampling/sampling_logp_difference/max": 1.3026633262634277, "sampling/sampling_logp_difference/mean": 0.060095276683568954, "step": 162, "step_time": 13.581988297984935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 267.0, "completions/mean_terminated_length": 267.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13954674219712615, "epoch": 0.00326, "frac_reward_zero_std": 0.0, "grad_norm": 0.3637569844722748, "kl": 1.251847218722105, "learning_rate": 9.608111272058044e-06, "loss": -0.0494, "num_tokens": 7689464.0, "reward": 1.28125, "reward_std": 0.9099595546722412, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.8884337544441223, "sampling/importance_sampling_ratio/max": 1.385473370552063, "sampling/importance_sampling_ratio/mean": 0.932716965675354, "sampling/importance_sampling_ratio/min": 0.26080644130706787, "sampling/sampling_logp_difference/max": 1.3459833860397339, "sampling/sampling_logp_difference/mean": 0.06417685747146606, "step": 163, "step_time": 14.575866096973186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 298.3125, "completions/mean_terminated_length": 298.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29949497035704553, "epoch": 0.00328, "frac_reward_zero_std": 0.0, "grad_norm": 1.5728527307510376, "kl": 1.2494107093662024, "learning_rate": 9.602027467916626e-06, "loss": -0.0866, "num_tokens": 7739551.0, "reward": 1.25, "reward_std": 0.8924203515052795, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9503819346427917, "sampling/importance_sampling_ratio/max": 1.9871065616607666, "sampling/importance_sampling_ratio/mean": 0.9812966585159302, "sampling/importance_sampling_ratio/min": 0.3860509991645813, "sampling/sampling_logp_difference/max": 0.9516582489013672, "sampling/sampling_logp_difference/mean": 0.04383520036935806, "step": 164, "step_time": 13.600367920997087 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 287.96875, "completions/mean_terminated_length": 287.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16942259250208735, "epoch": 0.0033, "frac_reward_zero_std": 0.0, "grad_norm": 0.4622916579246521, "kl": 1.7460040040314198, "learning_rate": 9.595899448266928e-06, "loss": -0.0646, "num_tokens": 7787967.0, "reward": 1.25, "reward_std": 1.2990679740905762, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.2700011730194092, "sampling/importance_sampling_ratio/max": 1.3974705934524536, "sampling/importance_sampling_ratio/mean": 0.9912136793136597, "sampling/importance_sampling_ratio/min": 0.20864850282669067, "sampling/sampling_logp_difference/max": 1.566923975944519, "sampling/sampling_logp_difference/mean": 0.05490101873874664, "step": 165, "step_time": 13.929057236018707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 287.84375, "completions/mean_terminated_length": 287.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19549519009888172, "epoch": 0.00332, "frac_reward_zero_std": 0.0, "grad_norm": 0.6291950345039368, "kl": 2.0193503238260746, "learning_rate": 9.589727293941669e-06, "loss": -0.0295, "num_tokens": 7836876.0, "reward": 0.96875, "reward_std": 1.2063648700714111, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 1.1773226261138916, "sampling/importance_sampling_ratio/max": 2.183983325958252, "sampling/importance_sampling_ratio/mean": 1.0185825824737549, "sampling/importance_sampling_ratio/min": 0.23785284161567688, "sampling/sampling_logp_difference/max": 1.4362261295318604, "sampling/sampling_logp_difference/mean": 0.05194416642189026, "step": 166, "step_time": 13.988386689001345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 285.9375, "completions/mean_terminated_length": 285.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13771076290868223, "epoch": 0.00334, "frac_reward_zero_std": 0.0, "grad_norm": 27.184120178222656, "kl": 89.37207950279117, "learning_rate": 9.583511086355738e-06, "loss": 0.0276, "num_tokens": 7887503.0, "reward": 1.3125, "reward_std": 1.2556688785552979, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.306003451347351, "sampling/importance_sampling_ratio/max": 1.3999048471450806, "sampling/importance_sampling_ratio/mean": 0.9074404835700989, "sampling/importance_sampling_ratio/min": 0.16397534310817719, "sampling/sampling_logp_difference/max": 1.807729721069336, "sampling/sampling_logp_difference/mean": 0.09755246341228485, "step": 167, "step_time": 13.837944600047194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 361.15625, "completions/mean_terminated_length": 361.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12089334265328944, "epoch": 0.00336, "frac_reward_zero_std": 0.0, "grad_norm": 0.31009796261787415, "kl": 1.602509792894125, "learning_rate": 9.577250907505117e-06, "loss": -0.051, "num_tokens": 7939606.0, "reward": 1.40625, "reward_std": 0.9385567903518677, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9791166186332703, "sampling/importance_sampling_ratio/max": 1.3149515390396118, "sampling/importance_sampling_ratio/mean": 0.9820249080657959, "sampling/importance_sampling_ratio/min": 0.34530651569366455, "sampling/sampling_logp_difference/max": 1.0716694593429565, "sampling/sampling_logp_difference/mean": 0.03663084656000137, "step": 168, "step_time": 14.478863046038896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 296.46875, "completions/mean_terminated_length": 296.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21735665504820645, "epoch": 0.00338, "frac_reward_zero_std": 0.0, "grad_norm": 0.5689941644668579, "kl": 1.371724370867014, "learning_rate": 9.570946839965795e-06, "loss": -0.0545, "num_tokens": 7988097.0, "reward": 1.25, "reward_std": 0.8694722056388855, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9158109426498413, "sampling/importance_sampling_ratio/max": 1.3911453485488892, "sampling/importance_sampling_ratio/mean": 0.953101634979248, "sampling/importance_sampling_ratio/min": 0.20569561421871185, "sampling/sampling_logp_difference/max": 1.5998129844665527, "sampling/sampling_logp_difference/mean": 0.055657703429460526, "step": 169, "step_time": 14.583978247959749 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1346.0, "completions/max_terminated_length": 1346.0, "completions/mean_length": 328.4375, "completions/mean_terminated_length": 328.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23315885034389794, "epoch": 0.0034, "frac_reward_zero_std": 0.0, "grad_norm": 1.0677882432937622, "kl": 2.131178052164614, "learning_rate": 9.564598966892683e-06, "loss": -0.0732, "num_tokens": 8039447.0, "reward": 1.125, "reward_std": 1.286641001701355, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 1.263635277748108, "sampling/importance_sampling_ratio/max": 1.5754448175430298, "sampling/importance_sampling_ratio/mean": 0.9204651713371277, "sampling/importance_sampling_ratio/min": 0.23435264825820923, "sampling/sampling_logp_difference/max": 1.4902777671813965, "sampling/sampling_logp_difference/mean": 0.07006184756755829, "step": 170, "step_time": 15.184720361008658 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1364.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 325.3125, "completions/mean_terminated_length": 325.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16263089329004288, "epoch": 0.00342, "frac_reward_zero_std": 0.0, "grad_norm": 0.39488735795021057, "kl": 0.8337633665651083, "learning_rate": 9.558207372018519e-06, "loss": -0.0355, "num_tokens": 8090906.0, "reward": 1.3125, "reward_std": 1.0235769748687744, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.029797911643982, "sampling/importance_sampling_ratio/max": 2.1576592922210693, "sampling/importance_sampling_ratio/mean": 1.0379202365875244, "sampling/importance_sampling_ratio/min": 0.46330130100250244, "sampling/sampling_logp_difference/max": 0.7692649364471436, "sampling/sampling_logp_difference/mean": 0.03706314414739609, "step": 171, "step_time": 14.689460188019439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 348.5, "completions/mean_terminated_length": 348.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17883591097779572, "epoch": 0.00344, "frac_reward_zero_std": 0.0, "grad_norm": 0.6022679805755615, "kl": 0.8012932501733303, "learning_rate": 9.551772139652762e-06, "loss": -0.0541, "num_tokens": 8141715.0, "reward": 1.34375, "reward_std": 1.1684106588363647, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.124776005744934, "sampling/importance_sampling_ratio/max": 1.4092795848846436, "sampling/importance_sampling_ratio/mean": 1.0081783533096313, "sampling/importance_sampling_ratio/min": 0.4463333189487457, "sampling/sampling_logp_difference/max": 0.8932759761810303, "sampling/sampling_logp_difference/mean": 0.03543531894683838, "step": 172, "step_time": 15.827840037018177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 325.15625, "completions/mean_terminated_length": 325.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16459123720414937, "epoch": 0.00346, "frac_reward_zero_std": 0.0, "grad_norm": 0.5445632338523865, "kl": 1.4005258809775114, "learning_rate": 9.545293354680477e-06, "loss": -0.0631, "num_tokens": 8193011.0, "reward": 1.53125, "reward_std": 0.928504228591919, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.915260374546051, "sampling/importance_sampling_ratio/max": 1.171015977859497, "sampling/importance_sampling_ratio/mean": 0.9427416324615479, "sampling/importance_sampling_ratio/min": 0.26780256628990173, "sampling/sampling_logp_difference/max": 1.2086211442947388, "sampling/sampling_logp_difference/mean": 0.043226733803749084, "step": 173, "step_time": 13.369890084009967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 209.84375, "completions/mean_terminated_length": 209.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2901621786877513, "epoch": 0.00348, "frac_reward_zero_std": 0.0, "grad_norm": 0.634759247303009, "kl": 1.9732017610222101, "learning_rate": 9.538771102561212e-06, "loss": -0.05, "num_tokens": 8238472.0, "reward": 0.875, "reward_std": 1.0588748455047607, "rewards/rollout_reward_func/mean": 0.875, "rewards/rollout_reward_func/std": 1.1570261716842651, "sampling/importance_sampling_ratio/max": 2.4032132625579834, "sampling/importance_sampling_ratio/mean": 0.8409847617149353, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.698652744293213, "sampling/sampling_logp_difference/mean": 0.13480344414710999, "step": 174, "step_time": 13.575253375005559 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 244.65625, "completions/mean_terminated_length": 244.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18622918124310672, "epoch": 0.0035, "frac_reward_zero_std": 0.0, "grad_norm": 0.2955858111381531, "kl": 1.1517218388617039, "learning_rate": 9.53220546932789e-06, "loss": -0.0547, "num_tokens": 8285608.0, "reward": 1.375, "reward_std": 0.8210572600364685, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.8327955007553101, "sampling/importance_sampling_ratio/max": 1.2845704555511475, "sampling/importance_sampling_ratio/mean": 0.9329302310943604, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2642581462860107, "sampling/sampling_logp_difference/mean": 0.05201810225844383, "step": 175, "step_time": 13.867819872990367 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 330.375, "completions/mean_terminated_length": 330.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21559877949766815, "epoch": 0.00352, "frac_reward_zero_std": 0.0, "grad_norm": 0.3361128270626068, "kl": 0.8025546362623572, "learning_rate": 9.525596541585646e-06, "loss": -0.0796, "num_tokens": 8336368.0, "reward": 1.4375, "reward_std": 1.127453088760376, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.1341474056243896, "sampling/importance_sampling_ratio/max": 1.2396892309188843, "sampling/importance_sampling_ratio/mean": 0.9526604413986206, "sampling/importance_sampling_ratio/min": 0.3235023021697998, "sampling/sampling_logp_difference/max": 1.0909478664398193, "sampling/sampling_logp_difference/mean": 0.04306083172559738, "step": 176, "step_time": 14.974545671997475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 227.6875, "completions/mean_terminated_length": 227.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28677472611889243, "epoch": 0.00354, "frac_reward_zero_std": 0.0, "grad_norm": 0.5376206636428833, "kl": 1.111189511604607, "learning_rate": 9.518944406510707e-06, "loss": -0.0519, "num_tokens": 8383901.0, "reward": 1.0625, "reward_std": 1.1343591213226318, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.1053389310836792, "sampling/importance_sampling_ratio/max": 1.2153719663619995, "sampling/importance_sampling_ratio/mean": 0.9033600687980652, "sampling/importance_sampling_ratio/min": 0.2580716609954834, "sampling/sampling_logp_difference/max": 1.3540840148925781, "sampling/sampling_logp_difference/mean": 0.06594488024711609, "step": 177, "step_time": 13.595258825997007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 357.21875, "completions/mean_terminated_length": 357.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19714849046431482, "epoch": 0.00356, "frac_reward_zero_std": 0.0, "grad_norm": 0.6272668242454529, "kl": 1.3984820172190666, "learning_rate": 9.512249151849229e-06, "loss": -0.0283, "num_tokens": 8435940.0, "reward": 1.21875, "reward_std": 1.1477062702178955, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.1841527223587036, "sampling/importance_sampling_ratio/max": 2.069371461868286, "sampling/importance_sampling_ratio/mean": 1.0621325969696045, "sampling/importance_sampling_ratio/min": 0.2917844355106354, "sampling/sampling_logp_difference/max": 1.2315311431884766, "sampling/sampling_logp_difference/mean": 0.056088805198669434, "step": 178, "step_time": 13.663673763003317 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1364.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 358.90625, "completions/mean_terminated_length": 358.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2198416767641902, "epoch": 0.00358, "frac_reward_zero_std": 0.0, "grad_norm": 0.4518677890300751, "kl": 1.7871792390942574, "learning_rate": 9.50551086591615e-06, "loss": -0.0969, "num_tokens": 8487150.0, "reward": 1.28125, "reward_std": 1.335723876953125, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.3966405391693115, "sampling/importance_sampling_ratio/max": 1.165761947631836, "sampling/importance_sampling_ratio/mean": 0.8715669512748718, "sampling/importance_sampling_ratio/min": 0.14450405538082123, "sampling/sampling_logp_difference/max": 1.4028983116149902, "sampling/sampling_logp_difference/mean": 0.0840442031621933, "step": 179, "step_time": 14.666652194020571 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1373.0, "completions/max_terminated_length": 1373.0, "completions/mean_length": 388.0, "completions/mean_terminated_length": 388.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21336916647851467, "epoch": 0.0036, "frac_reward_zero_std": 0.0, "grad_norm": 0.46947985887527466, "kl": 1.3935684487223625, "learning_rate": 9.498729637594016e-06, "loss": -0.0507, "num_tokens": 8540364.0, "reward": 1.1875, "reward_std": 1.358760118484497, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.3304740190505981, "sampling/importance_sampling_ratio/max": 2.30511212348938, "sampling/importance_sampling_ratio/mean": 1.0098342895507812, "sampling/importance_sampling_ratio/min": 0.24988484382629395, "sampling/sampling_logp_difference/max": 1.3861403465270996, "sampling/sampling_logp_difference/mean": 0.07781489193439484, "step": 180, "step_time": 15.219415931991534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1778.0, "completions/max_terminated_length": 1778.0, "completions/mean_length": 382.25, "completions/mean_terminated_length": 382.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21645626588724554, "epoch": 0.00362, "frac_reward_zero_std": 0.0, "grad_norm": 0.6303900480270386, "kl": 1.5697004608809948, "learning_rate": 9.491905556331809e-06, "loss": -0.0871, "num_tokens": 8593217.0, "reward": 1.21875, "reward_std": 1.32280695438385, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.4081416130065918, "sampling/importance_sampling_ratio/max": 1.189756989479065, "sampling/importance_sampling_ratio/mean": 0.859198808670044, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.242924690246582, "sampling/sampling_logp_difference/mean": 0.08271309733390808, "step": 181, "step_time": 16.7498509559955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 348.59375, "completions/mean_terminated_length": 348.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14983997703529894, "epoch": 0.00364, "frac_reward_zero_std": 0.0, "grad_norm": 0.2739172875881195, "kl": 1.038878995925188, "learning_rate": 9.485038712143778e-06, "loss": -0.0853, "num_tokens": 8645412.0, "reward": 1.5, "reward_std": 1.1307810544967651, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.1639753580093384, "sampling/importance_sampling_ratio/max": 1.17661714553833, "sampling/importance_sampling_ratio/mean": 0.9200025796890259, "sampling/importance_sampling_ratio/min": 0.20930854976177216, "sampling/sampling_logp_difference/max": 1.5627751350402832, "sampling/sampling_logp_difference/mean": 0.06680238246917725, "step": 182, "step_time": 13.419711348979035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 368.15625, "completions/mean_terminated_length": 368.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20172960357740521, "epoch": 0.00366, "frac_reward_zero_std": 0.0, "grad_norm": 1.1866910457611084, "kl": 6.640966213308275, "learning_rate": 9.478129195608238e-06, "loss": -0.0698, "num_tokens": 8698589.0, "reward": 1.15625, "reward_std": 1.2820370197296143, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.3704102039337158, "sampling/importance_sampling_ratio/max": 2.745638847351074, "sampling/importance_sampling_ratio/mean": 0.8952616453170776, "sampling/importance_sampling_ratio/min": 0.21626749634742737, "sampling/sampling_logp_difference/max": 1.529374122619629, "sampling/sampling_logp_difference/mean": 0.09814208745956421, "step": 183, "step_time": 16.38018704400747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1418.0, "completions/max_terminated_length": 1418.0, "completions/mean_length": 371.28125, "completions/mean_terminated_length": 371.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20890808710828424, "epoch": 0.00368, "frac_reward_zero_std": 0.0, "grad_norm": 1.2130542993545532, "kl": 1.0990847758948803, "learning_rate": 9.471177097866384e-06, "loss": -0.056, "num_tokens": 8750177.0, "reward": 1.15625, "reward_std": 1.0629606246948242, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.2471742630004883, "sampling/importance_sampling_ratio/max": 2.6095666885375977, "sampling/importance_sampling_ratio/mean": 1.0586988925933838, "sampling/importance_sampling_ratio/min": 0.2881774306297302, "sampling/sampling_logp_difference/max": 1.2618036270141602, "sampling/sampling_logp_difference/mean": 0.08401171863079071, "step": 184, "step_time": 15.321523061982589 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 290.0625, "completions/mean_terminated_length": 290.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2119570472277701, "epoch": 0.0037, "frac_reward_zero_std": 0.0, "grad_norm": 0.26172953844070435, "kl": 2.0389450872316957, "learning_rate": 9.464182510621085e-06, "loss": -0.0577, "num_tokens": 8798548.0, "reward": 1.21875, "reward_std": 1.0301777124404907, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.0390558242797852, "sampling/importance_sampling_ratio/max": 1.524176836013794, "sampling/importance_sampling_ratio/mean": 0.8776639103889465, "sampling/importance_sampling_ratio/min": 0.21692171692848206, "sampling/sampling_logp_difference/max": 1.528080940246582, "sampling/sampling_logp_difference/mean": 0.08688952773809433, "step": 185, "step_time": 15.347461892029969 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 369.28125, "completions/mean_terminated_length": 369.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14123649359680712, "epoch": 0.00372, "frac_reward_zero_std": 0.0, "grad_norm": 0.5554986596107483, "kl": 1.5217814035713673, "learning_rate": 9.457145526135673e-06, "loss": -0.0685, "num_tokens": 8851650.0, "reward": 1.53125, "reward_std": 0.962677001953125, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.9832262396812439, "sampling/importance_sampling_ratio/max": 1.1905300617218018, "sampling/importance_sampling_ratio/mean": 0.9592228531837463, "sampling/importance_sampling_ratio/min": 0.225468248128891, "sampling/sampling_logp_difference/max": 1.4879398345947266, "sampling/sampling_logp_difference/mean": 0.04089045524597168, "step": 186, "step_time": 14.308695178013295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "completions/clipped_ratio": 0.03125, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 169.84375, "completions/mean_terminated_length": 174.8064422607422, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31416424037888646, "epoch": 0.00374, "frac_reward_zero_std": 0.0, "grad_norm": 1.2469221353530884, "kl": 1.2589727696031332, "learning_rate": 9.450066237232736e-06, "loss": 0.0152, "num_tokens": 8894264.0, "reward": 0.9375, "reward_std": 1.1131513118743896, "rewards/rollout_reward_func/mean": 0.9375, "rewards/rollout_reward_func/std": 1.1622419357299805, "sampling/importance_sampling_ratio/max": 2.0308170318603516, "sampling/importance_sampling_ratio/mean": 0.905879020690918, "sampling/importance_sampling_ratio/min": 0.16853249073028564, "sampling/sampling_logp_difference/max": 1.784872055053711, "sampling/sampling_logp_difference/mean": 0.09794968366622925, "step": 187, "step_time": 13.852959524010657 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 288.25, "completions/mean_terminated_length": 288.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17494605109095573, "epoch": 0.00376, "frac_reward_zero_std": 0.0, "grad_norm": 0.48613932728767395, "kl": 1.2370421402156353, "learning_rate": 9.442944737292872e-06, "loss": -0.025, "num_tokens": 8942659.0, "reward": 1.25, "reward_std": 1.0928211212158203, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.1359236240386963, "sampling/importance_sampling_ratio/max": 1.9299782514572144, "sampling/importance_sampling_ratio/mean": 1.0136817693710327, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7644093036651611, "sampling/sampling_logp_difference/mean": 0.07459159940481186, "step": 188, "step_time": 13.986127013995429 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 273.0625, "completions/mean_terminated_length": 273.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16757149691693485, "epoch": 0.00378, "frac_reward_zero_std": 0.0, "grad_norm": 0.9083592891693115, "kl": 1.8984748758375645, "learning_rate": 9.435781120253486e-06, "loss": -0.0197, "num_tokens": 8991294.0, "reward": 1.1875, "reward_std": 1.025475263595581, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.203154444694519, "sampling/importance_sampling_ratio/max": 2.6983587741851807, "sampling/importance_sampling_ratio/mean": 1.0763177871704102, "sampling/importance_sampling_ratio/min": 0.26909682154655457, "sampling/sampling_logp_difference/max": 1.319843053817749, "sampling/sampling_logp_difference/mean": 0.07463992387056351, "step": 189, "step_time": 13.81435803398199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 344.0, "completions/mean_terminated_length": 344.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1240505794994533, "epoch": 0.0038, "frac_reward_zero_std": 0.0, "grad_norm": 0.3583745062351227, "kl": 3.567189732566476, "learning_rate": 9.428575480607526e-06, "loss": -0.0377, "num_tokens": 9041869.0, "reward": 1.40625, "reward_std": 0.9874311685562134, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.011526346206665, "sampling/importance_sampling_ratio/max": 2.354126453399658, "sampling/importance_sampling_ratio/mean": 0.9918596744537354, "sampling/importance_sampling_ratio/min": 0.19098372757434845, "sampling/sampling_logp_difference/max": 1.710942268371582, "sampling/sampling_logp_difference/mean": 0.05817093700170517, "step": 190, "step_time": 16.188331550030853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1697.0, "completions/max_terminated_length": 1697.0, "completions/mean_length": 418.5, "completions/mean_terminated_length": 418.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14863411127589643, "epoch": 0.00382, "frac_reward_zero_std": 0.0, "grad_norm": 0.3843335509300232, "kl": 1.1274133808910847, "learning_rate": 9.421327913402252e-06, "loss": -0.0834, "num_tokens": 9094809.0, "reward": 1.46875, "reward_std": 1.170583724975586, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.2177284955978394, "sampling/importance_sampling_ratio/max": 1.3721882104873657, "sampling/importance_sampling_ratio/mean": 1.0114108324050903, "sampling/importance_sampling_ratio/min": 0.5523188710212708, "sampling/sampling_logp_difference/max": 1.5775132179260254, "sampling/sampling_logp_difference/mean": 0.047100991010665894, "step": 191, "step_time": 16.463472430987167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 353.65625, "completions/mean_terminated_length": 353.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12143877241760492, "epoch": 0.00384, "frac_reward_zero_std": 0.0, "grad_norm": 0.4503401517868042, "kl": 0.6794926840811968, "learning_rate": 9.41403851423797e-06, "loss": -0.0515, "num_tokens": 9147233.0, "reward": 1.6875, "reward_std": 0.8174401521682739, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.8590129017829895, "sampling/importance_sampling_ratio/max": 1.2030808925628662, "sampling/importance_sampling_ratio/mean": 0.9834744930267334, "sampling/importance_sampling_ratio/min": 0.4303993284702301, "sampling/sampling_logp_difference/max": 0.8591508865356445, "sampling/sampling_logp_difference/mean": 0.026765502989292145, "step": 192, "step_time": 13.95657160798146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1373.0, "completions/max_terminated_length": 1373.0, "completions/mean_length": 345.96875, "completions/mean_terminated_length": 345.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0939671597443521, "epoch": 0.00386, "frac_reward_zero_std": 0.0, "grad_norm": 0.7040613889694214, "kl": 1.1681902557611465, "learning_rate": 9.406707379266792e-06, "loss": -0.0503, "num_tokens": 9198012.0, "reward": 1.59375, "reward_std": 0.9650580286979675, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.9791166186332703, "sampling/importance_sampling_ratio/max": 1.2319082021713257, "sampling/importance_sampling_ratio/mean": 0.9426047801971436, "sampling/importance_sampling_ratio/min": 0.2965182065963745, "sampling/sampling_logp_difference/max": 1.2308783531188965, "sampling/sampling_logp_difference/mean": 0.04657354578375816, "step": 193, "step_time": 14.690642724992358 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 424.09375, "completions/mean_terminated_length": 424.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13213365152478218, "epoch": 0.00388, "frac_reward_zero_std": 0.0, "grad_norm": 0.24588695168495178, "kl": 0.6708579147234559, "learning_rate": 9.399334605191332e-06, "loss": -0.0828, "num_tokens": 9252838.0, "reward": 1.65625, "reward_std": 1.1913070678710938, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 1.180742621421814, "sampling/importance_sampling_ratio/max": 1.2746598720550537, "sampling/importance_sampling_ratio/mean": 1.0355544090270996, "sampling/importance_sampling_ratio/min": 0.32298555970191956, "sampling/sampling_logp_difference/max": 1.137258529663086, "sampling/sampling_logp_difference/mean": 0.02456815168261528, "step": 194, "step_time": 14.422428131991182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 237.375, "completions/mean_terminated_length": 237.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08968915557488799, "epoch": 0.0039, "frac_reward_zero_std": 0.0, "grad_norm": 0.6155630946159363, "kl": 1.1509081721305847, "learning_rate": 9.391920289263475e-06, "loss": -0.053, "num_tokens": 9300785.0, "reward": 1.25, "reward_std": 1.0090751647949219, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.0472698211669922, "sampling/importance_sampling_ratio/max": 1.188950777053833, "sampling/importance_sampling_ratio/mean": 1.0020971298217773, "sampling/importance_sampling_ratio/min": 0.4002455472946167, "sampling/sampling_logp_difference/max": 0.9155163764953613, "sampling/sampling_logp_difference/mean": 0.023705115541815758, "step": 195, "step_time": 13.965831141002127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1571.0, "completions/max_terminated_length": 1571.0, "completions/mean_length": 308.625, "completions/mean_terminated_length": 308.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10004083684179932, "epoch": 0.00392, "frac_reward_zero_std": 0.0, "grad_norm": 0.5018308758735657, "kl": 1.0851574148982763, "learning_rate": 9.384464529283055e-06, "loss": -0.0917, "num_tokens": 9350607.0, "reward": 1.21875, "reward_std": 1.3713288307189941, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.313315749168396, "sampling/importance_sampling_ratio/max": 1.4730610847473145, "sampling/importance_sampling_ratio/mean": 1.0023605823516846, "sampling/importance_sampling_ratio/min": 0.15217050909996033, "sampling/sampling_logp_difference/max": 1.6793742179870605, "sampling/sampling_logp_difference/mean": 0.03612961247563362, "step": 196, "step_time": 15.709242348006228 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 226.6875, "completions/mean_terminated_length": 226.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12121704453602433, "epoch": 0.00394, "frac_reward_zero_std": 0.0, "grad_norm": 1.834710717201233, "kl": 7.423381371423602, "learning_rate": 9.376967423596588e-06, "loss": -0.0468, "num_tokens": 9397442.0, "reward": 1.0, "reward_std": 1.2082022428512573, "rewards/rollout_reward_func/mean": 1.0, "rewards/rollout_reward_func/std": 1.2443419694900513, "sampling/importance_sampling_ratio/max": 1.2366901636123657, "sampling/importance_sampling_ratio/mean": 0.9073233008384705, "sampling/importance_sampling_ratio/min": 0.19075310230255127, "sampling/sampling_logp_difference/max": 1.656846284866333, "sampling/sampling_logp_difference/mean": 0.0704764574766159, "step": 197, "step_time": 13.789084901014576 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 311.375, "completions/mean_terminated_length": 311.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07749416236765683, "epoch": 0.00396, "frac_reward_zero_std": 0.0, "grad_norm": 0.7313451766967773, "kl": 3.618083929643035, "learning_rate": 9.369429071095963e-06, "loss": -0.0579, "num_tokens": 9448280.0, "reward": 1.59375, "reward_std": 1.068298578262329, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.0734140872955322, "sampling/importance_sampling_ratio/max": 1.1243700981140137, "sampling/importance_sampling_ratio/mean": 1.0088170766830444, "sampling/importance_sampling_ratio/min": 0.20118875801563263, "sampling/sampling_logp_difference/max": 1.6035585403442383, "sampling/sampling_logp_difference/mean": 0.02696222811937332, "step": 198, "step_time": 13.814834578006412 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 273.90625, "completions/mean_terminated_length": 273.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12098254472948611, "epoch": 0.00398, "frac_reward_zero_std": 0.0, "grad_norm": 0.9888948798179626, "kl": 4.206395065411925, "learning_rate": 9.361849571217149e-06, "loss": -0.0467, "num_tokens": 9498111.0, "reward": 1.125, "reward_std": 1.0376826524734497, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 1.1570261716842651, "sampling/importance_sampling_ratio/max": 1.1641467809677124, "sampling/importance_sampling_ratio/mean": 0.9050825238227844, "sampling/importance_sampling_ratio/min": 0.20100919902324677, "sampling/sampling_logp_difference/max": 1.6043057441711426, "sampling/sampling_logp_difference/mean": 0.0621509775519371, "step": 199, "step_time": 14.291830296962871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 369.875, "completions/mean_terminated_length": 369.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0985232125967741, "epoch": 0.004, "frac_reward_zero_std": 0.0, "grad_norm": 0.20088058710098267, "kl": 1.8265731371939182, "learning_rate": 9.35422902393887e-06, "loss": -0.0679, "num_tokens": 9550956.0, "reward": 1.34375, "reward_std": 0.9399815797805786, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.0658745765686035, "sampling/importance_sampling_ratio/max": 1.3138608932495117, "sampling/importance_sampling_ratio/mean": 0.9718033671379089, "sampling/importance_sampling_ratio/min": 0.208550363779068, "sampling/sampling_logp_difference/max": 1.5675547122955322, "sampling/sampling_logp_difference/mean": 0.04647420346736908, "step": 200, "step_time": 14.064982684969436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 194.9375, "completions/mean_terminated_length": 194.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08526191441342235, "epoch": 0.00402, "frac_reward_zero_std": 0.0, "grad_norm": 0.6238107681274414, "kl": 2.1205563731491566, "learning_rate": 9.346567529781298e-06, "loss": -0.0541, "num_tokens": 9596055.0, "reward": 1.1875, "reward_std": 0.9414849281311035, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.029797911643982, "sampling/importance_sampling_ratio/max": 1.1659222841262817, "sampling/importance_sampling_ratio/mean": 0.9462637901306152, "sampling/importance_sampling_ratio/min": 0.2322554737329483, "sampling/sampling_logp_difference/max": 1.454620361328125, "sampling/sampling_logp_difference/mean": 0.05780951306223869, "step": 201, "step_time": 13.768107116004103 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 282.0625, "completions/mean_terminated_length": 282.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12259301287122071, "epoch": 0.00404, "frac_reward_zero_std": 0.0, "grad_norm": 0.42957884073257446, "kl": 1.5522825680673122, "learning_rate": 9.338865189804722e-06, "loss": -0.0535, "num_tokens": 9644811.0, "reward": 1.0625, "reward_std": 1.1004605293273926, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.1896733045578003, "sampling/importance_sampling_ratio/max": 1.1351240873336792, "sampling/importance_sampling_ratio/mean": 0.9428853392601013, "sampling/importance_sampling_ratio/min": 0.23325128853321075, "sampling/sampling_logp_difference/max": 1.455615520477295, "sampling/sampling_logp_difference/mean": 0.050469689071178436, "step": 202, "step_time": 13.919514591994812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 865.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 314.34375, "completions/mean_terminated_length": 314.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11463807220570743, "epoch": 0.00406, "frac_reward_zero_std": 0.0, "grad_norm": 0.5524126887321472, "kl": 1.769206816330552, "learning_rate": 9.33112210560821e-06, "loss": -0.0486, "num_tokens": 9695267.0, "reward": 1.3125, "reward_std": 0.9488139152526855, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.029797911643982, "sampling/importance_sampling_ratio/max": 1.6658360958099365, "sampling/importance_sampling_ratio/mean": 0.9899718761444092, "sampling/importance_sampling_ratio/min": 0.3015787899494171, "sampling/sampling_logp_difference/max": 1.1989150047302246, "sampling/sampling_logp_difference/mean": 0.051242996007204056, "step": 203, "step_time": 12.889491210051347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 296.9375, "completions/mean_terminated_length": 296.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13275096251163632, "epoch": 0.00408, "frac_reward_zero_std": 0.0, "grad_norm": 1.236741304397583, "kl": 2.725402519106865, "learning_rate": 9.323338379328278e-06, "loss": -0.0571, "num_tokens": 9745649.0, "reward": 1.25, "reward_std": 1.1341400146484375, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.1639753580093384, "sampling/importance_sampling_ratio/max": 1.6532357931137085, "sampling/importance_sampling_ratio/mean": 0.9787380695343018, "sampling/importance_sampling_ratio/min": 0.11171163618564606, "sampling/sampling_logp_difference/max": 1.4955406188964844, "sampling/sampling_logp_difference/mean": 0.060201436281204224, "step": 204, "step_time": 14.213767465989804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 501.78125, "completions/mean_terminated_length": 501.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14521465776488185, "epoch": 0.0041, "frac_reward_zero_std": 0.0, "grad_norm": 0.4737178087234497, "kl": 0.83409533649683, "learning_rate": 9.315514113637534e-06, "loss": -0.0807, "num_tokens": 9802165.0, "reward": 1.4375, "reward_std": 1.2100247144699097, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.1622419357299805, "sampling/importance_sampling_ratio/max": 1.4682997465133667, "sampling/importance_sampling_ratio/mean": 0.9961921572685242, "sampling/importance_sampling_ratio/min": 0.29500991106033325, "sampling/sampling_logp_difference/max": 1.1950554847717285, "sampling/sampling_logp_difference/mean": 0.05488754063844681, "step": 205, "step_time": 16.481237908024923 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 226.5625, "completions/mean_terminated_length": 226.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17072800686582923, "epoch": 0.00412, "frac_reward_zero_std": 0.0, "grad_norm": 0.9210678935050964, "kl": 0.8224727846682072, "learning_rate": 9.30764941174334e-06, "loss": -0.0585, "num_tokens": 9848299.0, "reward": 1.21875, "reward_std": 1.1542433500289917, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.156590461730957, "sampling/importance_sampling_ratio/max": 1.2415865659713745, "sampling/importance_sampling_ratio/mean": 1.018033504486084, "sampling/importance_sampling_ratio/min": 0.3816189169883728, "sampling/sampling_logp_difference/max": 0.9633865356445312, "sampling/sampling_logp_difference/mean": 0.0316489115357399, "step": 206, "step_time": 13.422212090023095 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 341.3125, "completions/mean_terminated_length": 341.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15200818376615644, "epoch": 0.00414, "frac_reward_zero_std": 0.0, "grad_norm": 0.9536416530609131, "kl": 2.741026673465967, "learning_rate": 9.299744377386424e-06, "loss": -0.0552, "num_tokens": 9899171.0, "reward": 1.4375, "reward_std": 0.995252251625061, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.0757592916488647, "sampling/importance_sampling_ratio/max": 1.216282844543457, "sampling/importance_sampling_ratio/mean": 0.9934506416320801, "sampling/importance_sampling_ratio/min": 0.32460159063339233, "sampling/sampling_logp_difference/max": 0.9950771331787109, "sampling/sampling_logp_difference/mean": 0.053372953087091446, "step": 207, "step_time": 14.17912077899382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1733.0, "completions/max_terminated_length": 1733.0, "completions/mean_length": 278.53125, "completions/mean_terminated_length": 278.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1537391748279333, "epoch": 0.00416, "frac_reward_zero_std": 0.0, "grad_norm": 0.7136819362640381, "kl": 1.818473320454359, "learning_rate": 9.291799114839533e-06, "loss": -0.0478, "num_tokens": 9948066.0, "reward": 1.09375, "reward_std": 1.2162957191467285, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.2536237239837646, "sampling/importance_sampling_ratio/max": 1.1999863386154175, "sampling/importance_sampling_ratio/mean": 0.9389309883117676, "sampling/importance_sampling_ratio/min": 0.28908127546310425, "sampling/sampling_logp_difference/max": 1.2424116134643555, "sampling/sampling_logp_difference/mean": 0.06222838535904884, "step": 208, "step_time": 16.470563351991586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 437.65625, "completions/mean_terminated_length": 437.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12265827576629817, "epoch": 0.00418, "frac_reward_zero_std": 0.0, "grad_norm": 0.41091543436050415, "kl": 0.80740911886096, "learning_rate": 9.283813728906054e-06, "loss": -0.0487, "num_tokens": 10003521.0, "reward": 1.625, "reward_std": 0.8263757228851318, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.8706690073013306, "sampling/importance_sampling_ratio/max": 2.164104700088501, "sampling/importance_sampling_ratio/mean": 1.0579164028167725, "sampling/importance_sampling_ratio/min": 0.40246373414993286, "sampling/sampling_logp_difference/max": 0.9437060356140137, "sampling/sampling_logp_difference/mean": 0.03730722516775131, "step": 209, "step_time": 15.811093774012988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 316.0625, "completions/mean_terminated_length": 316.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13205546652898192, "epoch": 0.0042, "frac_reward_zero_std": 0.0, "grad_norm": 0.28874728083610535, "kl": 1.0530955344438553, "learning_rate": 9.275788324918624e-06, "loss": -0.051, "num_tokens": 10054243.0, "reward": 1.25, "reward_std": 0.9286401867866516, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9158109426498413, "sampling/importance_sampling_ratio/max": 1.3430770635604858, "sampling/importance_sampling_ratio/mean": 0.9544137716293335, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2631142139434814, "sampling/sampling_logp_difference/mean": 0.0558340810239315, "step": 210, "step_time": 13.941580750019057 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 295.125, "completions/mean_terminated_length": 295.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12773664947599173, "epoch": 0.00422, "frac_reward_zero_std": 0.0, "grad_norm": 0.785184383392334, "kl": 6.5389632023870945, "learning_rate": 9.267723008737745e-06, "loss": -0.0456, "num_tokens": 10103786.0, "reward": 0.96875, "reward_std": 1.106730580329895, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 1.121185302734375, "sampling/importance_sampling_ratio/max": 1.2082732915878296, "sampling/importance_sampling_ratio/mean": 0.9459480047225952, "sampling/importance_sampling_ratio/min": 0.1990700215101242, "sampling/sampling_logp_difference/max": 1.6304454803466797, "sampling/sampling_logp_difference/mean": 0.059449974447488785, "step": 211, "step_time": 13.747419820021605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 306.90625, "completions/mean_terminated_length": 306.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1570441487710923, "epoch": 0.00424, "frac_reward_zero_std": 0.0, "grad_norm": 1.0162819623947144, "kl": 0.8846728354692459, "learning_rate": 9.259617886750395e-06, "loss": -0.0829, "num_tokens": 10152629.0, "reward": 1.21875, "reward_std": 1.0732471942901611, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.0993949174880981, "sampling/importance_sampling_ratio/max": 2.1286094188690186, "sampling/importance_sampling_ratio/mean": 1.0190753936767578, "sampling/importance_sampling_ratio/min": 0.3240217864513397, "sampling/sampling_logp_difference/max": 1.1054408550262451, "sampling/sampling_logp_difference/mean": 0.07046478986740112, "step": 212, "step_time": 16.66264008800499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 243.0, "completions/mean_terminated_length": 243.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21783826546743512, "epoch": 0.00426, "frac_reward_zero_std": 0.0, "grad_norm": 0.5038168430328369, "kl": 1.458826620830223, "learning_rate": 9.251473065868606e-06, "loss": -0.046, "num_tokens": 10199138.0, "reward": 1.1875, "reward_std": 1.170759677886963, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.1760376691818237, "sampling/importance_sampling_ratio/max": 2.0058465003967285, "sampling/importance_sampling_ratio/mean": 0.9360568523406982, "sampling/importance_sampling_ratio/min": 0.30380779504776, "sampling/sampling_logp_difference/max": 1.191335678100586, "sampling/sampling_logp_difference/mean": 0.07946042716503143, "step": 213, "step_time": 13.65441567900416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 245.9375, "completions/mean_terminated_length": 245.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.167520712595433, "epoch": 0.00428, "frac_reward_zero_std": 0.0, "grad_norm": 0.7499954700469971, "kl": 1.168584612198174, "learning_rate": 9.243288653528075e-06, "loss": -0.0316, "num_tokens": 10247318.0, "reward": 1.15625, "reward_std": 1.1509943008422852, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.1103435754776, "sampling/importance_sampling_ratio/max": 2.667958974838257, "sampling/importance_sampling_ratio/mean": 1.0183436870574951, "sampling/importance_sampling_ratio/min": 0.23992934823036194, "sampling/sampling_logp_difference/max": 1.427156686782837, "sampling/sampling_logp_difference/mean": 0.06245895475149155, "step": 214, "step_time": 13.911201023031026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 271.78125, "completions/mean_terminated_length": 271.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15265363547950983, "epoch": 0.0043, "frac_reward_zero_std": 0.0, "grad_norm": 0.8009637594223022, "kl": 1.7389877960085869, "learning_rate": 9.235064757686733e-06, "loss": -0.0593, "num_tokens": 10294430.0, "reward": 1.21875, "reward_std": 1.222548484802246, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.2110878229141235, "sampling/importance_sampling_ratio/max": 2.7131776809692383, "sampling/importance_sampling_ratio/mean": 1.0240558385849, "sampling/importance_sampling_ratio/min": 0.2494000494480133, "sampling/sampling_logp_difference/max": 1.3887858390808105, "sampling/sampling_logp_difference/mean": 0.0740998163819313, "step": 215, "step_time": 14.563489669031696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 336.09375, "completions/mean_terminated_length": 336.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2089499975554645, "epoch": 0.00432, "frac_reward_zero_std": 0.0, "grad_norm": 0.455205500125885, "kl": 0.9080703807994723, "learning_rate": 9.226801486823325e-06, "loss": -0.0656, "num_tokens": 10345916.0, "reward": 1.3125, "reward_std": 1.116783618927002, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.2296733856201172, "sampling/importance_sampling_ratio/max": 2.7321412563323975, "sampling/importance_sampling_ratio/mean": 0.9876856207847595, "sampling/importance_sampling_ratio/min": 0.3425862193107605, "sampling/sampling_logp_difference/max": 1.0711522102355957, "sampling/sampling_logp_difference/mean": 0.054715365171432495, "step": 216, "step_time": 15.20266906498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 311.53125, "completions/mean_terminated_length": 311.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12943901447579265, "epoch": 0.00434, "frac_reward_zero_std": 0.0, "grad_norm": 0.3051358163356781, "kl": 0.9466637168079615, "learning_rate": 9.21849894993598e-06, "loss": -0.0339, "num_tokens": 10395616.0, "reward": 1.5625, "reward_std": 0.7119823694229126, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.7156093716621399, "sampling/importance_sampling_ratio/max": 1.868273377418518, "sampling/importance_sampling_ratio/mean": 1.0446964502334595, "sampling/importance_sampling_ratio/min": 0.389697402715683, "sampling/sampling_logp_difference/max": 1.0564453601837158, "sampling/sampling_logp_difference/mean": 0.0355217382311821, "step": 217, "step_time": 15.822344379019341 }, { "clip_ratio/high_max": 0.0031250000465661287, "clip_ratio/high_mean": 0.0015625000232830644, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009375000023283064, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 322.65625, "completions/mean_terminated_length": 322.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18244640761986375, "epoch": 0.00436, "frac_reward_zero_std": 0.0, "grad_norm": 1.0080417394638062, "kl": 5.5743940733373165, "learning_rate": 9.21015725654077e-06, "loss": -0.0432, "num_tokens": 10446883.0, "reward": 1.1875, "reward_std": 1.2377774715423584, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.306003451347351, "sampling/importance_sampling_ratio/max": 1.943786859512329, "sampling/importance_sampling_ratio/mean": 0.929767370223999, "sampling/importance_sampling_ratio/min": 7.153630444088621e-13, "sampling/sampling_logp_difference/max": 16.834829330444336, "sampling/sampling_logp_difference/mean": 0.2580912709236145, "step": 218, "step_time": 14.540241378970677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 356.625, "completions/mean_terminated_length": 356.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11891352036036551, "epoch": 0.00438, "frac_reward_zero_std": 0.25, "grad_norm": 0.18802516162395477, "kl": 0.9030991364270449, "learning_rate": 9.201776516670274e-06, "loss": -0.0745, "num_tokens": 10499353.0, "reward": 1.71875, "reward_std": 0.9153753519058228, "rewards/rollout_reward_func/mean": 1.71875, "rewards/rollout_reward_func/std": 1.054464340209961, "sampling/importance_sampling_ratio/max": 1.6782267093658447, "sampling/importance_sampling_ratio/mean": 1.058341383934021, "sampling/importance_sampling_ratio/min": 0.32648342847824097, "sampling/sampling_logp_difference/max": 1.1194579601287842, "sampling/sampling_logp_difference/mean": 0.03352151811122894, "step": 219, "step_time": 15.303712084001745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 307.5, "completions/mean_terminated_length": 307.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14979984890669584, "epoch": 0.0044, "frac_reward_zero_std": 0.0, "grad_norm": 0.2713802754878998, "kl": 1.9086652603000402, "learning_rate": 9.193356840872111e-06, "loss": -0.0604, "num_tokens": 10549098.0, "reward": 1.03125, "reward_std": 1.0888326168060303, "rewards/rollout_reward_func/mean": 1.03125, "rewards/rollout_reward_func/std": 1.1773226261138916, "sampling/importance_sampling_ratio/max": 1.5724587440490723, "sampling/importance_sampling_ratio/mean": 0.9673080444335938, "sampling/importance_sampling_ratio/min": 0.3469741642475128, "sampling/sampling_logp_difference/max": 1.058229923248291, "sampling/sampling_logp_difference/mean": 0.04978657513856888, "step": 220, "step_time": 13.962533144978806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 395.40625, "completions/mean_terminated_length": 395.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13398305792361498, "epoch": 0.00442, "frac_reward_zero_std": 0.0, "grad_norm": 0.3885468542575836, "kl": 0.9203026611357927, "learning_rate": 9.184898340207502e-06, "loss": -0.0643, "num_tokens": 10603957.0, "reward": 1.53125, "reward_std": 1.0038725137710571, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 1.1635422706604004, "sampling/importance_sampling_ratio/max": 1.277575135231018, "sampling/importance_sampling_ratio/mean": 1.005955457687378, "sampling/importance_sampling_ratio/min": 0.115440733730793, "sampling/sampling_logp_difference/max": 2.1389541625976562, "sampling/sampling_logp_difference/mean": 0.03397507220506668, "step": 221, "step_time": 15.820864659996005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 297.5, "completions/mean_terminated_length": 297.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13638456398621202, "epoch": 0.00444, "frac_reward_zero_std": 0.0, "grad_norm": 0.38714227080345154, "kl": 2.0188424605876207, "learning_rate": 9.176401126249785e-06, "loss": -0.056, "num_tokens": 10652562.0, "reward": 1.1875, "reward_std": 1.1253458261489868, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.1482806205749512, "sampling/importance_sampling_ratio/max": 1.173561453819275, "sampling/importance_sampling_ratio/mean": 0.9820998907089233, "sampling/importance_sampling_ratio/min": 0.2977581024169922, "sampling/sampling_logp_difference/max": 1.245694875717163, "sampling/sampling_logp_difference/mean": 0.040058527141809464, "step": 222, "step_time": 13.832155344018247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 353.53125, "completions/mean_terminated_length": 353.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1078952681273222, "epoch": 0.00446, "frac_reward_zero_std": 0.0, "grad_norm": 0.35435837507247925, "kl": 0.8302435595542192, "learning_rate": 9.167865311082957e-06, "loss": -0.088, "num_tokens": 10704205.0, "reward": 1.65625, "reward_std": 0.8925373554229736, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.9019467830657959, "sampling/importance_sampling_ratio/max": 1.4884603023529053, "sampling/importance_sampling_ratio/mean": 1.0302798748016357, "sampling/importance_sampling_ratio/min": 0.6292629241943359, "sampling/sampling_logp_difference/max": 0.47994375228881836, "sampling/sampling_logp_difference/mean": 0.02155309170484543, "step": 223, "step_time": 16.506084503998864 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 217.3125, "completions/mean_terminated_length": 217.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12212947476655245, "epoch": 0.00448, "frac_reward_zero_std": 0.0, "grad_norm": 0.3618519604206085, "kl": 1.160217946395278, "learning_rate": 9.159291007300193e-06, "loss": -0.054, "num_tokens": 10750873.0, "reward": 1.40625, "reward_std": 0.9594908952713013, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9455967545509338, "sampling/importance_sampling_ratio/max": 1.1139588356018066, "sampling/importance_sampling_ratio/mean": 0.9892075061798096, "sampling/importance_sampling_ratio/min": 0.3445911407470703, "sampling/sampling_logp_difference/max": 1.0650396347045898, "sampling/sampling_logp_difference/mean": 0.02620631270110607, "step": 224, "step_time": 13.608502101022168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 280.53125, "completions/mean_terminated_length": 280.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10736913466826081, "epoch": 0.0045, "frac_reward_zero_std": 0.0, "grad_norm": 0.9219820499420166, "kl": 1.6071884483098984, "learning_rate": 9.150678328002352e-06, "loss": -0.0297, "num_tokens": 10798121.0, "reward": 1.28125, "reward_std": 1.0091667175292969, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.054464340209961, "sampling/importance_sampling_ratio/max": 2.437861204147339, "sampling/importance_sampling_ratio/mean": 1.1084327697753906, "sampling/importance_sampling_ratio/min": 0.36983245611190796, "sampling/sampling_logp_difference/max": 0.994239330291748, "sampling/sampling_logp_difference/mean": 0.040332987904548645, "step": 225, "step_time": 14.658665361974272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 315.78125, "completions/mean_terminated_length": 315.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1602309881709516, "epoch": 0.00452, "frac_reward_zero_std": 0.0, "grad_norm": 0.7294307947158813, "kl": 1.8528760708868504, "learning_rate": 9.1420273867965e-06, "loss": -0.0198, "num_tokens": 10848706.0, "reward": 1.1875, "reward_std": 0.9568804502487183, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 0.9651173949241638, "sampling/importance_sampling_ratio/max": 1.3804875612258911, "sampling/importance_sampling_ratio/mean": 0.980442464351654, "sampling/importance_sampling_ratio/min": 0.412701278924942, "sampling/sampling_logp_difference/max": 0.8849191665649414, "sampling/sampling_logp_difference/mean": 0.03309733420610428, "step": 226, "step_time": 13.677910253041773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 325.59375, "completions/mean_terminated_length": 325.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08814451866783202, "epoch": 0.00454, "frac_reward_zero_std": 0.0, "grad_norm": 0.35893702507019043, "kl": 1.54486595839262, "learning_rate": 9.133338297794396e-06, "loss": -0.0686, "num_tokens": 10899373.0, "reward": 1.5625, "reward_std": 0.9061473608016968, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.9482581615447998, "sampling/importance_sampling_ratio/max": 1.2629725933074951, "sampling/importance_sampling_ratio/mean": 1.0082244873046875, "sampling/importance_sampling_ratio/min": 0.23604166507720947, "sampling/sampling_logp_difference/max": 1.4705650806427002, "sampling/sampling_logp_difference/mean": 0.02582031674683094, "step": 227, "step_time": 15.767921669001225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 249.1875, "completions/mean_terminated_length": 249.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10846487991511822, "epoch": 0.00456, "frac_reward_zero_std": 0.0, "grad_norm": 0.9227733612060547, "kl": 1.8897805530577898, "learning_rate": 9.124611175610998e-06, "loss": -0.0482, "num_tokens": 10946869.0, "reward": 1.25, "reward_std": 1.090569257736206, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.1071614027023315, "sampling/importance_sampling_ratio/max": 1.178030252456665, "sampling/importance_sampling_ratio/mean": 1.0056136846542358, "sampling/importance_sampling_ratio/min": 0.16320613026618958, "sampling/sampling_logp_difference/max": 1.8628087043762207, "sampling/sampling_logp_difference/mean": 0.028773246333003044, "step": 228, "step_time": 13.756418410979677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 285.6875, "completions/mean_terminated_length": 285.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08085144916549325, "epoch": 0.00458, "frac_reward_zero_std": 0.0, "grad_norm": 0.22736281156539917, "kl": 0.8589506074786186, "learning_rate": 9.11584613536295e-06, "loss": -0.0628, "num_tokens": 10995873.0, "reward": 1.6875, "reward_std": 0.6842414736747742, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.692703902721405, "sampling/importance_sampling_ratio/max": 1.1523716449737549, "sampling/importance_sampling_ratio/mean": 1.0283491611480713, "sampling/importance_sampling_ratio/min": 0.913676381111145, "sampling/sampling_logp_difference/max": 0.13423918187618256, "sampling/sampling_logp_difference/mean": 0.01036637369543314, "step": 229, "step_time": 13.50209678101237 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 195.8125, "completions/mean_terminated_length": 195.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1318053442519158, "epoch": 0.0046, "frac_reward_zero_std": 0.0, "grad_norm": 0.9795252084732056, "kl": 0.6749920472502708, "learning_rate": 9.107043292667056e-06, "loss": -0.0513, "num_tokens": 11041990.0, "reward": 1.1875, "reward_std": 0.9632464647293091, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 0.9651173949241638, "sampling/importance_sampling_ratio/max": 1.304122805595398, "sampling/importance_sampling_ratio/mean": 1.016139268875122, "sampling/importance_sampling_ratio/min": 0.5854611396789551, "sampling/sampling_logp_difference/max": 0.5428088903427124, "sampling/sampling_logp_difference/mean": 0.018098121508955956, "step": 230, "step_time": 12.711955984006636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 245.0, "completions/mean_terminated_length": 245.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12414828524924815, "epoch": 0.00462, "frac_reward_zero_std": 0.0, "grad_norm": 0.46813881397247314, "kl": 2.6091012582182884, "learning_rate": 9.098202763638763e-06, "loss": -0.0554, "num_tokens": 11088212.0, "reward": 1.125, "reward_std": 1.0000264644622803, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 1.2115039825439453, "sampling/importance_sampling_ratio/max": 1.201664924621582, "sampling/importance_sampling_ratio/mean": 0.9831115007400513, "sampling/importance_sampling_ratio/min": 0.3047317862510681, "sampling/sampling_logp_difference/max": 1.1881709098815918, "sampling/sampling_logp_difference/mean": 0.03477189317345619, "step": 231, "step_time": 15.295584842999233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 393.1875, "completions/mean_terminated_length": 393.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09935805667191744, "epoch": 0.00464, "frac_reward_zero_std": 0.0, "grad_norm": 0.3952164649963379, "kl": 0.6702430881559849, "learning_rate": 9.089324664890625e-06, "loss": -0.0603, "num_tokens": 11141703.0, "reward": 1.40625, "reward_std": 1.0347082614898682, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.18755304813385, "sampling/importance_sampling_ratio/max": 1.4198819398880005, "sampling/importance_sampling_ratio/mean": 1.0305678844451904, "sampling/importance_sampling_ratio/min": 0.6832073330879211, "sampling/sampling_logp_difference/max": 0.3934903144836426, "sampling/sampling_logp_difference/mean": 0.01985514909029007, "step": 232, "step_time": 14.442392668017419 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1301.0, "completions/max_terminated_length": 1301.0, "completions/mean_length": 317.875, "completions/mean_terminated_length": 317.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12518491735681891, "epoch": 0.00466, "frac_reward_zero_std": 0.0, "grad_norm": 0.19884949922561646, "kl": 0.9374106507748365, "learning_rate": 9.080409113530767e-06, "loss": -0.0708, "num_tokens": 11192121.0, "reward": 1.28125, "reward_std": 1.1395549774169922, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.1139692068099976, "sampling/importance_sampling_ratio/max": 1.2062041759490967, "sampling/importance_sampling_ratio/mean": 0.943638801574707, "sampling/importance_sampling_ratio/min": 0.1204816922545433, "sampling/sampling_logp_difference/max": 1.5903708934783936, "sampling/sampling_logp_difference/mean": 0.053379856050014496, "step": 233, "step_time": 14.68896863599366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1643.0, "completions/max_terminated_length": 1643.0, "completions/mean_length": 422.0, "completions/mean_terminated_length": 422.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09790804795920849, "epoch": 0.00468, "frac_reward_zero_std": 0.0, "grad_norm": 0.44189345836639404, "kl": 1.2199495527893305, "learning_rate": 9.071456227161335e-06, "loss": -0.0512, "num_tokens": 11245369.0, "reward": 1.4375, "reward_std": 0.9269287586212158, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9482581615447998, "sampling/importance_sampling_ratio/max": 1.5021381378173828, "sampling/importance_sampling_ratio/mean": 0.9923089742660522, "sampling/importance_sampling_ratio/min": 0.20358984172344208, "sampling/sampling_logp_difference/max": 1.181546926498413, "sampling/sampling_logp_difference/mean": 0.049188315868377686, "step": 234, "step_time": 16.73850041895639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 336.65625, "completions/mean_terminated_length": 336.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09968592715449631, "epoch": 0.0047, "frac_reward_zero_std": 0.0, "grad_norm": 0.35689425468444824, "kl": 0.7851853631436825, "learning_rate": 9.062466123876954e-06, "loss": -0.0519, "num_tokens": 11297046.0, "reward": 1.71875, "reward_std": 0.7787095904350281, "rewards/rollout_reward_func/mean": 1.71875, "rewards/rollout_reward_func/std": 0.8513509035110474, "sampling/importance_sampling_ratio/max": 1.4568523168563843, "sampling/importance_sampling_ratio/mean": 1.0267759561538696, "sampling/importance_sampling_ratio/min": 0.35137057304382324, "sampling/sampling_logp_difference/max": 1.0500309467315674, "sampling/sampling_logp_difference/mean": 0.02173691615462303, "step": 235, "step_time": 15.100108928018017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 267.40625, "completions/mean_terminated_length": 267.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08282738085836172, "epoch": 0.00472, "frac_reward_zero_std": 0.0, "grad_norm": 0.16739898920059204, "kl": 1.3927096519619226, "learning_rate": 9.053438922263165e-06, "loss": -0.0751, "num_tokens": 11346526.0, "reward": 1.5, "reward_std": 0.7223619222640991, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.7620007395744324, "sampling/importance_sampling_ratio/max": 1.0795366764068604, "sampling/importance_sampling_ratio/mean": 0.9565446376800537, "sampling/importance_sampling_ratio/min": 0.30289846658706665, "sampling/sampling_logp_difference/max": 1.1942193508148193, "sampling/sampling_logp_difference/mean": 0.035945825278759, "step": 236, "step_time": 15.324068523026654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1616.0, "completions/max_terminated_length": 1616.0, "completions/mean_length": 469.0625, "completions/mean_terminated_length": 469.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07527415850199759, "epoch": 0.00474, "frac_reward_zero_std": 0.0, "grad_norm": 0.641362190246582, "kl": 0.8421198539435863, "learning_rate": 9.044374741394859e-06, "loss": -0.0386, "num_tokens": 11401266.0, "reward": 1.59375, "reward_std": 1.000810146331787, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.011526346206665, "sampling/importance_sampling_ratio/max": 1.602855920791626, "sampling/importance_sampling_ratio/mean": 1.045947790145874, "sampling/importance_sampling_ratio/min": 0.35751307010650635, "sampling/sampling_logp_difference/max": 1.0319983959197998, "sampling/sampling_logp_difference/mean": 0.030123725533485413, "step": 237, "step_time": 16.64800083897717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 231.65625, "completions/mean_terminated_length": 231.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08141120569780469, "epoch": 0.00476, "frac_reward_zero_std": 0.0, "grad_norm": 0.6528656482696533, "kl": 3.0339746326208115, "learning_rate": 9.035273700834707e-06, "loss": -0.0407, "num_tokens": 11446673.0, "reward": 1.09375, "reward_std": 0.8054715991020203, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 0.892960786819458, "sampling/importance_sampling_ratio/max": 1.4031739234924316, "sampling/importance_sampling_ratio/mean": 0.9937105178833008, "sampling/importance_sampling_ratio/min": 0.21718305349349976, "sampling/sampling_logp_difference/max": 1.540844202041626, "sampling/sampling_logp_difference/mean": 0.052510447800159454, "step": 238, "step_time": 14.345329218020197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1364.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 281.375, "completions/mean_terminated_length": 281.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17301856400445104, "epoch": 0.00478, "frac_reward_zero_std": 0.0, "grad_norm": 0.44997572898864746, "kl": 0.6696545835584402, "learning_rate": 9.026135920631592e-06, "loss": -0.0306, "num_tokens": 11495274.0, "reward": 0.9375, "reward_std": 1.156878113746643, "rewards/rollout_reward_func/mean": 0.9375, "rewards/rollout_reward_func/std": 1.1622419357299805, "sampling/importance_sampling_ratio/max": 1.0971434116363525, "sampling/importance_sampling_ratio/mean": 0.9266229271888733, "sampling/importance_sampling_ratio/min": 0.26853132247924805, "sampling/sampling_logp_difference/max": 1.3309600353240967, "sampling/sampling_logp_difference/mean": 0.05814372003078461, "step": 239, "step_time": 14.68571520896512 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 283.59375, "completions/mean_terminated_length": 283.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08052669744938612, "epoch": 0.0048, "frac_reward_zero_std": 0.0, "grad_norm": 0.27526047825813293, "kl": 0.8641680143773556, "learning_rate": 9.016961521319006e-06, "loss": -0.0654, "num_tokens": 11543503.0, "reward": 1.40625, "reward_std": 0.9000314474105835, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9108441472053528, "sampling/importance_sampling_ratio/max": 1.17037034034729, "sampling/importance_sampling_ratio/mean": 0.9762148261070251, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1216402053833008, "sampling/sampling_logp_difference/mean": 0.034696221351623535, "step": 240, "step_time": 13.978047422962845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 320.0, "completions/mean_terminated_length": 320.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10164949158206582, "epoch": 0.00482, "frac_reward_zero_std": 0.0, "grad_norm": 0.16956749558448792, "kl": 3.1088293697685003, "learning_rate": 9.00775062391348e-06, "loss": -0.0885, "num_tokens": 11592980.0, "reward": 1.4375, "reward_std": 1.1936533451080322, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.1896733045578003, "sampling/importance_sampling_ratio/max": 1.309027910232544, "sampling/importance_sampling_ratio/mean": 0.9838318824768066, "sampling/importance_sampling_ratio/min": 0.2955789566040039, "sampling/sampling_logp_difference/max": 1.2186384201049805, "sampling/sampling_logp_difference/mean": 0.042127564549446106, "step": 241, "step_time": 15.637358254971332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 283.875, "completions/mean_terminated_length": 283.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09727927763015032, "epoch": 0.00484, "frac_reward_zero_std": 0.0, "grad_norm": 0.27604708075523376, "kl": 1.1980737559497356, "learning_rate": 8.998503349912977e-06, "loss": -0.0478, "num_tokens": 11642618.0, "reward": 1.34375, "reward_std": 0.8309892416000366, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.8654431700706482, "sampling/importance_sampling_ratio/max": 1.298056721687317, "sampling/importance_sampling_ratio/mean": 1.0077266693115234, "sampling/importance_sampling_ratio/min": 0.40673720836639404, "sampling/sampling_logp_difference/max": 0.931018590927124, "sampling/sampling_logp_difference/mean": 0.028905192390084267, "step": 242, "step_time": 14.068391421984416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 300.03125, "completions/mean_terminated_length": 300.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10046549909748137, "epoch": 0.00486, "frac_reward_zero_std": 0.0, "grad_norm": 0.3246161639690399, "kl": 0.7210921384394169, "learning_rate": 8.989219821295294e-06, "loss": -0.0515, "num_tokens": 11693431.0, "reward": 1.5, "reward_std": 0.8483866453170776, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9158109426498413, "sampling/importance_sampling_ratio/max": 1.3111836910247803, "sampling/importance_sampling_ratio/mean": 1.0107685327529907, "sampling/importance_sampling_ratio/min": 0.38557326793670654, "sampling/sampling_logp_difference/max": 0.957760214805603, "sampling/sampling_logp_difference/mean": 0.018550503998994827, "step": 243, "step_time": 13.397223755004234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 268.0, "completions/mean_terminated_length": 268.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09657835424877703, "epoch": 0.00488, "frac_reward_zero_std": 0.0, "grad_norm": 0.2661888003349304, "kl": 1.4952743640169501, "learning_rate": 8.979900160516447e-06, "loss": -0.0499, "num_tokens": 11741664.0, "reward": 1.3125, "reward_std": 0.8369045257568359, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.9651173949241638, "sampling/importance_sampling_ratio/max": 1.1815680265426636, "sampling/importance_sampling_ratio/mean": 0.9697273373603821, "sampling/importance_sampling_ratio/min": 0.23890982568264008, "sampling/sampling_logp_difference/max": 1.4317219257354736, "sampling/sampling_logp_difference/mean": 0.04442039877176285, "step": 244, "step_time": 14.283400792017346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 326.15625, "completions/mean_terminated_length": 326.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08566545974463224, "epoch": 0.0049, "frac_reward_zero_std": 0.0, "grad_norm": 0.40552911162376404, "kl": 1.364677470177412, "learning_rate": 8.970544490509064e-06, "loss": -0.0577, "num_tokens": 11792678.0, "reward": 1.5, "reward_std": 0.907220184803009, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9158109426498413, "sampling/importance_sampling_ratio/max": 1.0996776819229126, "sampling/importance_sampling_ratio/mean": 0.9792197942733765, "sampling/importance_sampling_ratio/min": 0.2350950986146927, "sampling/sampling_logp_difference/max": 1.4495353698730469, "sampling/sampling_logp_difference/mean": 0.03296591341495514, "step": 245, "step_time": 14.406372619021568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 324.1875, "completions/mean_terminated_length": 324.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11658650892786682, "epoch": 0.00492, "frac_reward_zero_std": 0.0, "grad_norm": 1.5987896919250488, "kl": 1.0284083765000105, "learning_rate": 8.961152934680752e-06, "loss": -0.0485, "num_tokens": 11843662.0, "reward": 1.4375, "reward_std": 0.9381392598152161, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.2235479354858398, "sampling/importance_sampling_ratio/mean": 0.9894866943359375, "sampling/importance_sampling_ratio/min": 0.40438345074653625, "sampling/sampling_logp_difference/max": 0.9140520095825195, "sampling/sampling_logp_difference/mean": 0.04270927608013153, "step": 246, "step_time": 14.013665430000401 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 312.25, "completions/mean_terminated_length": 312.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08117587841115892, "epoch": 0.00494, "frac_reward_zero_std": 0.0, "grad_norm": 0.8992307782173157, "kl": 4.020289860665798, "learning_rate": 8.951725616912489e-06, "loss": -0.0651, "num_tokens": 11892451.0, "reward": 1.46875, "reward_std": 0.9129766225814819, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.915260374546051, "sampling/importance_sampling_ratio/max": 1.1524819135665894, "sampling/importance_sampling_ratio/mean": 0.958878755569458, "sampling/importance_sampling_ratio/min": 0.2629934847354889, "sampling/sampling_logp_difference/max": 1.2334647178649902, "sampling/sampling_logp_difference/mean": 0.04298032820224762, "step": 247, "step_time": 13.607961174027878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 281.875, "completions/mean_terminated_length": 281.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07009027688764036, "epoch": 0.00496, "frac_reward_zero_std": 0.0, "grad_norm": 0.11988794058561325, "kl": 1.5854855217039585, "learning_rate": 8.942262661556962e-06, "loss": -0.0602, "num_tokens": 11941634.0, "reward": 1.65625, "reward_std": 0.7975403666496277, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.9019467830657959, "sampling/importance_sampling_ratio/max": 1.223867654800415, "sampling/importance_sampling_ratio/mean": 1.0277488231658936, "sampling/importance_sampling_ratio/min": 0.5376176238059998, "sampling/sampling_logp_difference/max": 0.6209621429443359, "sampling/sampling_logp_difference/mean": 0.016925830394029617, "step": 248, "step_time": 13.832406766960048 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 272.46875, "completions/mean_terminated_length": 272.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.138085946906358, "epoch": 0.00498, "frac_reward_zero_std": 0.0, "grad_norm": 0.29023855924606323, "kl": 1.384967366233468, "learning_rate": 8.932764193436953e-06, "loss": -0.0465, "num_tokens": 11990644.0, "reward": 1.3125, "reward_std": 0.8312106728553772, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.8957785964012146, "sampling/importance_sampling_ratio/max": 1.2947938442230225, "sampling/importance_sampling_ratio/mean": 0.9974703192710876, "sampling/importance_sampling_ratio/min": 0.21944311261177063, "sampling/sampling_logp_difference/max": 1.5503525733947754, "sampling/sampling_logp_difference/mean": 0.04200103133916855, "step": 249, "step_time": 14.025227193997125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 425.1875, "completions/mean_terminated_length": 425.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08014899794943631, "epoch": 0.005, "frac_reward_zero_std": 0.0, "grad_norm": 0.3079468309879303, "kl": 0.6718435920774937, "learning_rate": 8.923230337843675e-06, "loss": -0.0565, "num_tokens": 12045430.0, "reward": 1.65625, "reward_std": 0.9197962284088135, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 1.159955620765686, "sampling/importance_sampling_ratio/mean": 1.003018856048584, "sampling/importance_sampling_ratio/min": 0.4277935326099396, "sampling/sampling_logp_difference/max": 0.9589986801147461, "sampling/sampling_logp_difference/mean": 0.023097338154911995, "step": 250, "step_time": 13.633998063000035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 317.03125, "completions/mean_terminated_length": 317.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08184562507085502, "epoch": 0.00502, "frac_reward_zero_std": 0.0, "grad_norm": 1.1434968709945679, "kl": 3.4570523779839277, "learning_rate": 8.913661220535132e-06, "loss": -0.0681, "num_tokens": 12095242.0, "reward": 1.09375, "reward_std": 1.338620901107788, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.3285784721374512, "sampling/importance_sampling_ratio/max": 1.227484107017517, "sampling/importance_sampling_ratio/mean": 0.960623025894165, "sampling/importance_sampling_ratio/min": 0.3780626654624939, "sampling/sampling_logp_difference/max": 0.9859414100646973, "sampling/sampling_logp_difference/mean": 0.040249053388834, "step": 251, "step_time": 15.53401587002736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 300.875, "completions/mean_terminated_length": 300.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.102696014335379, "epoch": 0.00504, "frac_reward_zero_std": 0.0, "grad_norm": 0.3894365131855011, "kl": 0.7267021667212248, "learning_rate": 8.904056967734447e-06, "loss": -0.0385, "num_tokens": 12146005.0, "reward": 1.4375, "reward_std": 0.8735049366950989, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9136068224906921, "sampling/importance_sampling_ratio/max": 1.084001898765564, "sampling/importance_sampling_ratio/mean": 0.9878262877464294, "sampling/importance_sampling_ratio/min": 0.4560406804084778, "sampling/sampling_logp_difference/max": 0.7890901565551758, "sampling/sampling_logp_difference/mean": 0.017361722886562347, "step": 252, "step_time": 14.112461563025136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 220.53125, "completions/mean_terminated_length": 220.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11474219616502523, "epoch": 0.00506, "frac_reward_zero_std": 0.0, "grad_norm": 0.33744409680366516, "kl": 1.3751843348145485, "learning_rate": 8.894417706128209e-06, "loss": -0.0723, "num_tokens": 12191904.0, "reward": 1.25, "reward_std": 1.0945011377334595, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.1639753580093384, "sampling/importance_sampling_ratio/max": 1.2292331457138062, "sampling/importance_sampling_ratio/mean": 0.9733908176422119, "sampling/importance_sampling_ratio/min": 0.3953664004802704, "sampling/sampling_logp_difference/max": 0.9277384281158447, "sampling/sampling_logp_difference/mean": 0.03401141241192818, "step": 253, "step_time": 13.515727403981145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 263.375, "completions/mean_terminated_length": 263.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13292220630683005, "epoch": 0.00508, "frac_reward_zero_std": 0.0, "grad_norm": 0.5057293772697449, "kl": 0.8170092161744833, "learning_rate": 8.884743562864792e-06, "loss": -0.0635, "num_tokens": 12239381.0, "reward": 1.34375, "reward_std": 1.0311028957366943, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.035167932510376, "sampling/importance_sampling_ratio/max": 1.1423014402389526, "sampling/importance_sampling_ratio/mean": 0.9787428379058838, "sampling/importance_sampling_ratio/min": 0.23192819952964783, "sampling/sampling_logp_difference/max": 1.0426201820373535, "sampling/sampling_logp_difference/mean": 0.03661753237247467, "step": 254, "step_time": 13.993328160999226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 316.75, "completions/mean_terminated_length": 316.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0888344265986234, "epoch": 0.0051, "frac_reward_zero_std": 0.0, "grad_norm": 0.6003686785697937, "kl": 5.493161119520664, "learning_rate": 8.875034665552686e-06, "loss": -0.0673, "num_tokens": 12289831.0, "reward": 1.46875, "reward_std": 0.9737815260887146, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.0155048370361328, "sampling/importance_sampling_ratio/max": 1.0427733659744263, "sampling/importance_sampling_ratio/mean": 0.97618168592453, "sampling/importance_sampling_ratio/min": 0.3670729100704193, "sampling/sampling_logp_difference/max": 1.0021953582763672, "sampling/sampling_logp_difference/mean": 0.023256974294781685, "step": 255, "step_time": 14.913897287973668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0017361111240461469, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017361111240461469, "completions/clipped_ratio": 0.0, "completions/max_length": 1373.0, "completions/max_terminated_length": 1373.0, "completions/mean_length": 382.84375, "completions/mean_terminated_length": 382.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1566730767954141, "epoch": 0.00512, "frac_reward_zero_std": 0.0, "grad_norm": 0.8743329048156738, "kl": 1.1250209882855415, "learning_rate": 8.865291142258813e-06, "loss": -0.0179, "num_tokens": 12341645.0, "reward": 1.3125, "reward_std": 1.1674624681472778, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.1760376691818237, "sampling/importance_sampling_ratio/max": 1.2596169710159302, "sampling/importance_sampling_ratio/mean": 1.0020778179168701, "sampling/importance_sampling_ratio/min": 0.35746413469314575, "sampling/sampling_logp_difference/max": 1.0287585258483887, "sampling/sampling_logp_difference/mean": 0.03975918889045715, "step": 256, "step_time": 15.340565324004274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 395.28125, "completions/mean_terminated_length": 395.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0935573645401746, "epoch": 0.00514, "frac_reward_zero_std": 0.0, "grad_norm": 0.7670416831970215, "kl": 1.3798261815682054, "learning_rate": 8.855513121506826e-06, "loss": -0.0591, "num_tokens": 12396162.0, "reward": 1.4375, "reward_std": 1.0507385730743408, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.5893577337265015, "sampling/importance_sampling_ratio/mean": 0.9331083297729492, "sampling/importance_sampling_ratio/min": 0.2921406924724579, "sampling/sampling_logp_difference/max": 1.2376227378845215, "sampling/sampling_logp_difference/mean": 0.05640053376555443, "step": 257, "step_time": 13.963443431959604 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 254.78125, "completions/mean_terminated_length": 254.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10073775076307356, "epoch": 0.00516, "frac_reward_zero_std": 0.0, "grad_norm": 1.0867664813995361, "kl": 0.7578689604997635, "learning_rate": 8.845700732275435e-06, "loss": -0.048, "num_tokens": 12444913.0, "reward": 1.375, "reward_std": 1.0500544309616089, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.0701221227645874, "sampling/importance_sampling_ratio/max": 1.5138684511184692, "sampling/importance_sampling_ratio/mean": 1.0013521909713745, "sampling/importance_sampling_ratio/min": 0.5390868186950684, "sampling/sampling_logp_difference/max": 0.6179139614105225, "sampling/sampling_logp_difference/mean": 0.019726503640413284, "step": 258, "step_time": 13.602261229971191 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.020833333488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 292.71875, "completions/mean_terminated_length": 292.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1111083677969873, "epoch": 0.00518, "frac_reward_zero_std": 0.0, "grad_norm": 2.247680425643921, "kl": 11.660772928968072, "learning_rate": 8.835854103996683e-06, "loss": -0.0437, "num_tokens": 12494708.0, "reward": 1.25, "reward_std": 1.1511906385421753, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.1359236240386963, "sampling/importance_sampling_ratio/max": 1.106790542602539, "sampling/importance_sampling_ratio/mean": 0.9568185210227966, "sampling/importance_sampling_ratio/min": 0.3671298623085022, "sampling/sampling_logp_difference/max": 1.0017986297607422, "sampling/sampling_logp_difference/mean": 0.04740433394908905, "step": 259, "step_time": 12.9492457590095 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 387.8125, "completions/mean_terminated_length": 387.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12483911961317062, "epoch": 0.0052, "frac_reward_zero_std": 0.0, "grad_norm": 0.4033021628856659, "kl": 1.111620333045721, "learning_rate": 8.825973366554254e-06, "loss": -0.0681, "num_tokens": 12548658.0, "reward": 1.53125, "reward_std": 1.130218744277954, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 1.1909435987472534, "sampling/importance_sampling_ratio/max": 1.1906005144119263, "sampling/importance_sampling_ratio/mean": 0.9892475605010986, "sampling/importance_sampling_ratio/min": 0.23581169545650482, "sampling/sampling_logp_difference/max": 1.113175392150879, "sampling/sampling_logp_difference/mean": 0.022995106875896454, "step": 260, "step_time": 13.902674147975631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 360.03125, "completions/mean_terminated_length": 360.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09530675993300974, "epoch": 0.00522, "frac_reward_zero_std": 0.0, "grad_norm": 0.4638402462005615, "kl": 0.8403158504515886, "learning_rate": 8.816058650281758e-06, "loss": -0.053, "num_tokens": 12600481.0, "reward": 1.40625, "reward_std": 1.158005952835083, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.1319231986999512, "sampling/importance_sampling_ratio/max": 1.2076596021652222, "sampling/importance_sampling_ratio/mean": 1.0003247261047363, "sampling/importance_sampling_ratio/min": 0.40143489837646484, "sampling/sampling_logp_difference/max": 0.9637255668640137, "sampling/sampling_logp_difference/mean": 0.023256100714206696, "step": 261, "step_time": 15.92632226900605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 257.3125, "completions/mean_terminated_length": 257.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11201775004155934, "epoch": 0.00524, "frac_reward_zero_std": 0.0, "grad_norm": 0.29623347520828247, "kl": 2.727382617071271, "learning_rate": 8.806110085960999e-06, "loss": -0.0601, "num_tokens": 12648311.0, "reward": 1.375, "reward_std": 1.0849111080169678, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.0701221227645874, "sampling/importance_sampling_ratio/max": 1.3177125453948975, "sampling/importance_sampling_ratio/mean": 0.9916880130767822, "sampling/importance_sampling_ratio/min": 0.29077616333961487, "sampling/sampling_logp_difference/max": 1.2350788116455078, "sampling/sampling_logp_difference/mean": 0.02930108644068241, "step": 262, "step_time": 13.926361921985517 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 305.25, "completions/mean_terminated_length": 305.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13128690258599818, "epoch": 0.00526, "frac_reward_zero_std": 0.0, "grad_norm": 0.2938515543937683, "kl": 0.7680724579840899, "learning_rate": 8.79612780482027e-06, "loss": -0.072, "num_tokens": 12698182.0, "reward": 1.34375, "reward_std": 1.138615369796753, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.124776005744934, "sampling/importance_sampling_ratio/max": 1.1658339500427246, "sampling/importance_sampling_ratio/mean": 0.9460206627845764, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1331530809402466, "sampling/sampling_logp_difference/mean": 0.038839343935251236, "step": 263, "step_time": 13.764468119959929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 230.78125, "completions/mean_terminated_length": 230.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0992603269405663, "epoch": 0.00528, "frac_reward_zero_std": 0.0, "grad_norm": 0.40031886100769043, "kl": 1.3319956753402948, "learning_rate": 8.786111938532606e-06, "loss": -0.0336, "num_tokens": 12743989.0, "reward": 1.34375, "reward_std": 0.7919631004333496, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.8273305296897888, "sampling/importance_sampling_ratio/max": 1.3245327472686768, "sampling/importance_sampling_ratio/mean": 0.9718840718269348, "sampling/importance_sampling_ratio/min": 0.11272499710321426, "sampling/sampling_logp_difference/max": 1.7496297359466553, "sampling/sampling_logp_difference/mean": 0.043508995324373245, "step": 264, "step_time": 13.791843004990369 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01953125, "completions/clipped_ratio": 0.0, "completions/max_length": 1953.0, "completions/max_terminated_length": 1953.0, "completions/mean_length": 354.375, "completions/mean_terminated_length": 354.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08334721811115742, "epoch": 0.0053, "frac_reward_zero_std": 0.0, "grad_norm": 0.3095047175884247, "kl": 4.274423848837614, "learning_rate": 8.776062619214053e-06, "loss": -0.0873, "num_tokens": 12795061.0, "reward": 1.59375, "reward_std": 1.0233594179153442, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.0734140872955322, "sampling/importance_sampling_ratio/max": 1.1791836023330688, "sampling/importance_sampling_ratio/mean": 0.9324245452880859, "sampling/importance_sampling_ratio/min": 3.34911511856717e-08, "sampling/sampling_logp_difference/max": 17.274972915649414, "sampling/sampling_logp_difference/mean": 0.1650036722421646, "step": 265, "step_time": 17.22934354598692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 284.0, "completions/mean_terminated_length": 284.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09558197460137308, "epoch": 0.00532, "frac_reward_zero_std": 0.0, "grad_norm": 0.7579146027565002, "kl": 0.7370162792503834, "learning_rate": 8.765979979421928e-06, "loss": -0.0353, "num_tokens": 12844418.0, "reward": 1.5, "reward_std": 0.787649393081665, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.8032193183898926, "sampling/importance_sampling_ratio/max": 1.9182674884796143, "sampling/importance_sampling_ratio/mean": 1.047323226928711, "sampling/importance_sampling_ratio/min": 0.9931232929229736, "sampling/sampling_logp_difference/max": 0.6513676643371582, "sampling/sampling_logp_difference/mean": 0.011450281366705894, "step": 266, "step_time": 13.389409087016247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 276.0, "completions/mean_terminated_length": 276.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08452939731068909, "epoch": 0.00534, "frac_reward_zero_std": 0.0, "grad_norm": 0.3134991228580475, "kl": 2.9652703180909157, "learning_rate": 8.755864152153064e-06, "loss": -0.044, "num_tokens": 12893845.0, "reward": 1.34375, "reward_std": 0.9897522926330566, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.0035220384597778, "sampling/importance_sampling_ratio/max": 1.414560317993164, "sampling/importance_sampling_ratio/mean": 0.9890952110290527, "sampling/importance_sampling_ratio/min": 0.32738175988197327, "sampling/sampling_logp_difference/max": 1.1164069175720215, "sampling/sampling_logp_difference/mean": 0.024528127163648605, "step": 267, "step_time": 13.976216500988812 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 236.25, "completions/mean_terminated_length": 236.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11103677190840244, "epoch": 0.00536, "frac_reward_zero_std": 0.0, "grad_norm": 0.13963337242603302, "kl": 1.0295788384974003, "learning_rate": 8.745715270842064e-06, "loss": -0.042, "num_tokens": 12941477.0, "reward": 1.3125, "reward_std": 0.8371957540512085, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.8590129017829895, "sampling/importance_sampling_ratio/max": 1.0819278955459595, "sampling/importance_sampling_ratio/mean": 0.9608079791069031, "sampling/importance_sampling_ratio/min": 0.4596293270587921, "sampling/sampling_logp_difference/max": 0.8356952667236328, "sampling/sampling_logp_difference/mean": 0.03136633336544037, "step": 268, "step_time": 14.012741471000481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 279.09375, "completions/mean_terminated_length": 279.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09816933493129909, "epoch": 0.00538, "frac_reward_zero_std": 0.0, "grad_norm": 0.3283131420612335, "kl": 2.0866503212600946, "learning_rate": 8.73553346935953e-06, "loss": -0.0618, "num_tokens": 12990962.0, "reward": 1.53125, "reward_std": 0.8612684011459351, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.8793096542358398, "sampling/importance_sampling_ratio/max": 1.5369125604629517, "sampling/importance_sampling_ratio/mean": 0.979290246963501, "sampling/importance_sampling_ratio/min": 0.26374030113220215, "sampling/sampling_logp_difference/max": 1.3426828384399414, "sampling/sampling_logp_difference/mean": 0.037485718727111816, "step": 269, "step_time": 13.724543240023195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 290.90625, "completions/mean_terminated_length": 290.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13302519475109875, "epoch": 0.0054, "frac_reward_zero_std": 0.0, "grad_norm": 1.0483406782150269, "kl": 1.6224409695714712, "learning_rate": 8.725318882010307e-06, "loss": -0.047, "num_tokens": 13040468.0, "reward": 1.03125, "reward_std": 1.101474642753601, "rewards/rollout_reward_func/mean": 1.03125, "rewards/rollout_reward_func/std": 1.2044105529785156, "sampling/importance_sampling_ratio/max": 1.2741550207138062, "sampling/importance_sampling_ratio/mean": 0.9549925327301025, "sampling/importance_sampling_ratio/min": 0.4267982840538025, "sampling/sampling_logp_difference/max": 0.8588795065879822, "sampling/sampling_logp_difference/mean": 0.041925422847270966, "step": 270, "step_time": 13.427145226989524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 267.03125, "completions/mean_terminated_length": 267.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11823441227898002, "epoch": 0.00542, "frac_reward_zero_std": 0.0, "grad_norm": 0.33732616901397705, "kl": 0.943746991455555, "learning_rate": 8.715071643531708e-06, "loss": -0.04, "num_tokens": 13089873.0, "reward": 1.34375, "reward_std": 0.8249304294586182, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.8654431700706482, "sampling/importance_sampling_ratio/max": 1.3262381553649902, "sampling/importance_sampling_ratio/mean": 1.0056177377700806, "sampling/importance_sampling_ratio/min": 0.5322513580322266, "sampling/sampling_logp_difference/max": 0.6305403709411621, "sampling/sampling_logp_difference/mean": 0.019287914037704468, "step": 271, "step_time": 14.014621780021116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 347.78125, "completions/mean_terminated_length": 347.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10865119379013777, "epoch": 0.00544, "frac_reward_zero_std": 0.0, "grad_norm": 0.5838483572006226, "kl": 1.581650273874402, "learning_rate": 8.704791889091737e-06, "loss": -0.0379, "num_tokens": 13141830.0, "reward": 1.375, "reward_std": 1.0831727981567383, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.1570261716842651, "sampling/importance_sampling_ratio/max": 1.1811027526855469, "sampling/importance_sampling_ratio/mean": 0.9767369031906128, "sampling/importance_sampling_ratio/min": 0.1355292797088623, "sampling/sampling_logp_difference/max": 1.9775941371917725, "sampling/sampling_logp_difference/mean": 0.029678059741854668, "step": 272, "step_time": 15.4122490259615 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 270.96875, "completions/mean_terminated_length": 270.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09818936884403229, "epoch": 0.00546, "frac_reward_zero_std": 0.0, "grad_norm": 0.45708930492401123, "kl": 1.038133542984724, "learning_rate": 8.6944797542873e-06, "loss": -0.0652, "num_tokens": 13190880.0, "reward": 1.46875, "reward_std": 1.0001347064971924, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.0155048370361328, "sampling/importance_sampling_ratio/max": 1.1006351709365845, "sampling/importance_sampling_ratio/mean": 0.9790790677070618, "sampling/importance_sampling_ratio/min": 0.25271889567375183, "sampling/sampling_logp_difference/max": 1.3754684925079346, "sampling/sampling_logp_difference/mean": 0.024490296840667725, "step": 273, "step_time": 14.204273935989477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 287.375, "completions/mean_terminated_length": 287.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10281372210010886, "epoch": 0.00548, "frac_reward_zero_std": 0.0, "grad_norm": 0.3445265293121338, "kl": 2.314538447186351, "learning_rate": 8.68413537514243e-06, "loss": -0.0539, "num_tokens": 13239644.0, "reward": 1.375, "reward_std": 0.9862726330757141, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.9755064845085144, "sampling/importance_sampling_ratio/max": 1.1258896589279175, "sampling/importance_sampling_ratio/mean": 0.9603457450866699, "sampling/importance_sampling_ratio/min": 0.37237346172332764, "sampling/sampling_logp_difference/max": 0.9878091812133789, "sampling/sampling_logp_difference/mean": 0.033477894961833954, "step": 274, "step_time": 12.466059407001012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 219.40625, "completions/mean_terminated_length": 219.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1237832298502326, "epoch": 0.0055, "frac_reward_zero_std": 0.0, "grad_norm": 0.5319567918777466, "kl": 2.2323594223707914, "learning_rate": 8.673758888106481e-06, "loss": -0.0474, "num_tokens": 13286358.0, "reward": 1.125, "reward_std": 1.0032790899276733, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 1.0395407676696777, "sampling/importance_sampling_ratio/max": 1.413484811782837, "sampling/importance_sampling_ratio/mean": 0.9589335918426514, "sampling/importance_sampling_ratio/min": 0.3358842134475708, "sampling/sampling_logp_difference/max": 1.0910255908966064, "sampling/sampling_logp_difference/mean": 0.04563205689191818, "step": 275, "step_time": 11.650065983980312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 264.5, "completions/mean_terminated_length": 264.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11489069275557995, "epoch": 0.00552, "frac_reward_zero_std": 0.0, "grad_norm": 0.5983346104621887, "kl": 0.9036147501319647, "learning_rate": 8.663350430052328e-06, "loss": -0.0676, "num_tokens": 13335334.0, "reward": 1.46875, "reward_std": 0.8637456893920898, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.8793096542358398, "sampling/importance_sampling_ratio/max": 1.4481984376907349, "sampling/importance_sampling_ratio/mean": 1.0221105813980103, "sampling/importance_sampling_ratio/min": 0.1384451538324356, "sampling/sampling_logp_difference/max": 1.9737523794174194, "sampling/sampling_logp_difference/mean": 0.03738903999328613, "step": 276, "step_time": 14.599634634054382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 229.125, "completions/mean_terminated_length": 229.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11721920152194798, "epoch": 0.00554, "frac_reward_zero_std": 0.0, "grad_norm": 0.6636167168617249, "kl": 0.7731942106038332, "learning_rate": 8.652910138274573e-06, "loss": -0.0317, "num_tokens": 13381062.0, "reward": 1.34375, "reward_std": 0.8137833476066589, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.8654431700706482, "sampling/importance_sampling_ratio/max": 2.959550142288208, "sampling/importance_sampling_ratio/mean": 1.0769323110580444, "sampling/importance_sampling_ratio/min": 0.7804954648017883, "sampling/sampling_logp_difference/max": 1.0632247924804688, "sampling/sampling_logp_difference/mean": 0.022454487159848213, "step": 277, "step_time": 13.4091721179866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 343.4375, "completions/mean_terminated_length": 343.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0853138449601829, "epoch": 0.00556, "frac_reward_zero_std": 0.0, "grad_norm": 0.8518053293228149, "kl": 1.898139794357121, "learning_rate": 8.642438150487718e-06, "loss": -0.0597, "num_tokens": 13431657.0, "reward": 1.59375, "reward_std": 0.9814831614494324, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.0429292917251587, "sampling/importance_sampling_ratio/max": 2.1138503551483154, "sampling/importance_sampling_ratio/mean": 1.0190589427947998, "sampling/importance_sampling_ratio/min": 0.3509523272514343, "sampling/sampling_logp_difference/max": 1.196131706237793, "sampling/sampling_logp_difference/mean": 0.03318548575043678, "step": 278, "step_time": 15.936478171992349 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.009765625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009765625, "completions/clipped_ratio": 0.03125, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 245.0, "completions/mean_terminated_length": 240.45159912109375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08832342340610921, "epoch": 0.00558, "frac_reward_zero_std": 0.0, "grad_norm": 0.6372953653335571, "kl": 1.9281429946422577, "learning_rate": 8.631934604824366e-06, "loss": 0.0076, "num_tokens": 13479455.0, "reward": 1.25, "reward_std": 0.897043764591217, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.0776318311691284, "sampling/importance_sampling_ratio/max": 1.2254072427749634, "sampling/importance_sampling_ratio/mean": 0.9887272119522095, "sampling/importance_sampling_ratio/min": 0.3732326328754425, "sampling/sampling_logp_difference/max": 0.9917397499084473, "sampling/sampling_logp_difference/mean": 0.023841235786676407, "step": 279, "step_time": 13.932243722010753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1670.0, "completions/max_terminated_length": 1670.0, "completions/mean_length": 466.53125, "completions/mean_terminated_length": 466.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07136438461020589, "epoch": 0.0056, "frac_reward_zero_std": 0.0, "grad_norm": 0.22683152556419373, "kl": 0.758972835727036, "learning_rate": 8.62139963983338e-06, "loss": -0.0444, "num_tokens": 13536042.0, "reward": 1.65625, "reward_std": 0.8377669453620911, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.9708451628684998, "sampling/importance_sampling_ratio/max": 1.3955389261245728, "sampling/importance_sampling_ratio/mean": 1.0143842697143555, "sampling/importance_sampling_ratio/min": 0.6581124663352966, "sampling/sampling_logp_difference/max": 0.4491918087005615, "sampling/sampling_logp_difference/mean": 0.015033384785056114, "step": 280, "step_time": 16.203823532006936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 294.53125, "completions/mean_terminated_length": 294.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05511784693226218, "epoch": 0.00562, "frac_reward_zero_std": 0.0, "grad_norm": 0.043939150869846344, "kl": 0.6950906272977591, "learning_rate": 8.610833394478073e-06, "loss": -0.0542, "num_tokens": 13586634.0, "reward": 1.71875, "reward_std": 0.5677276253700256, "rewards/rollout_reward_func/mean": 1.71875, "rewards/rollout_reward_func/std": 0.5811209678649902, "sampling/importance_sampling_ratio/max": 1.0573389530181885, "sampling/importance_sampling_ratio/mean": 1.0178786516189575, "sampling/importance_sampling_ratio/min": 1.0045820474624634, "sampling/sampling_logp_difference/max": 0.04996723681688309, "sampling/sampling_logp_difference/mean": 0.005013709422200918, "step": 281, "step_time": 13.868129355003475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1328.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 370.96875, "completions/mean_terminated_length": 370.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07468422898091376, "epoch": 0.00564, "frac_reward_zero_std": 0.0, "grad_norm": 0.2947354018688202, "kl": 0.6959607852622867, "learning_rate": 8.60023600813436e-06, "loss": -0.0704, "num_tokens": 13638769.0, "reward": 1.59375, "reward_std": 1.0758296251296997, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.0429292917251587, "sampling/importance_sampling_ratio/max": 1.2472316026687622, "sampling/importance_sampling_ratio/mean": 1.0156919956207275, "sampling/importance_sampling_ratio/min": 0.3212793171405792, "sampling/sampling_logp_difference/max": 1.1370902061462402, "sampling/sampling_logp_difference/mean": 0.019893914461135864, "step": 282, "step_time": 15.136773952981457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 286.6875, "completions/mean_terminated_length": 286.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0867858121637255, "epoch": 0.00566, "frac_reward_zero_std": 0.0, "grad_norm": 0.09330998361110687, "kl": 0.689325655810535, "learning_rate": 8.589607620588937e-06, "loss": -0.0536, "num_tokens": 13688686.0, "reward": 1.53125, "reward_std": 0.9216750860214233, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.915260374546051, "sampling/importance_sampling_ratio/max": 1.0783170461654663, "sampling/importance_sampling_ratio/mean": 1.020011305809021, "sampling/importance_sampling_ratio/min": 0.8907535076141357, "sampling/sampling_logp_difference/max": 0.11559915542602539, "sampling/sampling_logp_difference/mean": 0.00807584822177887, "step": 283, "step_time": 13.272749578987714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 343.78125, "completions/mean_terminated_length": 343.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0784856912214309, "epoch": 0.00568, "frac_reward_zero_std": 0.0, "grad_norm": 0.3393053710460663, "kl": 1.5955575183033943, "learning_rate": 8.578948372037416e-06, "loss": -0.0956, "num_tokens": 13739308.0, "reward": 1.28125, "reward_std": 1.24917733669281, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.1976959705352783, "sampling/importance_sampling_ratio/max": 2.6169824600219727, "sampling/importance_sampling_ratio/mean": 1.049271583557129, "sampling/importance_sampling_ratio/min": 0.39646756649017334, "sampling/sampling_logp_difference/max": 0.9533447027206421, "sampling/sampling_logp_difference/mean": 0.03259889408946037, "step": 284, "step_time": 15.649562473990954 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 324.5625, "completions/mean_terminated_length": 324.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.047980795381590724, "epoch": 0.0057, "frac_reward_zero_std": 0.0, "grad_norm": 1.113858699798584, "kl": 4.261272646486759, "learning_rate": 8.568258403082492e-06, "loss": -0.0542, "num_tokens": 13790812.0, "reward": 1.53125, "reward_std": 0.8262453079223633, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.8025915622711182, "sampling/importance_sampling_ratio/max": 1.06440269947052, "sampling/importance_sampling_ratio/mean": 0.9285975098609924, "sampling/importance_sampling_ratio/min": 0.199860617518425, "sampling/sampling_logp_difference/max": 1.6217350959777832, "sampling/sampling_logp_difference/mean": 0.04652699828147888, "step": 285, "step_time": 14.087205847958103 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 329.5625, "completions/mean_terminated_length": 329.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10327459778636694, "epoch": 0.00572, "frac_reward_zero_std": 0.0, "grad_norm": 0.5136629939079285, "kl": 1.1346641182899475, "learning_rate": 8.557537854732082e-06, "loss": -0.0519, "num_tokens": 13840210.0, "reward": 1.25, "reward_std": 1.0132696628570557, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.0472698211669922, "sampling/importance_sampling_ratio/max": 1.3463066816329956, "sampling/importance_sampling_ratio/mean": 0.994109034538269, "sampling/importance_sampling_ratio/min": 0.39774569869041443, "sampling/sampling_logp_difference/max": 0.9218034744262695, "sampling/sampling_logp_difference/mean": 0.035899482667446136, "step": 286, "step_time": 14.048193586058915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1535.0, "completions/max_terminated_length": 1535.0, "completions/mean_length": 219.28125, "completions/mean_terminated_length": 219.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1350629802327603, "epoch": 0.00574, "frac_reward_zero_std": 0.0, "grad_norm": 0.6591318249702454, "kl": 2.8329565078020096, "learning_rate": 8.546786868397465e-06, "loss": -0.0397, "num_tokens": 13885762.0, "reward": 0.96875, "reward_std": 0.9111547470092773, "rewards/rollout_reward_func/mean": 0.96875, "rewards/rollout_reward_func/std": 1.0312652587890625, "sampling/importance_sampling_ratio/max": 2.0082907676696777, "sampling/importance_sampling_ratio/mean": 0.9927774667739868, "sampling/importance_sampling_ratio/min": 0.2980915904045105, "sampling/sampling_logp_difference/max": 1.2150969505310059, "sampling/sampling_logp_difference/mean": 0.055558204650878906, "step": 287, "step_time": 15.087795287996414 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 415.40625, "completions/mean_terminated_length": 415.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0774734579026699, "epoch": 0.00576, "frac_reward_zero_std": 0.0, "grad_norm": 0.24726420640945435, "kl": 0.705130934715271, "learning_rate": 8.536005585891422e-06, "loss": -0.0481, "num_tokens": 13939717.0, "reward": 1.53125, "reward_std": 0.734941840171814, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.8025915622711182, "sampling/importance_sampling_ratio/max": 2.593583822250366, "sampling/importance_sampling_ratio/mean": 1.0685712099075317, "sampling/importance_sampling_ratio/min": 0.4532102942466736, "sampling/sampling_logp_difference/max": 0.8719208240509033, "sampling/sampling_logp_difference/mean": 0.028080035001039505, "step": 288, "step_time": 15.642244296017452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1288.0, "completions/max_terminated_length": 1288.0, "completions/mean_length": 366.5, "completions/mean_terminated_length": 366.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07991849025711417, "epoch": 0.00578, "frac_reward_zero_std": 0.0, "grad_norm": 0.26695048809051514, "kl": 0.9313383521512151, "learning_rate": 8.525194149426354e-06, "loss": -0.0551, "num_tokens": 13991855.0, "reward": 1.34375, "reward_std": 1.0755021572113037, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.035167932510376, "sampling/importance_sampling_ratio/max": 1.1474710702896118, "sampling/importance_sampling_ratio/mean": 0.9857193231582642, "sampling/importance_sampling_ratio/min": 0.36802107095718384, "sampling/sampling_logp_difference/max": 0.9994828701019287, "sampling/sampling_logp_difference/mean": 0.02917088195681572, "step": 289, "step_time": 15.006288876043982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1648.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 351.34375, "completions/mean_terminated_length": 289.3333435058594, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12669304688461125, "epoch": 0.0058, "frac_reward_zero_std": 0.0, "grad_norm": 0.6556295156478882, "kl": 0.7734925132244825, "learning_rate": 8.514352701612418e-06, "loss": -0.1194, "num_tokens": 14043128.0, "reward": 1.53125, "reward_std": 0.8763304948806763, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 1.0467884540557861, "sampling/importance_sampling_ratio/max": 1.6475485563278198, "sampling/importance_sampling_ratio/mean": 1.033975601196289, "sampling/importance_sampling_ratio/min": 0.3613293170928955, "sampling/sampling_logp_difference/max": 1.0177342891693115, "sampling/sampling_logp_difference/mean": 0.023341447114944458, "step": 290, "step_time": 16.528232826982276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 262.40625, "completions/mean_terminated_length": 262.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10510900733061135, "epoch": 0.00582, "frac_reward_zero_std": 0.0, "grad_norm": 0.3406846523284912, "kl": 0.6601763349026442, "learning_rate": 8.503481385455637e-06, "loss": -0.0579, "num_tokens": 14090267.0, "reward": 1.3125, "reward_std": 0.9774547815322876, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.029797911643982, "sampling/importance_sampling_ratio/max": 1.2391613721847534, "sampling/importance_sampling_ratio/mean": 1.0528333187103271, "sampling/importance_sampling_ratio/min": 0.9998276233673096, "sampling/sampling_logp_difference/max": 0.17716360092163086, "sampling/sampling_logp_difference/mean": 0.01568150520324707, "step": 291, "step_time": 15.687901110984967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 292.28125, "completions/mean_terminated_length": 292.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0752953237388283, "epoch": 0.00584, "frac_reward_zero_std": 0.0, "grad_norm": 0.6370899081230164, "kl": 1.2695056423544884, "learning_rate": 8.492580344356023e-06, "loss": -0.0595, "num_tokens": 14140282.0, "reward": 1.34375, "reward_std": 0.9408319592475891, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 1.3050611019134521, "sampling/importance_sampling_ratio/mean": 0.9609746932983398, "sampling/importance_sampling_ratio/min": 0.48023974895477295, "sampling/sampling_logp_difference/max": 0.7436337471008301, "sampling/sampling_logp_difference/mean": 0.03670945763587952, "step": 292, "step_time": 13.682484338030918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 429.25, "completions/mean_terminated_length": 429.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08539227955043316, "epoch": 0.00586, "frac_reward_zero_std": 0.0, "grad_norm": 0.25514253973960876, "kl": 1.3134182691574097, "learning_rate": 8.481649722105677e-06, "loss": -0.0553, "num_tokens": 14195721.0, "reward": 1.6875, "reward_std": 0.8406715989112854, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.9651173949241638, "sampling/importance_sampling_ratio/max": 1.472534418106079, "sampling/importance_sampling_ratio/mean": 0.9923534989356995, "sampling/importance_sampling_ratio/min": 0.24544839560985565, "sampling/sampling_logp_difference/max": 1.4113378524780273, "sampling/sampling_logp_difference/mean": 0.034731220453977585, "step": 293, "step_time": 14.308566065956256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 370.8125, "completions/mean_terminated_length": 370.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08329200325533748, "epoch": 0.00588, "frac_reward_zero_std": 0.0, "grad_norm": 0.3114226758480072, "kl": 0.745486181229353, "learning_rate": 8.470689662886895e-06, "loss": -0.0451, "num_tokens": 14249114.0, "reward": 1.6875, "reward_std": 0.778884768486023, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.8590129017829895, "sampling/importance_sampling_ratio/max": 1.5300674438476562, "sampling/importance_sampling_ratio/mean": 1.0443334579467773, "sampling/importance_sampling_ratio/min": 0.6379504203796387, "sampling/sampling_logp_difference/max": 0.45720064640045166, "sampling/sampling_logp_difference/mean": 0.01711101457476616, "step": 294, "step_time": 15.048991384959663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 239.1875, "completions/mean_terminated_length": 239.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08950319909490645, "epoch": 0.0059, "frac_reward_zero_std": 0.0, "grad_norm": 0.6556344628334045, "kl": 1.2991194548085332, "learning_rate": 8.459700311270268e-06, "loss": -0.0492, "num_tokens": 14295613.0, "reward": 1.53125, "reward_std": 0.7285416126251221, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.7177192568778992, "sampling/importance_sampling_ratio/max": 1.1077699661254883, "sampling/importance_sampling_ratio/mean": 0.9949522614479065, "sampling/importance_sampling_ratio/min": 0.3938406705856323, "sampling/sampling_logp_difference/max": 0.9385786056518555, "sampling/sampling_logp_difference/mean": 0.0282721146941185, "step": 295, "step_time": 13.46961512597045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 354.5625, "completions/mean_terminated_length": 354.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07839838566724211, "epoch": 0.00592, "frac_reward_zero_std": 0.0, "grad_norm": 0.8866714239120483, "kl": 0.8153871446847916, "learning_rate": 8.44868181221277e-06, "loss": -0.0373, "num_tokens": 14346937.0, "reward": 1.625, "reward_std": 0.7567965984344482, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.7513428926467896, "sampling/importance_sampling_ratio/max": 1.2735679149627686, "sampling/importance_sampling_ratio/mean": 1.0525459051132202, "sampling/importance_sampling_ratio/min": 0.9832791686058044, "sampling/sampling_logp_difference/max": 0.23373502492904663, "sampling/sampling_logp_difference/mean": 0.014003325253725052, "step": 296, "step_time": 14.08253407801385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 395.28125, "completions/mean_terminated_length": 395.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08914102951530367, "epoch": 0.00594, "frac_reward_zero_std": 0.0, "grad_norm": 0.794955313205719, "kl": 1.6697278879582882, "learning_rate": 8.437634311055855e-06, "loss": -0.0374, "num_tokens": 14400608.0, "reward": 1.5, "reward_std": 0.8879650831222534, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9503819346427917, "sampling/importance_sampling_ratio/max": 1.1424190998077393, "sampling/importance_sampling_ratio/mean": 0.979055643081665, "sampling/importance_sampling_ratio/min": 0.48267456889152527, "sampling/sampling_logp_difference/max": 0.7302103042602539, "sampling/sampling_logp_difference/mean": 0.027194220572710037, "step": 297, "step_time": 14.142161135023343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1661.0, "completions/max_terminated_length": 1661.0, "completions/mean_length": 256.5, "completions/mean_terminated_length": 256.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08853093488141894, "epoch": 0.00596, "frac_reward_zero_std": 0.0, "grad_norm": 0.4123913049697876, "kl": 0.7631368776783347, "learning_rate": 8.42655795352353e-06, "loss": -0.043, "num_tokens": 14447657.0, "reward": 1.1875, "reward_std": 1.0058131217956543, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.0906493663787842, "sampling/importance_sampling_ratio/max": 1.1003013849258423, "sampling/importance_sampling_ratio/mean": 0.9770933985710144, "sampling/importance_sampling_ratio/min": 0.3351301848888397, "sampling/sampling_logp_difference/max": 1.093078851699829, "sampling/sampling_logp_difference/mean": 0.030077455565333366, "step": 298, "step_time": 16.466892400974757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 268.4375, "completions/mean_terminated_length": 268.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08435093890875578, "epoch": 0.00598, "frac_reward_zero_std": 0.0, "grad_norm": 0.37255775928497314, "kl": 0.8746938537806273, "learning_rate": 8.415452885720438e-06, "loss": -0.0404, "num_tokens": 14495494.0, "reward": 1.28125, "reward_std": 0.8151521682739258, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.8125775456428528, "sampling/importance_sampling_ratio/max": 1.3435392379760742, "sampling/importance_sampling_ratio/mean": 0.9870996475219727, "sampling/importance_sampling_ratio/min": 0.25004446506500244, "sampling/sampling_logp_difference/max": 1.401390790939331, "sampling/sampling_logp_difference/mean": 0.03266336768865585, "step": 299, "step_time": 15.367857999983244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 350.78125, "completions/mean_terminated_length": 350.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09030970511958003, "epoch": 0.006, "frac_reward_zero_std": 0.0, "grad_norm": 0.16687996685504913, "kl": 2.2280637016519904, "learning_rate": 8.40431925412993e-06, "loss": -0.0594, "num_tokens": 14546845.0, "reward": 1.15625, "reward_std": 1.2500749826431274, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.3466655015945435, "sampling/importance_sampling_ratio/max": 1.4111995697021484, "sampling/importance_sampling_ratio/mean": 0.9355779886245728, "sampling/importance_sampling_ratio/min": 0.13443221151828766, "sampling/sampling_logp_difference/max": 1.5119023323059082, "sampling/sampling_logp_difference/mean": 0.06796932220458984, "step": 300, "step_time": 14.35720923099143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 335.28125, "completions/mean_terminated_length": 335.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0921296481974423, "epoch": 0.00602, "frac_reward_zero_std": 0.0, "grad_norm": 0.8170443773269653, "kl": 1.1967176469042897, "learning_rate": 8.393157205612137e-06, "loss": -0.0505, "num_tokens": 14598515.0, "reward": 1.15625, "reward_std": 1.016467571258545, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.167002558708191, "sampling/importance_sampling_ratio/max": 1.169921875, "sampling/importance_sampling_ratio/mean": 0.9785693883895874, "sampling/importance_sampling_ratio/min": 0.22836005687713623, "sampling/sampling_logp_difference/max": 1.4774158000946045, "sampling/sampling_logp_difference/mean": 0.029474887996912003, "step": 301, "step_time": 13.47236469296331 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1670.0, "completions/max_terminated_length": 1670.0, "completions/mean_length": 407.28125, "completions/mean_terminated_length": 407.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08721973653882742, "epoch": 0.00604, "frac_reward_zero_std": 0.0, "grad_norm": 0.25367146730422974, "kl": 1.067612448707223, "learning_rate": 8.38196688740202e-06, "loss": -0.0404, "num_tokens": 14652407.0, "reward": 1.40625, "reward_std": 0.9576541185379028, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.011526346206665, "sampling/importance_sampling_ratio/max": 1.3172560930252075, "sampling/importance_sampling_ratio/mean": 0.9967836737632751, "sampling/importance_sampling_ratio/min": 0.4605599641799927, "sampling/sampling_logp_difference/max": 0.7753089666366577, "sampling/sampling_logp_difference/mean": 0.03446618467569351, "step": 302, "step_time": 16.57837349300098 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 371.53125, "completions/mean_terminated_length": 371.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08954495075158775, "epoch": 0.00606, "frac_reward_zero_std": 0.0, "grad_norm": 0.21064989268779755, "kl": 1.3080012192949653, "learning_rate": 8.370748447107445e-06, "loss": -0.081, "num_tokens": 14705290.0, "reward": 1.59375, "reward_std": 1.1956455707550049, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.18755304813385, "sampling/importance_sampling_ratio/max": 1.2939273118972778, "sampling/importance_sampling_ratio/mean": 1.0357820987701416, "sampling/importance_sampling_ratio/min": 0.4271899163722992, "sampling/sampling_logp_difference/max": 0.8503704071044922, "sampling/sampling_logp_difference/mean": 0.021530620753765106, "step": 303, "step_time": 14.352066621970152 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 255.0625, "completions/mean_terminated_length": 255.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08024383219890296, "epoch": 0.00608, "frac_reward_zero_std": 0.0, "grad_norm": 0.7880098223686218, "kl": 1.140376805793494, "learning_rate": 8.359502032707219e-06, "loss": -0.0462, "num_tokens": 14754743.0, "reward": 1.25, "reward_std": 0.9586974382400513, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.1071614027023315, "sampling/importance_sampling_ratio/max": 2.392191171646118, "sampling/importance_sampling_ratio/mean": 1.0108351707458496, "sampling/importance_sampling_ratio/min": 0.37181827425956726, "sampling/sampling_logp_difference/max": 0.9892969131469727, "sampling/sampling_logp_difference/mean": 0.03432567045092583, "step": 304, "step_time": 14.947748272985336 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 261.40625, "completions/mean_terminated_length": 261.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0944844561163336, "epoch": 0.0061, "frac_reward_zero_std": 0.0, "grad_norm": 0.584414541721344, "kl": 1.9190959418192506, "learning_rate": 8.34822779254915e-06, "loss": -0.025, "num_tokens": 14802751.0, "reward": 1.21875, "reward_std": 0.9358478784561157, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 0.974989652633667, "sampling/importance_sampling_ratio/max": 1.0675166845321655, "sampling/importance_sampling_ratio/mean": 0.9538424015045166, "sampling/importance_sampling_ratio/min": 0.16347403824329376, "sampling/sampling_logp_difference/max": 1.846358060836792, "sampling/sampling_logp_difference/mean": 0.04503060132265091, "step": 305, "step_time": 14.01311188901309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 270.125, "completions/mean_terminated_length": 270.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08572874160017818, "epoch": 0.00612, "frac_reward_zero_std": 0.0, "grad_norm": 0.4988723695278168, "kl": 0.6232482995837927, "learning_rate": 8.336925875348093e-06, "loss": -0.0522, "num_tokens": 14851065.0, "reward": 1.34375, "reward_std": 0.7085968255996704, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.7873751521110535, "sampling/importance_sampling_ratio/max": 1.54747474193573, "sampling/importance_sampling_ratio/mean": 1.038214921951294, "sampling/importance_sampling_ratio/min": 0.7084565162658691, "sampling/sampling_logp_difference/max": 0.35434508323669434, "sampling/sampling_logp_difference/mean": 0.016231495887041092, "step": 306, "step_time": 15.331950996973319 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 272.1875, "completions/mean_terminated_length": 272.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07249273569323123, "epoch": 0.00614, "frac_reward_zero_std": 0.0, "grad_norm": 0.42614510655403137, "kl": 0.9089286532253027, "learning_rate": 8.32559643018397e-06, "loss": -0.0795, "num_tokens": 14899837.0, "reward": 1.625, "reward_std": 0.8256731033325195, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.8327955007553101, "sampling/importance_sampling_ratio/max": 1.1736811399459839, "sampling/importance_sampling_ratio/mean": 0.9936076998710632, "sampling/importance_sampling_ratio/min": 0.39731767773628235, "sampling/sampling_logp_difference/max": 0.9230406284332275, "sampling/sampling_logp_difference/mean": 0.024106943979859352, "step": 307, "step_time": 15.717645894023008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 375.625, "completions/mean_terminated_length": 375.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0657530736643821, "epoch": 0.00616, "frac_reward_zero_std": 0.0, "grad_norm": 0.4563533365726471, "kl": 5.481445174664259, "learning_rate": 8.314239606499828e-06, "loss": -0.0465, "num_tokens": 14952105.0, "reward": 1.4375, "reward_std": 0.9198552370071411, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9136068224906921, "sampling/importance_sampling_ratio/max": 1.3083598613739014, "sampling/importance_sampling_ratio/mean": 0.9657081365585327, "sampling/importance_sampling_ratio/min": 0.2917124629020691, "sampling/sampling_logp_difference/max": 1.2318754196166992, "sampling/sampling_logp_difference/mean": 0.04316555708646774, "step": 308, "step_time": 14.394577493978431 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 285.71875, "completions/mean_terminated_length": 285.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0772999282926321, "epoch": 0.00618, "frac_reward_zero_std": 0.0, "grad_norm": 0.4037633240222931, "kl": 1.5819082483649254, "learning_rate": 8.302855554099842e-06, "loss": -0.0643, "num_tokens": 15001239.0, "reward": 1.375, "reward_std": 1.0431849956512451, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.0395407676696777, "sampling/importance_sampling_ratio/max": 1.1273672580718994, "sampling/importance_sampling_ratio/mean": 0.9947542548179626, "sampling/importance_sampling_ratio/min": 0.3000514507293701, "sampling/sampling_logp_difference/max": 1.2045948505401611, "sampling/sampling_logp_difference/mean": 0.029862388968467712, "step": 309, "step_time": 14.138037718003034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 380.375, "completions/mean_terminated_length": 380.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05525479163043201, "epoch": 0.0062, "frac_reward_zero_std": 0.0, "grad_norm": 1.3400070667266846, "kl": 3.872776061296463, "learning_rate": 8.29144442314736e-06, "loss": -0.0477, "num_tokens": 15054983.0, "reward": 1.625, "reward_std": 0.8290277123451233, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.8327955007553101, "sampling/importance_sampling_ratio/max": 1.264603614807129, "sampling/importance_sampling_ratio/mean": 0.9698255062103271, "sampling/importance_sampling_ratio/min": 0.256183922290802, "sampling/sampling_logp_difference/max": 1.3654975891113281, "sampling/sampling_logp_difference/mean": 0.036039821803569794, "step": 310, "step_time": 14.119267518020933 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 2043.0, "completions/max_terminated_length": 2043.0, "completions/mean_length": 381.84375, "completions/mean_terminated_length": 381.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12051319377496839, "epoch": 0.00622, "frac_reward_zero_std": 0.0, "grad_norm": 0.5727736353874207, "kl": 1.3352529220283031, "learning_rate": 8.280006364162915e-06, "loss": -0.0733, "num_tokens": 15107958.0, "reward": 1.53125, "reward_std": 1.2230838537216187, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 1.2177284955978394, "sampling/importance_sampling_ratio/max": 1.0862810611724854, "sampling/importance_sampling_ratio/mean": 0.9253400564193726, "sampling/importance_sampling_ratio/min": 0.1314869523048401, "sampling/sampling_logp_difference/max": 2.0394411087036133, "sampling/sampling_logp_difference/mean": 0.05665311962366104, "step": 311, "step_time": 17.054313289001584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1706.0, "completions/max_terminated_length": 1706.0, "completions/mean_length": 410.90625, "completions/mean_terminated_length": 410.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07386863441206515, "epoch": 0.00624, "frac_reward_zero_std": 0.0, "grad_norm": 0.2234606295824051, "kl": 0.8887217249721289, "learning_rate": 8.268541528022238e-06, "loss": -0.0758, "num_tokens": 15161670.0, "reward": 1.59375, "reward_std": 0.9095531702041626, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.9455967545509338, "sampling/importance_sampling_ratio/max": 1.285459041595459, "sampling/importance_sampling_ratio/mean": 1.0428831577301025, "sampling/importance_sampling_ratio/min": 0.3842592239379883, "sampling/sampling_logp_difference/max": 0.9570469856262207, "sampling/sampling_logp_difference/mean": 0.02425125613808632, "step": 312, "step_time": 16.720235903994762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1328.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 441.8125, "completions/mean_terminated_length": 441.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06670265388675034, "epoch": 0.00626, "frac_reward_zero_std": 0.0, "grad_norm": 0.09969241172075272, "kl": 1.6947118919342756, "learning_rate": 8.257050065954267e-06, "loss": -0.0375, "num_tokens": 15216690.0, "reward": 1.28125, "reward_std": 1.0927234888076782, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.054464340209961, "sampling/importance_sampling_ratio/max": 1.126037359237671, "sampling/importance_sampling_ratio/mean": 0.9639785289764404, "sampling/importance_sampling_ratio/min": 0.29942941665649414, "sampling/sampling_logp_difference/max": 1.2064414024353027, "sampling/sampling_logp_difference/mean": 0.038012247532606125, "step": 313, "step_time": 15.250441037016571 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 179.625, "completions/mean_terminated_length": 179.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.057839723653160036, "epoch": 0.00628, "frac_reward_zero_std": 0.0, "grad_norm": 0.6487220525741577, "kl": 4.5804465571418405, "learning_rate": 8.245532129539153e-06, "loss": -0.0425, "num_tokens": 15261743.0, "reward": 1.28125, "reward_std": 0.795923113822937, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.8513509035110474, "sampling/importance_sampling_ratio/max": 1.154721736907959, "sampling/importance_sampling_ratio/mean": 1.0144717693328857, "sampling/importance_sampling_ratio/min": 0.543287456035614, "sampling/sampling_logp_difference/max": 0.6101622581481934, "sampling/sampling_logp_difference/mean": 0.015166133642196655, "step": 314, "step_time": 13.296049603013671 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 441.5, "completions/mean_terminated_length": 441.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0727600040845573, "epoch": 0.0063, "frac_reward_zero_std": 0.0, "grad_norm": 0.34759870171546936, "kl": 1.449299966916442, "learning_rate": 8.233987870706267e-06, "loss": -0.057, "num_tokens": 15316848.0, "reward": 1.71875, "reward_std": 1.0245473384857178, "rewards/rollout_reward_func/mean": 1.71875, "rewards/rollout_reward_func/std": 1.0846248865127563, "sampling/importance_sampling_ratio/max": 1.2144243717193604, "sampling/importance_sampling_ratio/mean": 0.985441267490387, "sampling/importance_sampling_ratio/min": 0.34902283549308777, "sampling/sampling_logp_difference/max": 1.0525298118591309, "sampling/sampling_logp_difference/mean": 0.028305940330028534, "step": 315, "step_time": 15.536732945023687 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 293.6875, "completions/mean_terminated_length": 293.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07425861072260886, "epoch": 0.00632, "frac_reward_zero_std": 0.0, "grad_norm": 0.24241790175437927, "kl": 2.108775420114398, "learning_rate": 8.222417441732182e-06, "loss": -0.0548, "num_tokens": 15366341.0, "reward": 1.59375, "reward_std": 0.9464322328567505, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.9455967545509338, "sampling/importance_sampling_ratio/max": 1.1895103454589844, "sampling/importance_sampling_ratio/mean": 0.9900081157684326, "sampling/importance_sampling_ratio/min": 0.3115485608577728, "sampling/sampling_logp_difference/max": 1.1786243915557861, "sampling/sampling_logp_difference/mean": 0.03110402449965477, "step": 316, "step_time": 14.15066446099081 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 211.375, "completions/mean_terminated_length": 211.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0790014237863943, "epoch": 0.00634, "frac_reward_zero_std": 0.0, "grad_norm": 0.3244011402130127, "kl": 2.168258440680802, "learning_rate": 8.210820995238682e-06, "loss": -0.0509, "num_tokens": 15413635.0, "reward": 1.21875, "reward_std": 0.9251530766487122, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.0075321197509766, "sampling/importance_sampling_ratio/max": 1.1964659690856934, "sampling/importance_sampling_ratio/mean": 0.984208345413208, "sampling/importance_sampling_ratio/min": 0.290736585855484, "sampling/sampling_logp_difference/max": 1.2351741790771484, "sampling/sampling_logp_difference/mean": 0.03063313663005829, "step": 317, "step_time": 13.452554467017762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 355.5625, "completions/mean_terminated_length": 355.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06634300667792559, "epoch": 0.00636, "frac_reward_zero_std": 0.0, "grad_norm": 0.39285656809806824, "kl": 1.6152858026325703, "learning_rate": 8.199198684190737e-06, "loss": -0.0512, "num_tokens": 15464532.0, "reward": 1.34375, "reward_std": 0.9315851926803589, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 1.3068232536315918, "sampling/importance_sampling_ratio/mean": 0.9944989085197449, "sampling/importance_sampling_ratio/min": 0.2952912747859955, "sampling/sampling_logp_difference/max": 1.2260849475860596, "sampling/sampling_logp_difference/mean": 0.04005632922053337, "step": 318, "step_time": 14.299321534053888 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 340.34375, "completions/mean_terminated_length": 340.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07664945907890797, "epoch": 0.00638, "frac_reward_zero_std": 0.0, "grad_norm": 0.33397039771080017, "kl": 2.208487250842154, "learning_rate": 8.187550661894482e-06, "loss": -0.0783, "num_tokens": 15515640.0, "reward": 1.65625, "reward_std": 0.921564519405365, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.9708451628684998, "sampling/importance_sampling_ratio/max": 1.2098703384399414, "sampling/importance_sampling_ratio/mean": 0.9953365325927734, "sampling/importance_sampling_ratio/min": 0.39326706528663635, "sampling/sampling_logp_difference/max": 0.9569251537322998, "sampling/sampling_logp_difference/mean": 0.03098393976688385, "step": 319, "step_time": 13.963726728994516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 308.46875, "completions/mean_terminated_length": 308.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07439474819693714, "epoch": 0.0064, "frac_reward_zero_std": 0.0, "grad_norm": 0.18438389897346497, "kl": 0.6723110191524029, "learning_rate": 8.175877081995205e-06, "loss": -0.0483, "num_tokens": 15565916.0, "reward": 1.625, "reward_std": 0.657589852809906, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.6599119901657104, "sampling/importance_sampling_ratio/max": 1.2054295539855957, "sampling/importance_sampling_ratio/mean": 1.010236382484436, "sampling/importance_sampling_ratio/min": 0.47450128197669983, "sampling/sampling_logp_difference/max": 0.7453242540359497, "sampling/sampling_logp_difference/mean": 0.022235753014683723, "step": 320, "step_time": 13.941675449998002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.017578125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017578125, "completions/clipped_ratio": 0.03125, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 288.8125, "completions/mean_terminated_length": 297.6128845214844, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1267588222399354, "epoch": 0.00642, "frac_reward_zero_std": 0.0, "grad_norm": 0.11181928217411041, "kl": 1.0619275625795126, "learning_rate": 8.16417809847532e-06, "loss": -0.0404, "num_tokens": 15614121.0, "reward": 1.40625, "reward_std": 0.8775908350944519, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.8747119307518005, "sampling/importance_sampling_ratio/max": 1.1837166547775269, "sampling/importance_sampling_ratio/mean": 0.9954933524131775, "sampling/importance_sampling_ratio/min": 0.15756510198116302, "sampling/sampling_logp_difference/max": 1.849823236465454, "sampling/sampling_logp_difference/mean": 0.03536437451839447, "step": 321, "step_time": 13.826848158016219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 748.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 240.90625, "completions/mean_terminated_length": 240.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07648280658759177, "epoch": 0.00644, "frac_reward_zero_std": 0.0, "grad_norm": 0.31952956318855286, "kl": 1.2405702508985996, "learning_rate": 8.152453865652327e-06, "loss": -0.046, "num_tokens": 15660626.0, "reward": 1.40625, "reward_std": 0.7847509980201721, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.797551691532135, "sampling/importance_sampling_ratio/max": 1.1050820350646973, "sampling/importance_sampling_ratio/mean": 0.9958600997924805, "sampling/importance_sampling_ratio/min": 0.34183749556541443, "sampling/sampling_logp_difference/max": 1.073441505432129, "sampling/sampling_logp_difference/mean": 0.024710357189178467, "step": 322, "step_time": 12.230468166002538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 245.9375, "completions/mean_terminated_length": 245.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09082006267271936, "epoch": 0.00646, "frac_reward_zero_std": 0.0, "grad_norm": 0.6322816610336304, "kl": 2.223157327622175, "learning_rate": 8.140704538176782e-06, "loss": -0.0182, "num_tokens": 15708040.0, "reward": 1.15625, "reward_std": 0.7499462366104126, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 0.919655442237854, "sampling/importance_sampling_ratio/max": 1.1832178831100464, "sampling/importance_sampling_ratio/mean": 0.9848122596740723, "sampling/importance_sampling_ratio/min": 0.44246241450309753, "sampling/sampling_logp_difference/max": 0.8261568546295166, "sampling/sampling_logp_difference/mean": 0.03513138368725777, "step": 323, "step_time": 15.353912246020627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 339.9375, "completions/mean_terminated_length": 339.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06777191674336791, "epoch": 0.00648, "frac_reward_zero_std": 0.0, "grad_norm": 0.6057721972465515, "kl": 0.7729474827647209, "learning_rate": 8.128930271030256e-06, "loss": -0.0675, "num_tokens": 15758400.0, "reward": 1.6875, "reward_std": 0.804724395275116, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.8590129017829895, "sampling/importance_sampling_ratio/max": 1.1108508110046387, "sampling/importance_sampling_ratio/mean": 1.02580726146698, "sampling/importance_sampling_ratio/min": 0.7746536135673523, "sampling/sampling_logp_difference/max": 0.2711408734321594, "sampling/sampling_logp_difference/mean": 0.011448200792074203, "step": 324, "step_time": 14.072513908016845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 282.5, "completions/mean_terminated_length": 282.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09049241710454226, "epoch": 0.0065, "frac_reward_zero_std": 0.0, "grad_norm": 0.49685442447662354, "kl": 1.5651277303695679, "learning_rate": 8.117131219523293e-06, "loss": -0.0265, "num_tokens": 15807431.0, "reward": 1.34375, "reward_std": 0.8428435325622559, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.8273305296897888, "sampling/importance_sampling_ratio/max": 1.4953417778015137, "sampling/importance_sampling_ratio/mean": 1.0298619270324707, "sampling/importance_sampling_ratio/min": 0.6752949357032776, "sampling/sampling_logp_difference/max": 0.4016726613044739, "sampling/sampling_logp_difference/mean": 0.02082931622862816, "step": 325, "step_time": 13.908629640005529 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 340.09375, "completions/mean_terminated_length": 340.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08924616733565927, "epoch": 0.00652, "frac_reward_zero_std": 0.0, "grad_norm": 0.34975048899650574, "kl": 1.1045972472056746, "learning_rate": 8.10530753929336e-06, "loss": -0.0581, "num_tokens": 15858010.0, "reward": 1.15625, "reward_std": 1.1584274768829346, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.1390254497528076, "sampling/importance_sampling_ratio/max": 1.6891486644744873, "sampling/importance_sampling_ratio/mean": 1.0012288093566895, "sampling/importance_sampling_ratio/min": 0.4532870352268219, "sampling/sampling_logp_difference/max": 0.8018844127655029, "sampling/sampling_logp_difference/mean": 0.03634491190314293, "step": 326, "step_time": 15.21087951005029 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 318.40625, "completions/mean_terminated_length": 318.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07141300989314914, "epoch": 0.00654, "frac_reward_zero_std": 0.0, "grad_norm": 0.8450720906257629, "kl": 1.0449221078306437, "learning_rate": 8.093459386302788e-06, "loss": -0.039, "num_tokens": 15909547.0, "reward": 1.46875, "reward_std": 0.804314136505127, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.915260374546051, "sampling/importance_sampling_ratio/max": 1.1532553434371948, "sampling/importance_sampling_ratio/mean": 0.9914566278457642, "sampling/importance_sampling_ratio/min": 0.4976964592933655, "sampling/sampling_logp_difference/max": 0.7329864501953125, "sampling/sampling_logp_difference/mean": 0.026445236057043076, "step": 327, "step_time": 12.905169728051987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 358.21875, "completions/mean_terminated_length": 358.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07333477621432394, "epoch": 0.00656, "frac_reward_zero_std": 0.0, "grad_norm": 0.09530594944953918, "kl": 1.5897016525268555, "learning_rate": 8.081586916836728e-06, "loss": -0.0754, "num_tokens": 15961356.0, "reward": 1.6875, "reward_std": 0.9251691102981567, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.9651173949241638, "sampling/importance_sampling_ratio/max": 1.107797384262085, "sampling/importance_sampling_ratio/mean": 1.0103181600570679, "sampling/importance_sampling_ratio/min": 0.42480018734931946, "sampling/sampling_logp_difference/max": 0.8563315868377686, "sampling/sampling_logp_difference/mean": 0.016143910586833954, "step": 328, "step_time": 13.955051157987327 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1670.0, "completions/max_terminated_length": 1670.0, "completions/mean_length": 272.21875, "completions/mean_terminated_length": 272.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07540769106708467, "epoch": 0.00658, "frac_reward_zero_std": 0.0, "grad_norm": 0.5059762597084045, "kl": 0.6852445863187313, "learning_rate": 8.069690287501078e-06, "loss": -0.0581, "num_tokens": 16009405.0, "reward": 1.5625, "reward_std": 0.6546903848648071, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.6690146923065186, "sampling/importance_sampling_ratio/max": 1.118624210357666, "sampling/importance_sampling_ratio/mean": 0.994063138961792, "sampling/importance_sampling_ratio/min": 0.42231059074401855, "sampling/sampling_logp_difference/max": 1.121964454650879, "sampling/sampling_logp_difference/mean": 0.02712535485625267, "step": 329, "step_time": 16.4589390650508 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 784.0, "completions/max_terminated_length": 784.0, "completions/mean_length": 229.5, "completions/mean_terminated_length": 229.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0642009349539876, "epoch": 0.0066, "frac_reward_zero_std": 0.0, "grad_norm": 0.14060275256633759, "kl": 1.3409475795924664, "learning_rate": 8.05776965522042e-06, "loss": -0.0522, "num_tokens": 16056856.0, "reward": 1.375, "reward_std": 0.7220425605773926, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.9069623351097107, "sampling/importance_sampling_ratio/max": 1.600313425064087, "sampling/importance_sampling_ratio/mean": 1.0107377767562866, "sampling/importance_sampling_ratio/min": 0.2859753668308258, "sampling/sampling_logp_difference/max": 1.2517189979553223, "sampling/sampling_logp_difference/mean": 0.030138425529003143, "step": 330, "step_time": 12.419247320955037 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 282.3125, "completions/mean_terminated_length": 282.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08626498281955719, "epoch": 0.00662, "frac_reward_zero_std": 0.0, "grad_norm": 2.2630817890167236, "kl": 4.559206388890743, "learning_rate": 8.045825177235952e-06, "loss": -0.0422, "num_tokens": 16105889.0, "reward": 1.4375, "reward_std": 1.0741145610809326, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.0757592916488647, "sampling/importance_sampling_ratio/max": 1.5848839282989502, "sampling/importance_sampling_ratio/mean": 1.040325403213501, "sampling/importance_sampling_ratio/min": 0.22796764969825745, "sampling/sampling_logp_difference/max": 0.8546962738037109, "sampling/sampling_logp_difference/mean": 0.0299236997961998, "step": 331, "step_time": 13.235384045008686 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 298.09375, "completions/mean_terminated_length": 298.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08270549192093313, "epoch": 0.00664, "frac_reward_zero_std": 0.0, "grad_norm": 0.5013669729232788, "kl": 1.0255382964387536, "learning_rate": 8.033857011103411e-06, "loss": -0.0909, "num_tokens": 16153953.0, "reward": 1.5625, "reward_std": 0.7975823879241943, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.8007053136825562, "sampling/importance_sampling_ratio/max": 1.4080402851104736, "sampling/importance_sampling_ratio/mean": 0.9764435291290283, "sampling/importance_sampling_ratio/min": 0.0781923457980156, "sampling/sampling_logp_difference/max": 1.4463911056518555, "sampling/sampling_logp_difference/mean": 0.05439011752605438, "step": 332, "step_time": 14.720763463032199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 829.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 258.15625, "completions/mean_terminated_length": 258.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08022959670051932, "epoch": 0.00666, "frac_reward_zero_std": 0.25, "grad_norm": 0.4311225116252899, "kl": 0.9100431688129902, "learning_rate": 8.021865314691006e-06, "loss": -0.0546, "num_tokens": 16202358.0, "reward": 1.4375, "reward_std": 0.9109994173049927, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.0757592916488647, "sampling/importance_sampling_ratio/max": 1.0631318092346191, "sampling/importance_sampling_ratio/mean": 0.97799152135849, "sampling/importance_sampling_ratio/min": 0.409382700920105, "sampling/sampling_logp_difference/max": 0.8929774761199951, "sampling/sampling_logp_difference/mean": 0.027531813830137253, "step": 333, "step_time": 12.693874434015015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 311.25, "completions/mean_terminated_length": 311.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0685589739587158, "epoch": 0.00668, "frac_reward_zero_std": 0.0, "grad_norm": 0.11956529319286346, "kl": 0.8864520965144038, "learning_rate": 8.009850246177311e-06, "loss": -0.0386, "num_tokens": 16252566.0, "reward": 1.3125, "reward_std": 0.9244064092636108, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.931093692779541, "sampling/importance_sampling_ratio/max": 1.7834413051605225, "sampling/importance_sampling_ratio/mean": 1.028597354888916, "sampling/importance_sampling_ratio/min": 0.35422009229660034, "sampling/sampling_logp_difference/max": 1.0602282285690308, "sampling/sampling_logp_difference/mean": 0.026446696370840073, "step": 334, "step_time": 15.207233224020456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 247.8125, "completions/mean_terminated_length": 247.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.057666282868012786, "epoch": 0.0067, "frac_reward_zero_std": 0.0, "grad_norm": 0.6151423454284668, "kl": 1.6768543031066656, "learning_rate": 7.997811964049207e-06, "loss": -0.0459, "num_tokens": 16298567.0, "reward": 1.3125, "reward_std": 0.8146313428878784, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.931093692779541, "sampling/importance_sampling_ratio/max": 1.0950756072998047, "sampling/importance_sampling_ratio/mean": 0.9994427561759949, "sampling/importance_sampling_ratio/min": 0.332416832447052, "sampling/sampling_logp_difference/max": 1.1013360023498535, "sampling/sampling_logp_difference/mean": 0.02426333539187908, "step": 335, "step_time": 13.51993502197729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 343.71875, "completions/mean_terminated_length": 343.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06656928965821862, "epoch": 0.00672, "frac_reward_zero_std": 0.0, "grad_norm": 0.3652057647705078, "kl": 1.521996658295393, "learning_rate": 7.98575062709977e-06, "loss": -0.0643, "num_tokens": 16350627.0, "reward": 1.65625, "reward_std": 0.9203606843948364, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 1.2409756183624268, "sampling/importance_sampling_ratio/mean": 1.0067756175994873, "sampling/importance_sampling_ratio/min": 0.33227333426475525, "sampling/sampling_logp_difference/max": 1.1131696701049805, "sampling/sampling_logp_difference/mean": 0.028648795560002327, "step": 336, "step_time": 14.323841526012984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 227.53125, "completions/mean_terminated_length": 227.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07901212153956294, "epoch": 0.00674, "frac_reward_zero_std": 0.0, "grad_norm": 0.48067915439605713, "kl": 2.9078687075525522, "learning_rate": 7.97366639442619e-06, "loss": -0.0473, "num_tokens": 16398237.0, "reward": 1.3125, "reward_std": 0.9904576539993286, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.9979817867279053, "sampling/importance_sampling_ratio/max": 1.1005268096923828, "sampling/importance_sampling_ratio/mean": 0.9984295964241028, "sampling/importance_sampling_ratio/min": 0.4783664345741272, "sampling/sampling_logp_difference/max": 0.7372312545776367, "sampling/sampling_logp_difference/mean": 0.021926848217844963, "step": 337, "step_time": 13.651005747029558 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01171875, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 302.5625, "completions/mean_terminated_length": 302.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07855536974966526, "epoch": 0.00676, "frac_reward_zero_std": 0.0, "grad_norm": 0.6852776408195496, "kl": 0.9892375245690346, "learning_rate": 7.961559425427666e-06, "loss": -0.0176, "num_tokens": 16447255.0, "reward": 1.0625, "reward_std": 1.192363977432251, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.2427207231521606, "sampling/importance_sampling_ratio/max": 1.2442858219146729, "sampling/importance_sampling_ratio/mean": 0.9876965284347534, "sampling/importance_sampling_ratio/min": 0.2394169569015503, "sampling/sampling_logp_difference/max": 1.4391316175460815, "sampling/sampling_logp_difference/mean": 0.04124888777732849, "step": 338, "step_time": 15.95109957101522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1382.0, "completions/max_terminated_length": 1382.0, "completions/mean_length": 360.59375, "completions/mean_terminated_length": 360.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09067258005961776, "epoch": 0.00678, "frac_reward_zero_std": 0.0, "grad_norm": 0.34793367981910706, "kl": 0.7566601447761059, "learning_rate": 7.949429879803298e-06, "loss": -0.0612, "num_tokens": 16498991.0, "reward": 1.46875, "reward_std": 0.9801523685455322, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.0467884540557861, "sampling/importance_sampling_ratio/max": 1.1041431427001953, "sampling/importance_sampling_ratio/mean": 0.980398416519165, "sampling/importance_sampling_ratio/min": 0.3892126679420471, "sampling/sampling_logp_difference/max": 0.9436318874359131, "sampling/sampling_logp_difference/mean": 0.02802012860774994, "step": 339, "step_time": 14.840039278991753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 263.09375, "completions/mean_terminated_length": 263.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07146835583262146, "epoch": 0.0068, "frac_reward_zero_std": 0.0, "grad_norm": 0.5472195744514465, "kl": 1.5228505954146385, "learning_rate": 7.937277917549998e-06, "loss": -0.0454, "num_tokens": 16546377.0, "reward": 1.21875, "reward_std": 0.9337700605392456, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 0.9413228034973145, "sampling/importance_sampling_ratio/max": 1.1255929470062256, "sampling/importance_sampling_ratio/mean": 0.9670411944389343, "sampling/importance_sampling_ratio/min": 0.4445599913597107, "sampling/sampling_logp_difference/max": 0.8106861114501953, "sampling/sampling_logp_difference/mean": 0.04318308085203171, "step": 340, "step_time": 13.738785943991388 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 263.5, "completions/mean_terminated_length": 263.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08136977697722614, "epoch": 0.00682, "frac_reward_zero_std": 0.0, "grad_norm": 4.163798809051514, "kl": 26.28340332955122, "learning_rate": 7.925103698960361e-06, "loss": -0.0082, "num_tokens": 16595298.0, "reward": 1.1875, "reward_std": 1.103473424911499, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.1198358535766602, "sampling/importance_sampling_ratio/max": 1.2189536094665527, "sampling/importance_sampling_ratio/mean": 1.030428171157837, "sampling/importance_sampling_ratio/min": 0.5049135088920593, "sampling/sampling_logp_difference/max": 0.6833820343017578, "sampling/sampling_logp_difference/mean": 0.01969885267317295, "step": 341, "step_time": 13.733867158021894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 381.65625, "completions/mean_terminated_length": 381.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06345216510817409, "epoch": 0.00684, "frac_reward_zero_std": 0.0, "grad_norm": 0.7212512493133545, "kl": 0.7322565279901028, "learning_rate": 7.91290738462056e-06, "loss": -0.0797, "num_tokens": 16647611.0, "reward": 1.65625, "reward_std": 0.9308560490608215, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 1.2146347761154175, "sampling/importance_sampling_ratio/mean": 1.0118942260742188, "sampling/importance_sampling_ratio/min": 0.3726795017719269, "sampling/sampling_logp_difference/max": 0.995783805847168, "sampling/sampling_logp_difference/mean": 0.019543493166565895, "step": 342, "step_time": 13.827005106009892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 224.4375, "completions/mean_terminated_length": 224.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.061880517983809114, "epoch": 0.00686, "frac_reward_zero_std": 0.0, "grad_norm": 0.12725041806697845, "kl": 1.1949670426547527, "learning_rate": 7.900689135408226e-06, "loss": -0.0559, "num_tokens": 16694805.0, "reward": 1.375, "reward_std": 0.684157133102417, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.8327955007553101, "sampling/importance_sampling_ratio/max": 1.1437480449676514, "sampling/importance_sampling_ratio/mean": 0.9830254316329956, "sampling/importance_sampling_ratio/min": 0.32698890566825867, "sampling/sampling_logp_difference/max": 1.1276737451553345, "sampling/sampling_logp_difference/mean": 0.02731797657907009, "step": 343, "step_time": 12.696551994013134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 298.21875, "completions/mean_terminated_length": 298.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06967082037590444, "epoch": 0.00688, "frac_reward_zero_std": 0.0, "grad_norm": 0.39169713854789734, "kl": 3.1438942467793822, "learning_rate": 7.888449112490325e-06, "loss": -0.0665, "num_tokens": 16743357.0, "reward": 1.21875, "reward_std": 1.0919890403747559, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.0696510076522827, "sampling/importance_sampling_ratio/max": 1.4344124794006348, "sampling/importance_sampling_ratio/mean": 0.9767509698867798, "sampling/importance_sampling_ratio/min": 0.2652463912963867, "sampling/sampling_logp_difference/max": 1.332040548324585, "sampling/sampling_logp_difference/mean": 0.04583330452442169, "step": 344, "step_time": 13.618536549969576 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1364.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 298.75, "completions/mean_terminated_length": 298.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06325064925476909, "epoch": 0.0069, "frac_reward_zero_std": 0.0, "grad_norm": 2.794086217880249, "kl": 17.428212663158774, "learning_rate": 7.876187477321033e-06, "loss": -0.0325, "num_tokens": 16793200.0, "reward": 1.09375, "reward_std": 1.1872563362121582, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.1460838317871094, "sampling/importance_sampling_ratio/max": 1.2318196296691895, "sampling/importance_sampling_ratio/mean": 0.9256033301353455, "sampling/importance_sampling_ratio/min": 0.3263181447982788, "sampling/sampling_logp_difference/max": 1.1198034286499023, "sampling/sampling_logp_difference/mean": 0.052745699882507324, "step": 345, "step_time": 14.686215773035656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 227.625, "completions/mean_terminated_length": 227.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09916875930503011, "epoch": 0.00692, "frac_reward_zero_std": 0.0, "grad_norm": 0.4307337999343872, "kl": 1.329666217789054, "learning_rate": 7.86390439163961e-06, "loss": -0.042, "num_tokens": 16839293.0, "reward": 1.3125, "reward_std": 0.8423935174942017, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.8206016421318054, "sampling/importance_sampling_ratio/max": 1.4576643705368042, "sampling/importance_sampling_ratio/mean": 0.9559917449951172, "sampling/importance_sampling_ratio/min": 1.168475862687999e-16, "sampling/sampling_logp_difference/max": 22.780445098876953, "sampling/sampling_logp_difference/mean": 0.33874639868736267, "step": 346, "step_time": 14.073130169999786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 229.9375, "completions/mean_terminated_length": 229.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08630913309752941, "epoch": 0.00694, "frac_reward_zero_std": 0.0, "grad_norm": 0.515609622001648, "kl": 1.2585805915296078, "learning_rate": 7.851600017468257e-06, "loss": -0.0702, "num_tokens": 16886276.0, "reward": 1.34375, "reward_std": 1.0222105979919434, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.035167932510376, "sampling/importance_sampling_ratio/max": 1.1286344528198242, "sampling/importance_sampling_ratio/mean": 1.0087521076202393, "sampling/importance_sampling_ratio/min": 0.5204976201057434, "sampling/sampling_logp_difference/max": 0.6529934406280518, "sampling/sampling_logp_difference/mean": 0.019980117678642273, "step": 347, "step_time": 13.882192685006885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 304.65625, "completions/mean_terminated_length": 304.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07162089948542416, "epoch": 0.00696, "frac_reward_zero_std": 0.0, "grad_norm": 0.3086729645729065, "kl": 1.338092427700758, "learning_rate": 7.839274517109987e-06, "loss": -0.0755, "num_tokens": 16936412.0, "reward": 1.59375, "reward_std": 0.921572208404541, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.9455967545509338, "sampling/importance_sampling_ratio/max": 1.1382695436477661, "sampling/importance_sampling_ratio/mean": 1.0045685768127441, "sampling/importance_sampling_ratio/min": 0.14215263724327087, "sampling/sampling_logp_difference/max": 1.0946416854858398, "sampling/sampling_logp_difference/mean": 0.025395456701517105, "step": 348, "step_time": 13.780877493991284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1589.0, "completions/max_terminated_length": 1589.0, "completions/mean_length": 361.3125, "completions/mean_terminated_length": 361.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08324604923836887, "epoch": 0.00698, "frac_reward_zero_std": 0.0, "grad_norm": 0.7182756066322327, "kl": 1.2072866335511208, "learning_rate": 7.826928053146482e-06, "loss": -0.0886, "num_tokens": 16987846.0, "reward": 1.4375, "reward_std": 1.0356190204620361, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.3452544212341309, "sampling/importance_sampling_ratio/mean": 0.962256669998169, "sampling/importance_sampling_ratio/min": 0.27855604887008667, "sampling/sampling_logp_difference/max": 1.2877919673919678, "sampling/sampling_logp_difference/mean": 0.04879767820239067, "step": 349, "step_time": 16.302370366975083 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 331.0625, "completions/mean_terminated_length": 331.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0869124149903655, "epoch": 0.007, "frac_reward_zero_std": 0.0, "grad_norm": 0.2843640148639679, "kl": 2.5638146102428436, "learning_rate": 7.814560788435947e-06, "loss": -0.0811, "num_tokens": 17038553.0, "reward": 1.59375, "reward_std": 1.10627019405365, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.0734140872955322, "sampling/importance_sampling_ratio/max": 1.338942527770996, "sampling/importance_sampling_ratio/mean": 1.0145212411880493, "sampling/importance_sampling_ratio/min": 0.3308914005756378, "sampling/sampling_logp_difference/max": 1.1060080528259277, "sampling/sampling_logp_difference/mean": 0.030624685809016228, "step": 350, "step_time": 15.700546531021246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 351.46875, "completions/mean_terminated_length": 351.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10632460657507181, "epoch": 0.00702, "frac_reward_zero_std": 0.0, "grad_norm": 0.415811151266098, "kl": 0.7054446469992399, "learning_rate": 7.802172886110968e-06, "loss": -0.0715, "num_tokens": 17090632.0, "reward": 1.375, "reward_std": 0.8513727188110352, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.941858172416687, "sampling/importance_sampling_ratio/max": 1.6589792966842651, "sampling/importance_sampling_ratio/mean": 1.0273079872131348, "sampling/importance_sampling_ratio/min": 0.5358610153198242, "sampling/sampling_logp_difference/max": 0.773922324180603, "sampling/sampling_logp_difference/mean": 0.02746276557445526, "step": 351, "step_time": 15.401933711982565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2178.0, "completions/max_terminated_length": 2178.0, "completions/mean_length": 374.25, "completions/mean_terminated_length": 374.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11354491231031716, "epoch": 0.00704, "frac_reward_zero_std": 0.0, "grad_norm": 0.8214868903160095, "kl": 0.7062755227088928, "learning_rate": 7.789764509576347e-06, "loss": -0.0665, "num_tokens": 17142373.0, "reward": 1.25, "reward_std": 1.2464840412139893, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.2443419694900513, "sampling/importance_sampling_ratio/max": 1.2395614385604858, "sampling/importance_sampling_ratio/mean": 0.9526293873786926, "sampling/importance_sampling_ratio/min": 0.3497345447540283, "sampling/sampling_logp_difference/max": 1.0470484495162964, "sampling/sampling_logp_difference/mean": 0.042348623275756836, "step": 352, "step_time": 19.044171675981488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 272.125, "completions/mean_terminated_length": 272.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0769479745067656, "epoch": 0.00706, "frac_reward_zero_std": 0.0, "grad_norm": 0.3642632067203522, "kl": 3.3549687694758177, "learning_rate": 7.77733582250696e-06, "loss": -0.0731, "num_tokens": 17189972.0, "reward": 1.28125, "reward_std": 0.8839800357818604, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.9913944005966187, "sampling/importance_sampling_ratio/max": 1.3531084060668945, "sampling/importance_sampling_ratio/mean": 1.0052297115325928, "sampling/importance_sampling_ratio/min": 0.37174803018569946, "sampling/sampling_logp_difference/max": 0.9894475936889648, "sampling/sampling_logp_difference/mean": 0.030002325773239136, "step": 353, "step_time": 14.07008309601224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 244.6875, "completions/mean_terminated_length": 244.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0690098674967885, "epoch": 0.00708, "frac_reward_zero_std": 0.0, "grad_norm": 0.3549570143222809, "kl": 1.5794625133275986, "learning_rate": 7.764886988845588e-06, "loss": -0.053, "num_tokens": 17236848.0, "reward": 1.375, "reward_std": 0.912533164024353, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.008032202720642, "sampling/importance_sampling_ratio/max": 1.132253646850586, "sampling/importance_sampling_ratio/mean": 1.0080080032348633, "sampling/importance_sampling_ratio/min": 0.7059341073036194, "sampling/sampling_logp_difference/max": 0.34828662872314453, "sampling/sampling_logp_difference/mean": 0.012896637432277203, "step": 354, "step_time": 13.841083535022335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 243.3125, "completions/mean_terminated_length": 243.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05660919507499784, "epoch": 0.0071, "frac_reward_zero_std": 0.0, "grad_norm": 0.13747484982013702, "kl": 0.9258291032165289, "learning_rate": 7.752418172800769e-06, "loss": -0.0642, "num_tokens": 17282800.0, "reward": 1.4375, "reward_std": 0.7308446168899536, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.7593502998352051, "sampling/importance_sampling_ratio/max": 1.0663443803787231, "sampling/importance_sampling_ratio/mean": 1.0028932094573975, "sampling/importance_sampling_ratio/min": 0.3394679129123688, "sampling/sampling_logp_difference/max": 1.0959577560424805, "sampling/sampling_logp_difference/mean": 0.017451390624046326, "step": 355, "step_time": 13.948604663994047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 296.375, "completions/mean_terminated_length": 296.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09069631155580282, "epoch": 0.00712, "frac_reward_zero_std": 0.0, "grad_norm": 0.2577815651893616, "kl": 0.7355632176622748, "learning_rate": 7.73992953884461e-06, "loss": -0.0414, "num_tokens": 17332423.0, "reward": 1.5, "reward_std": 0.7766784429550171, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.8798826932907104, "sampling/importance_sampling_ratio/max": 1.2718400955200195, "sampling/importance_sampling_ratio/mean": 1.0201115608215332, "sampling/importance_sampling_ratio/min": 0.5915483236312866, "sampling/sampling_logp_difference/max": 0.5447999238967896, "sampling/sampling_logp_difference/mean": 0.018186945468187332, "step": 356, "step_time": 13.410246477011242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1998.0, "completions/max_terminated_length": 1998.0, "completions/mean_length": 352.84375, "completions/mean_terminated_length": 352.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05471473187208176, "epoch": 0.00714, "frac_reward_zero_std": 0.0, "grad_norm": 0.3116195797920227, "kl": 0.6193066774867475, "learning_rate": 7.727421251710637e-06, "loss": -0.0772, "num_tokens": 17383709.0, "reward": 1.625, "reward_std": 1.0907782316207886, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 1.099853277206421, "sampling/importance_sampling_ratio/max": 1.129883885383606, "sampling/importance_sampling_ratio/mean": 1.0260865688323975, "sampling/importance_sampling_ratio/min": 1.0053026676177979, "sampling/sampling_logp_difference/max": 0.07216069102287292, "sampling/sampling_logp_difference/mean": 0.0069001419469714165, "step": 357, "step_time": 17.295675992994802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1607.0, "completions/max_terminated_length": 1607.0, "completions/mean_length": 356.375, "completions/mean_terminated_length": 356.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04695643577724695, "epoch": 0.00716, "frac_reward_zero_std": 0.0, "grad_norm": 0.6624308228492737, "kl": 0.7412352785468102, "learning_rate": 7.714893476391613e-06, "loss": -0.0639, "num_tokens": 17435026.0, "reward": 1.65625, "reward_std": 0.7848250269889832, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.8654431700706482, "sampling/importance_sampling_ratio/max": 1.165865182876587, "sampling/importance_sampling_ratio/mean": 1.0278801918029785, "sampling/importance_sampling_ratio/min": 1.0014177560806274, "sampling/sampling_logp_difference/max": 0.1371741145849228, "sampling/sampling_logp_difference/mean": 0.007478790823370218, "step": 358, "step_time": 16.115803429987864 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 260.8125, "completions/mean_terminated_length": 260.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07910573668777943, "epoch": 0.00718, "frac_reward_zero_std": 0.0, "grad_norm": 0.47359591722488403, "kl": 1.7569020176306367, "learning_rate": 7.702346378137364e-06, "loss": -0.0606, "num_tokens": 17484163.0, "reward": 1.59375, "reward_std": 0.8367807865142822, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.9108441472053528, "sampling/importance_sampling_ratio/max": 1.1128829717636108, "sampling/importance_sampling_ratio/mean": 1.014042854309082, "sampling/importance_sampling_ratio/min": 0.6922163367271423, "sampling/sampling_logp_difference/max": 0.36789655685424805, "sampling/sampling_logp_difference/mean": 0.010390779934823513, "step": 359, "step_time": 13.503129546981654 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 349.1875, "completions/mean_terminated_length": 349.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05934691824950278, "epoch": 0.0072, "frac_reward_zero_std": 0.0, "grad_norm": 0.05528108775615692, "kl": 0.5980807933956385, "learning_rate": 7.689780122452598e-06, "loss": -0.0603, "num_tokens": 17536448.0, "reward": 1.8125, "reward_std": 0.7100419998168945, "rewards/rollout_reward_func/mean": 1.8125, "rewards/rollout_reward_func/std": 0.7378040552139282, "sampling/importance_sampling_ratio/max": 1.0961010456085205, "sampling/importance_sampling_ratio/mean": 1.0123248100280762, "sampling/importance_sampling_ratio/min": 0.699709951877594, "sampling/sampling_logp_difference/max": 0.4463735818862915, "sampling/sampling_logp_difference/mean": 0.010307662189006805, "step": 360, "step_time": 13.753182557993568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1706.0, "completions/max_terminated_length": 1706.0, "completions/mean_length": 303.21875, "completions/mean_terminated_length": 303.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07566933776251972, "epoch": 0.00722, "frac_reward_zero_std": 0.0, "grad_norm": 0.9570629596710205, "kl": 1.6444646827876568, "learning_rate": 7.67719487509472e-06, "loss": -0.0463, "num_tokens": 17586183.0, "reward": 1.4375, "reward_std": 0.9161993861198425, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9482581615447998, "sampling/importance_sampling_ratio/max": 1.0438416004180908, "sampling/importance_sampling_ratio/mean": 0.9703623056411743, "sampling/importance_sampling_ratio/min": 0.21684998273849487, "sampling/sampling_logp_difference/max": 1.6425492763519287, "sampling/sampling_logp_difference/mean": 0.028598811477422714, "step": 361, "step_time": 16.310172690966283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 317.875, "completions/mean_terminated_length": 317.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.049430534010753036, "epoch": 0.00724, "frac_reward_zero_std": 0.0, "grad_norm": 0.96527498960495, "kl": 5.785588998347521, "learning_rate": 7.664590802071653e-06, "loss": -0.0585, "num_tokens": 17636248.0, "reward": 1.28125, "reward_std": 1.1165721416473389, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.1704527139663696, "sampling/importance_sampling_ratio/max": 1.1024870872497559, "sampling/importance_sampling_ratio/mean": 0.9886479377746582, "sampling/importance_sampling_ratio/min": 0.4284111559391022, "sampling/sampling_logp_difference/max": 0.8471269607543945, "sampling/sampling_logp_difference/mean": 0.020098723471164703, "step": 362, "step_time": 13.781131299983826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 350.75, "completions/mean_terminated_length": 350.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.051845282316207886, "epoch": 0.00726, "frac_reward_zero_std": 0.0, "grad_norm": 0.2560412287712097, "kl": 2.121172308921814, "learning_rate": 7.651968069639637e-06, "loss": -0.0411, "num_tokens": 17686521.0, "reward": 1.34375, "reward_std": 1.0799229145050049, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.0658745765686035, "sampling/importance_sampling_ratio/max": 1.1243211030960083, "sampling/importance_sampling_ratio/mean": 0.9435084462165833, "sampling/importance_sampling_ratio/min": 0.33156874775886536, "sampling/sampling_logp_difference/max": 1.1683626174926758, "sampling/sampling_logp_difference/mean": 0.04507530480623245, "step": 363, "step_time": 15.751915040993481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 361.125, "completions/mean_terminated_length": 361.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.03708069759886712, "epoch": 0.00728, "frac_reward_zero_std": 0.0, "grad_norm": 1.3213690519332886, "kl": 2.6611892338842154, "learning_rate": 7.63932684430105e-06, "loss": -0.0365, "num_tokens": 17738658.0, "reward": 1.59375, "reward_std": 0.705165445804596, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.756024181842804, "sampling/importance_sampling_ratio/max": 1.0444221496582031, "sampling/importance_sampling_ratio/mean": 0.9922463297843933, "sampling/importance_sampling_ratio/min": 0.23264481127262115, "sampling/sampling_logp_difference/max": 1.496016025543213, "sampling/sampling_logp_difference/mean": 0.01600499637424946, "step": 364, "step_time": 13.988923765995423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 323.40625, "completions/mean_terminated_length": 323.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07162704155780375, "epoch": 0.0073, "frac_reward_zero_std": 0.0, "grad_norm": 0.2832232117652893, "kl": 1.6595613397657871, "learning_rate": 7.626667292802197e-06, "loss": -0.0677, "num_tokens": 17789658.0, "reward": 1.5, "reward_std": 1.0946159362792969, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.0776318311691284, "sampling/importance_sampling_ratio/max": 1.27531099319458, "sampling/importance_sampling_ratio/mean": 1.0071734189987183, "sampling/importance_sampling_ratio/min": 0.5733243823051453, "sampling/sampling_logp_difference/max": 0.5613565444946289, "sampling/sampling_logp_difference/mean": 0.016609370708465576, "step": 365, "step_time": 13.824653961972217 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 217.15625, "completions/mean_terminated_length": 217.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05460408120416105, "epoch": 0.00732, "frac_reward_zero_std": 0.0, "grad_norm": 0.5363184213638306, "kl": 0.6845191903412342, "learning_rate": 7.613989582131121e-06, "loss": -0.0473, "num_tokens": 17836214.0, "reward": 1.375, "reward_std": 0.6820099353790283, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.7931155562400818, "sampling/importance_sampling_ratio/max": 1.1939507722854614, "sampling/importance_sampling_ratio/mean": 1.0037026405334473, "sampling/importance_sampling_ratio/min": 0.335627019405365, "sampling/sampling_logp_difference/max": 1.0918567180633545, "sampling/sampling_logp_difference/mean": 0.019719406962394714, "step": 366, "step_time": 14.41802060502232 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 309.875, "completions/mean_terminated_length": 309.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07386370748281479, "epoch": 0.00734, "frac_reward_zero_std": 0.0, "grad_norm": 0.3642902076244354, "kl": 4.514974342659116, "learning_rate": 7.6012938795153966e-06, "loss": -0.0858, "num_tokens": 17885228.0, "reward": 1.375, "reward_std": 1.3297853469848633, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.2889105081558228, "sampling/importance_sampling_ratio/max": 1.1710454225540161, "sampling/importance_sampling_ratio/mean": 0.9787076115608215, "sampling/importance_sampling_ratio/min": 0.37435048818588257, "sampling/sampling_logp_difference/max": 0.982170581817627, "sampling/sampling_logp_difference/mean": 0.030226096510887146, "step": 367, "step_time": 15.708497198007535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1715.0, "completions/max_terminated_length": 1715.0, "completions/mean_length": 270.5625, "completions/mean_terminated_length": 270.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08712287549860775, "epoch": 0.00736, "frac_reward_zero_std": 0.0, "grad_norm": 0.2666812837123871, "kl": 0.8209542240947485, "learning_rate": 7.588580352419923e-06, "loss": -0.0468, "num_tokens": 17932817.0, "reward": 1.25, "reward_std": 0.866815984249115, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9503819346427917, "sampling/importance_sampling_ratio/max": 1.1567319631576538, "sampling/importance_sampling_ratio/mean": 0.9895401000976562, "sampling/importance_sampling_ratio/min": 0.3360939621925354, "sampling/sampling_logp_difference/max": 1.1713030338287354, "sampling/sampling_logp_difference/mean": 0.026293519884347916, "step": 368, "step_time": 15.892519930988783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1679.0, "completions/max_terminated_length": 1679.0, "completions/mean_length": 240.90625, "completions/mean_terminated_length": 240.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04989067139104009, "epoch": 0.00738, "frac_reward_zero_std": 0.0, "grad_norm": 0.30835723876953125, "kl": 0.636130828410387, "learning_rate": 7.575849168544717e-06, "loss": -0.0735, "num_tokens": 17979448.0, "reward": 1.40625, "reward_std": 0.96531081199646, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.0429292917251587, "sampling/importance_sampling_ratio/max": 1.6251829862594604, "sampling/importance_sampling_ratio/mean": 1.0411250591278076, "sampling/importance_sampling_ratio/min": 1.0023727416992188, "sampling/sampling_logp_difference/max": 0.46709269285202026, "sampling/sampling_logp_difference/mean": 0.01116114016622305, "step": 369, "step_time": 16.617664346951642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 299.5, "completions/mean_terminated_length": 299.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.059734353329986334, "epoch": 0.0074, "frac_reward_zero_std": 0.0, "grad_norm": 0.4425124526023865, "kl": 0.6992466077208519, "learning_rate": 7.5631004958227e-06, "loss": -0.0369, "num_tokens": 18028145.0, "reward": 1.25, "reward_std": 0.9186146259307861, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9503819346427917, "sampling/importance_sampling_ratio/max": 1.1547449827194214, "sampling/importance_sampling_ratio/mean": 1.0100460052490234, "sampling/importance_sampling_ratio/min": 0.4965401887893677, "sampling/sampling_logp_difference/max": 0.6995611190795898, "sampling/sampling_logp_difference/mean": 0.014956122264266014, "step": 370, "step_time": 13.555447786973673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 317.46875, "completions/mean_terminated_length": 317.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07282940880395472, "epoch": 0.00742, "frac_reward_zero_std": 0.0, "grad_norm": 0.6647711396217346, "kl": 1.6282650250941515, "learning_rate": 7.550334502417483e-06, "loss": -0.0481, "num_tokens": 18078200.0, "reward": 1.25, "reward_std": 1.1551947593688965, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.2181423902511597, "sampling/importance_sampling_ratio/max": 1.0836889743804932, "sampling/importance_sampling_ratio/mean": 0.9906908273696899, "sampling/importance_sampling_ratio/min": 0.38107600808143616, "sampling/sampling_logp_difference/max": 0.9648261070251465, "sampling/sampling_logp_difference/mean": 0.02316080778837204, "step": 371, "step_time": 14.16658007701335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1823.0, "completions/max_terminated_length": 1823.0, "completions/mean_length": 441.5, "completions/mean_terminated_length": 441.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06296468060463667, "epoch": 0.00744, "frac_reward_zero_std": 0.0, "grad_norm": 0.08971308171749115, "kl": 0.7537894807755947, "learning_rate": 7.537551356721149e-06, "loss": -0.0763, "num_tokens": 18132919.0, "reward": 1.71875, "reward_std": 1.1035435199737549, "rewards/rollout_reward_func/mean": 1.71875, "rewards/rollout_reward_func/std": 1.1139692068099976, "sampling/importance_sampling_ratio/max": 1.1770190000534058, "sampling/importance_sampling_ratio/mean": 1.0408624410629272, "sampling/importance_sampling_ratio/min": 1.0096709728240967, "sampling/sampling_logp_difference/max": 0.12226465344429016, "sampling/sampling_logp_difference/mean": 0.009761339984834194, "step": 372, "step_time": 17.304689931013854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 332.5, "completions/mean_terminated_length": 332.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08048009895719588, "epoch": 0.00746, "frac_reward_zero_std": 0.0, "grad_norm": 0.461036741733551, "kl": 2.5371130611747503, "learning_rate": 7.5247512273520325e-06, "loss": -0.0726, "num_tokens": 18183116.0, "reward": 1.21875, "reward_std": 1.3106391429901123, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.2885193824768066, "sampling/importance_sampling_ratio/max": 1.3269292116165161, "sampling/importance_sampling_ratio/mean": 0.9630589485168457, "sampling/importance_sampling_ratio/min": 0.3704884946346283, "sampling/sampling_logp_difference/max": 0.9929574728012085, "sampling/sampling_logp_difference/mean": 0.051928598433732986, "step": 373, "step_time": 15.224629075964913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 222.5, "completions/mean_terminated_length": 222.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08185886172577739, "epoch": 0.00748, "frac_reward_zero_std": 0.0, "grad_norm": 0.16760827600955963, "kl": 0.6157350987195969, "learning_rate": 7.511934283152491e-06, "loss": -0.0361, "num_tokens": 18230035.0, "reward": 1.25, "reward_std": 0.924098789691925, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9158109426498413, "sampling/importance_sampling_ratio/max": 1.328692078590393, "sampling/importance_sampling_ratio/mean": 1.0390639305114746, "sampling/importance_sampling_ratio/min": 1.0009406805038452, "sampling/sampling_logp_difference/max": 0.28316640853881836, "sampling/sampling_logp_difference/mean": 0.011687503196299076, "step": 374, "step_time": 14.163359410973499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 174.75, "completions/mean_terminated_length": 174.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10390859143808484, "epoch": 0.0075, "frac_reward_zero_std": 0.0, "grad_norm": 0.15506701171398163, "kl": 1.9637095760554075, "learning_rate": 7.499100693186684e-06, "loss": -0.0427, "num_tokens": 18272495.0, "reward": 1.0, "reward_std": 0.9902001023292542, "rewards/rollout_reward_func/mean": 1.0, "rewards/rollout_reward_func/std": 0.9837387204170227, "sampling/importance_sampling_ratio/max": 1.1731393337249756, "sampling/importance_sampling_ratio/mean": 0.9673970937728882, "sampling/importance_sampling_ratio/min": 0.2907693386077881, "sampling/sampling_logp_difference/max": 1.0671815872192383, "sampling/sampling_logp_difference/mean": 0.056237950921058655, "step": 375, "step_time": 11.63287172895798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 340.0625, "completions/mean_terminated_length": 340.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08101053419522941, "epoch": 0.00752, "frac_reward_zero_std": 0.0, "grad_norm": 0.586043655872345, "kl": 1.249971805140376, "learning_rate": 7.486250626738338e-06, "loss": -0.0703, "num_tokens": 18324006.0, "reward": 1.59375, "reward_std": 0.9967105388641357, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.011526346206665, "sampling/importance_sampling_ratio/max": 1.1510957479476929, "sampling/importance_sampling_ratio/mean": 0.9871293306350708, "sampling/importance_sampling_ratio/min": 0.42993053793907166, "sampling/sampling_logp_difference/max": 0.8445301055908203, "sampling/sampling_logp_difference/mean": 0.02707456424832344, "step": 376, "step_time": 14.995038006003597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 409.1875, "completions/mean_terminated_length": 409.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0802356037311256, "epoch": 0.00754, "frac_reward_zero_std": 0.0, "grad_norm": 0.4572793245315552, "kl": 0.7445751056075096, "learning_rate": 7.473384253308518e-06, "loss": -0.0707, "num_tokens": 18377374.0, "reward": 1.71875, "reward_std": 0.9101955890655518, "rewards/rollout_reward_func/mean": 1.71875, "rewards/rollout_reward_func/std": 0.9240295886993408, "sampling/importance_sampling_ratio/max": 2.3642258644104004, "sampling/importance_sampling_ratio/mean": 1.110743761062622, "sampling/importance_sampling_ratio/min": 1.0039225816726685, "sampling/sampling_logp_difference/max": 0.850750207901001, "sampling/sampling_logp_difference/mean": 0.02170286513864994, "step": 377, "step_time": 14.180610168987187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 442.46875, "completions/mean_terminated_length": 442.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06432010093703866, "epoch": 0.00756, "frac_reward_zero_std": 0.0, "grad_norm": 0.13079342246055603, "kl": 1.0286617502570152, "learning_rate": 7.460501742613385e-06, "loss": -0.0676, "num_tokens": 18432034.0, "reward": 1.75, "reward_std": 0.8575467467308044, "rewards/rollout_reward_func/mean": 1.75, "rewards/rollout_reward_func/std": 0.8424234986305237, "sampling/importance_sampling_ratio/max": 1.334968090057373, "sampling/importance_sampling_ratio/mean": 0.998167872428894, "sampling/importance_sampling_ratio/min": 0.2364993393421173, "sampling/sampling_logp_difference/max": 0.8627514839172363, "sampling/sampling_logp_difference/mean": 0.026404447853565216, "step": 378, "step_time": 15.86538246597047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 350.90625, "completions/mean_terminated_length": 350.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08740552421659231, "epoch": 0.00758, "frac_reward_zero_std": 0.0, "grad_norm": 0.5863131284713745, "kl": 1.1519179865717888, "learning_rate": 7.447603264581964e-06, "loss": -0.0611, "num_tokens": 18483843.0, "reward": 1.375, "reward_std": 1.1516191959381104, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.1570261716842651, "sampling/importance_sampling_ratio/max": 1.3544632196426392, "sampling/importance_sampling_ratio/mean": 1.0121811628341675, "sampling/importance_sampling_ratio/min": 0.3431219458580017, "sampling/sampling_logp_difference/max": 1.0696783065795898, "sampling/sampling_logp_difference/mean": 0.0224886704236269, "step": 379, "step_time": 13.804151917021954 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1373.0, "completions/max_terminated_length": 1373.0, "completions/mean_length": 309.0, "completions/mean_terminated_length": 309.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10355916025582701, "epoch": 0.0076, "frac_reward_zero_std": 0.0, "grad_norm": 0.12012919038534164, "kl": 1.022422886453569, "learning_rate": 7.4346889893539e-06, "loss": -0.0708, "num_tokens": 18534280.0, "reward": 1.5625, "reward_std": 1.2314343452453613, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 1.2164862155914307, "sampling/importance_sampling_ratio/max": 1.1331866979599, "sampling/importance_sampling_ratio/mean": 0.9772738218307495, "sampling/importance_sampling_ratio/min": 2.8295521481425112e-09, "sampling/sampling_logp_difference/max": 17.451433181762695, "sampling/sampling_logp_difference/mean": 0.17286992073059082, "step": 380, "step_time": 15.11991430701164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 368.03125, "completions/mean_terminated_length": 368.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06458626734092832, "epoch": 0.00762, "frac_reward_zero_std": 0.0, "grad_norm": 0.1631869077682495, "kl": 0.7273617256432772, "learning_rate": 7.421759087277214e-06, "loss": -0.0654, "num_tokens": 18586755.0, "reward": 1.75, "reward_std": 0.7939802408218384, "rewards/rollout_reward_func/mean": 1.75, "rewards/rollout_reward_func/std": 0.8424234986305237, "sampling/importance_sampling_ratio/max": 1.1977448463439941, "sampling/importance_sampling_ratio/mean": 1.0340975522994995, "sampling/importance_sampling_ratio/min": 1.0018922090530396, "sampling/sampling_logp_difference/max": 0.1477850377559662, "sampling/sampling_logp_difference/mean": 0.009034901857376099, "step": 381, "step_time": 14.01576771303371 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1688.0, "completions/max_terminated_length": 1688.0, "completions/mean_length": 436.59375, "completions/mean_terminated_length": 436.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05943168560042977, "epoch": 0.00764, "frac_reward_zero_std": 0.0, "grad_norm": 0.40648406744003296, "kl": 0.9123413823544979, "learning_rate": 7.408813728906053e-06, "loss": -0.0874, "num_tokens": 18641929.0, "reward": 1.75, "reward_std": 1.015371561050415, "rewards/rollout_reward_func/mean": 1.75, "rewards/rollout_reward_func/std": 1.0776318311691284, "sampling/importance_sampling_ratio/max": 1.28948974609375, "sampling/importance_sampling_ratio/mean": 1.019989252090454, "sampling/importance_sampling_ratio/min": 0.43548908829689026, "sampling/sampling_logp_difference/max": 0.8453783988952637, "sampling/sampling_logp_difference/mean": 0.01566363126039505, "step": 382, "step_time": 16.264249066982302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1697.0, "completions/max_terminated_length": 1697.0, "completions/mean_length": 308.75, "completions/mean_terminated_length": 308.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06072766555007547, "epoch": 0.00766, "frac_reward_zero_std": 0.0, "grad_norm": 0.2981581389904022, "kl": 2.3944799788296223, "learning_rate": 7.395853084998448e-06, "loss": -0.058, "num_tokens": 18691984.0, "reward": 1.3125, "reward_std": 1.100199818611145, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.0906493663787842, "sampling/importance_sampling_ratio/max": 1.2276467084884644, "sampling/importance_sampling_ratio/mean": 0.9868571758270264, "sampling/importance_sampling_ratio/min": 0.20840640366077423, "sampling/sampling_logp_difference/max": 1.571047067642212, "sampling/sampling_logp_difference/mean": 0.02926461212337017, "step": 383, "step_time": 16.5876753909979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 423.71875, "completions/mean_terminated_length": 423.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0702296826057136, "epoch": 0.00768, "frac_reward_zero_std": 0.0, "grad_norm": 1.3215219974517822, "kl": 3.118452152237296, "learning_rate": 7.382877326514051e-06, "loss": -0.0641, "num_tokens": 18746147.0, "reward": 1.6875, "reward_std": 0.9402616024017334, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.9979817867279053, "sampling/importance_sampling_ratio/max": 1.139930248260498, "sampling/importance_sampling_ratio/mean": 0.9584792256355286, "sampling/importance_sampling_ratio/min": 0.15771639347076416, "sampling/sampling_logp_difference/max": 1.8542470932006836, "sampling/sampling_logp_difference/mean": 0.03584183752536774, "step": 384, "step_time": 16.105189620007877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 307.09375, "completions/mean_terminated_length": 307.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04926766571588814, "epoch": 0.0077, "frac_reward_zero_std": 0.0, "grad_norm": 0.775784969329834, "kl": 6.796053307130933, "learning_rate": 7.369886624611889e-06, "loss": -0.0642, "num_tokens": 18795743.0, "reward": 1.3125, "reward_std": 1.0992558002471924, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.1760376691818237, "sampling/importance_sampling_ratio/max": 1.0845770835876465, "sampling/importance_sampling_ratio/mean": 0.9813372492790222, "sampling/importance_sampling_ratio/min": 0.26329854130744934, "sampling/sampling_logp_difference/max": 1.3345108032226562, "sampling/sampling_logp_difference/mean": 0.025495007634162903, "step": 385, "step_time": 15.455622994952137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 185.6875, "completions/mean_terminated_length": 185.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07437512627802789, "epoch": 0.00772, "frac_reward_zero_std": 0.0, "grad_norm": 0.10191413015127182, "kl": 2.376024032011628, "learning_rate": 7.356881150648103e-06, "loss": -0.0516, "num_tokens": 18840228.0, "reward": 1.3125, "reward_std": 0.8284125328063965, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.8957785964012146, "sampling/importance_sampling_ratio/max": 1.1613572835922241, "sampling/importance_sampling_ratio/mean": 0.9882324934005737, "sampling/importance_sampling_ratio/min": 0.30752918124198914, "sampling/sampling_logp_difference/max": 1.1791870594024658, "sampling/sampling_logp_difference/mean": 0.03262249007821083, "step": 386, "step_time": 14.470769041014137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2169.0, "completions/max_terminated_length": 2169.0, "completions/mean_length": 328.125, "completions/mean_terminated_length": 328.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05693867732770741, "epoch": 0.00774, "frac_reward_zero_std": 0.0, "grad_norm": 0.4174410402774811, "kl": 3.6509500136598945, "learning_rate": 7.343861076173684e-06, "loss": -0.0406, "num_tokens": 18890393.0, "reward": 1.4375, "reward_std": 1.2426517009735107, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.268412709236145, "sampling/importance_sampling_ratio/max": 1.1159828901290894, "sampling/importance_sampling_ratio/mean": 0.9407739639282227, "sampling/importance_sampling_ratio/min": 0.21665342152118683, "sampling/sampling_logp_difference/max": 1.5546143054962158, "sampling/sampling_logp_difference/mean": 0.0472775399684906, "step": 387, "step_time": 18.810720365057932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 400.59375, "completions/mean_terminated_length": 400.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07897469797171652, "epoch": 0.00776, "frac_reward_zero_std": 0.0, "grad_norm": 0.3611973822116852, "kl": 0.6084265522658825, "learning_rate": 7.330826572932217e-06, "loss": -0.075, "num_tokens": 18944818.0, "reward": 1.6875, "reward_std": 1.1097307205200195, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 1.1482806205749512, "sampling/importance_sampling_ratio/max": 1.3717613220214844, "sampling/importance_sampling_ratio/mean": 1.0290247201919556, "sampling/importance_sampling_ratio/min": 0.7184183597564697, "sampling/sampling_logp_difference/max": 0.3307483196258545, "sampling/sampling_logp_difference/mean": 0.011360364966094494, "step": 388, "step_time": 15.692212683934486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 433.0625, "completions/mean_terminated_length": 433.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06064801779575646, "epoch": 0.00778, "frac_reward_zero_std": 0.0, "grad_norm": 0.19888167083263397, "kl": 0.6257044682279229, "learning_rate": 7.3177778128576124e-06, "loss": -0.0657, "num_tokens": 19000237.0, "reward": 1.90625, "reward_std": 0.7141392827033997, "rewards/rollout_reward_func/mean": 1.90625, "rewards/rollout_reward_func/std": 0.7343803644180298, "sampling/importance_sampling_ratio/max": 1.138800024986267, "sampling/importance_sampling_ratio/mean": 1.022121548652649, "sampling/importance_sampling_ratio/min": 0.7955330014228821, "sampling/sampling_logp_difference/max": 0.22858214378356934, "sampling/sampling_logp_difference/mean": 0.008351555094122887, "step": 389, "step_time": 17.10578300897032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 269.4375, "completions/mean_terminated_length": 269.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06448594713583589, "epoch": 0.0078, "frac_reward_zero_std": 0.0, "grad_norm": 0.2729074954986572, "kl": 2.354939538985491, "learning_rate": 7.304714968071834e-06, "loss": -0.0544, "num_tokens": 19048006.0, "reward": 1.25, "reward_std": 1.0953571796417236, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.1071614027023315, "sampling/importance_sampling_ratio/max": 1.1142116785049438, "sampling/importance_sampling_ratio/mean": 0.9840567111968994, "sampling/importance_sampling_ratio/min": 0.33245083689689636, "sampling/sampling_logp_difference/max": 1.101069450378418, "sampling/sampling_logp_difference/mean": 0.02371019311249256, "step": 390, "step_time": 13.794532965999679 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1787.0, "completions/max_terminated_length": 1787.0, "completions/mean_length": 357.09375, "completions/mean_terminated_length": 357.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07859957660548389, "epoch": 0.00782, "frac_reward_zero_std": 0.0, "grad_norm": 0.19257037341594696, "kl": 2.7370558213442564, "learning_rate": 7.291638210882638e-06, "loss": -0.0954, "num_tokens": 19099511.0, "reward": 1.5625, "reward_std": 1.1227459907531738, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 1.2427207231521606, "sampling/importance_sampling_ratio/max": 1.090930461883545, "sampling/importance_sampling_ratio/mean": 0.9328962564468384, "sampling/importance_sampling_ratio/min": 0.28068146109580994, "sampling/sampling_logp_difference/max": 1.2754381895065308, "sampling/sampling_logp_difference/mean": 0.04392492398619652, "step": 391, "step_time": 16.738205485991784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 276.625, "completions/mean_terminated_length": 276.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10267557227052748, "epoch": 0.00784, "frac_reward_zero_std": 0.0, "grad_norm": 0.6116975545883179, "kl": 0.5846525188535452, "learning_rate": 7.278547713781288e-06, "loss": -0.0237, "num_tokens": 19148239.0, "reward": 1.1875, "reward_std": 0.9780248403549194, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.029797911643982, "sampling/importance_sampling_ratio/max": 1.276092767715454, "sampling/importance_sampling_ratio/mean": 1.0199925899505615, "sampling/importance_sampling_ratio/min": 0.20627310872077942, "sampling/sampling_logp_difference/max": 0.9592505693435669, "sampling/sampling_logp_difference/mean": 0.022838229313492775, "step": 392, "step_time": 14.64835872297408 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 188.34375, "completions/mean_terminated_length": 188.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06509832548908889, "epoch": 0.00786, "frac_reward_zero_std": 0.0, "grad_norm": 0.34326067566871643, "kl": 0.6118893679231405, "learning_rate": 7.2654436494402955e-06, "loss": -0.0558, "num_tokens": 19193053.0, "reward": 1.40625, "reward_std": 0.7445083856582642, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.756024181842804, "sampling/importance_sampling_ratio/max": 1.1172587871551514, "sampling/importance_sampling_ratio/mean": 1.014183521270752, "sampling/importance_sampling_ratio/min": 0.748864471912384, "sampling/sampling_logp_difference/max": 0.30338793992996216, "sampling/sampling_logp_difference/mean": 0.010256921872496605, "step": 393, "step_time": 12.296690247021616 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 865.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 171.0625, "completions/mean_terminated_length": 171.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08541043312288821, "epoch": 0.00788, "frac_reward_zero_std": 0.0, "grad_norm": 0.10366006195545197, "kl": 0.676116032525897, "learning_rate": 7.252326190711121e-06, "loss": -0.0499, "num_tokens": 19237224.0, "reward": 1.0625, "reward_std": 0.9224802851676941, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.0453429222106934, "sampling/importance_sampling_ratio/max": 1.1793513298034668, "sampling/importance_sampling_ratio/mean": 1.02058744430542, "sampling/importance_sampling_ratio/min": 0.5797234773635864, "sampling/sampling_logp_difference/max": 0.5453404188156128, "sampling/sampling_logp_difference/mean": 0.01602039486169815, "step": 394, "step_time": 13.330337930019596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 355.78125, "completions/mean_terminated_length": 355.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06259494950063527, "epoch": 0.0079, "frac_reward_zero_std": 0.0, "grad_norm": 0.5608261823654175, "kl": 1.0635932423174381, "learning_rate": 7.239195510621918e-06, "loss": -0.0805, "num_tokens": 19288852.0, "reward": 1.59375, "reward_std": 1.2448524236679077, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.2406911849975586, "sampling/importance_sampling_ratio/max": 1.2798669338226318, "sampling/importance_sampling_ratio/mean": 1.0132962465286255, "sampling/importance_sampling_ratio/min": 0.6339022517204285, "sampling/sampling_logp_difference/max": 0.4956474304199219, "sampling/sampling_logp_difference/mean": 0.013314342126250267, "step": 395, "step_time": 16.110190326013253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 326.03125, "completions/mean_terminated_length": 326.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06446739984676242, "epoch": 0.00792, "frac_reward_zero_std": 0.0, "grad_norm": 0.20436976850032806, "kl": 2.9734915364533663, "learning_rate": 7.22605178237523e-06, "loss": -0.0693, "num_tokens": 19339312.0, "reward": 1.59375, "reward_std": 0.9545402526855469, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.9791166186332703, "sampling/importance_sampling_ratio/max": 1.1033490896224976, "sampling/importance_sampling_ratio/mean": 1.0018994808197021, "sampling/importance_sampling_ratio/min": 0.32897716760635376, "sampling/sampling_logp_difference/max": 1.1117687225341797, "sampling/sampling_logp_difference/mean": 0.018591543659567833, "step": 396, "step_time": 15.68712431404856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 290.03125, "completions/mean_terminated_length": 290.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04743219446390867, "epoch": 0.00794, "frac_reward_zero_std": 0.0, "grad_norm": 0.16274887323379517, "kl": 1.6597296418622136, "learning_rate": 7.212895179345718e-06, "loss": -0.0655, "num_tokens": 19388404.0, "reward": 1.65625, "reward_std": 0.7432413101196289, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.7873751521110535, "sampling/importance_sampling_ratio/max": 1.0643976926803589, "sampling/importance_sampling_ratio/mean": 1.002095103263855, "sampling/importance_sampling_ratio/min": 0.3723677694797516, "sampling/sampling_logp_difference/max": 0.9898953437805176, "sampling/sampling_logp_difference/mean": 0.015168091282248497, "step": 397, "step_time": 15.861692431033589 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1607.0, "completions/max_terminated_length": 1607.0, "completions/mean_length": 323.1875, "completions/mean_terminated_length": 323.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.052929720375686884, "epoch": 0.00796, "frac_reward_zero_std": 0.0, "grad_norm": 0.24755342304706573, "kl": 0.9253554902970791, "learning_rate": 7.199725875077871e-06, "loss": -0.0434, "num_tokens": 19439769.0, "reward": 1.65625, "reward_std": 0.8998414874076843, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 1.0035220384597778, "sampling/importance_sampling_ratio/max": 1.1585965156555176, "sampling/importance_sampling_ratio/mean": 1.0033199787139893, "sampling/importance_sampling_ratio/min": 0.3170027434825897, "sampling/sampling_logp_difference/max": 1.188420057296753, "sampling/sampling_logp_difference/mean": 0.01711221970617771, "step": 398, "step_time": 16.360832645033952 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 325.3125, "completions/mean_terminated_length": 325.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04721159779001027, "epoch": 0.00798, "frac_reward_zero_std": 0.0, "grad_norm": 0.09079373627901077, "kl": 0.8802452087402344, "learning_rate": 7.186544043283715e-06, "loss": -0.0724, "num_tokens": 19489791.0, "reward": 1.625, "reward_std": 1.0153720378875732, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 1.099853277206421, "sampling/importance_sampling_ratio/max": 1.0618900060653687, "sampling/importance_sampling_ratio/mean": 0.9955589771270752, "sampling/importance_sampling_ratio/min": 0.19505809247493744, "sampling/sampling_logp_difference/max": 1.6380505561828613, "sampling/sampling_logp_difference/mean": 0.019351262599229813, "step": 399, "step_time": 15.04261480497371 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 255.90625, "completions/mean_terminated_length": 255.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07912093843333423, "epoch": 0.008, "frac_reward_zero_std": 0.0, "grad_norm": 0.3353623151779175, "kl": 4.350010893307626, "learning_rate": 7.173349857840521e-06, "loss": -0.0343, "num_tokens": 19538159.0, "reward": 0.875, "reward_std": 1.078991174697876, "rewards/rollout_reward_func/mean": 0.875, "rewards/rollout_reward_func/std": 1.0701221227645874, "sampling/importance_sampling_ratio/max": 1.1283812522888184, "sampling/importance_sampling_ratio/mean": 0.9178875684738159, "sampling/importance_sampling_ratio/min": 0.35321134328842163, "sampling/sampling_logp_difference/max": 1.0407066345214844, "sampling/sampling_logp_difference/mean": 0.05246298015117645, "step": 400, "step_time": 14.391940836998401 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 291.71875, "completions/mean_terminated_length": 291.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07076037023216486, "epoch": 0.00802, "frac_reward_zero_std": 0.0, "grad_norm": 0.45128124952316284, "kl": 1.0078845620155334, "learning_rate": 7.160143492788516e-06, "loss": -0.0815, "num_tokens": 19587983.0, "reward": 1.59375, "reward_std": 0.8476226329803467, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.9108441472053528, "sampling/importance_sampling_ratio/max": 1.208299160003662, "sampling/importance_sampling_ratio/mean": 0.9686658382415771, "sampling/importance_sampling_ratio/min": 0.4372366666793823, "sampling/sampling_logp_difference/max": 0.827120304107666, "sampling/sampling_logp_difference/mean": 0.025795988738536835, "step": 401, "step_time": 15.108031774987467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 227.6875, "completions/mean_terminated_length": 227.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07114814408123493, "epoch": 0.00804, "frac_reward_zero_std": 0.0, "grad_norm": 0.16125835478305817, "kl": 0.6812199344858527, "learning_rate": 7.146925122328581e-06, "loss": -0.0507, "num_tokens": 19634359.0, "reward": 1.40625, "reward_std": 0.771014392375946, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.797551691532135, "sampling/importance_sampling_ratio/max": 1.1388535499572754, "sampling/importance_sampling_ratio/mean": 1.0198423862457275, "sampling/importance_sampling_ratio/min": 0.944155216217041, "sampling/sampling_logp_difference/max": 0.12269306182861328, "sampling/sampling_logp_difference/mean": 0.007214098237454891, "step": 402, "step_time": 12.44958048996341 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 2223.0, "completions/max_terminated_length": 2223.0, "completions/mean_length": 409.875, "completions/mean_terminated_length": 409.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.050951665034517646, "epoch": 0.00806, "frac_reward_zero_std": 0.0, "grad_norm": 0.6197277307510376, "kl": 1.4313660562038422, "learning_rate": 7.133694920819958e-06, "loss": -0.1437, "num_tokens": 19688182.0, "reward": 1.75, "reward_std": 0.9700947999954224, "rewards/rollout_reward_func/mean": 1.75, "rewards/rollout_reward_func/std": 1.0160009860992432, "sampling/importance_sampling_ratio/max": 2.4184460639953613, "sampling/importance_sampling_ratio/mean": 1.0222516059875488, "sampling/importance_sampling_ratio/min": 0.3756645619869232, "sampling/sampling_logp_difference/max": 0.9962835311889648, "sampling/sampling_logp_difference/mean": 0.027807416394352913, "step": 403, "step_time": 19.880305527010933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2205.0, "completions/max_terminated_length": 2205.0, "completions/mean_length": 404.125, "completions/mean_terminated_length": 404.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0565129267051816, "epoch": 0.00808, "frac_reward_zero_std": 0.0, "grad_norm": 0.42273297905921936, "kl": 1.5321689322590828, "learning_rate": 7.12045306277795e-06, "loss": -0.0722, "num_tokens": 19741515.0, "reward": 1.59375, "reward_std": 0.9594908952713013, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.9455967545509338, "sampling/importance_sampling_ratio/max": 1.1296184062957764, "sampling/importance_sampling_ratio/mean": 0.9788661003112793, "sampling/importance_sampling_ratio/min": 0.23237344622612, "sampling/sampling_logp_difference/max": 1.4872751235961914, "sampling/sampling_logp_difference/mean": 0.027316372841596603, "step": 404, "step_time": 19.10529418401711 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 335.3125, "completions/mean_terminated_length": 335.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06293152854777873, "epoch": 0.0081, "frac_reward_zero_std": 0.0, "grad_norm": 0.07579856365919113, "kl": 1.2067679092288017, "learning_rate": 7.107199722871614e-06, "loss": -0.0498, "num_tokens": 19791816.0, "reward": 1.21875, "reward_std": 1.143191933631897, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.2110878229141235, "sampling/importance_sampling_ratio/max": 1.1458475589752197, "sampling/importance_sampling_ratio/mean": 0.9781112670898438, "sampling/importance_sampling_ratio/min": 0.35325849056243896, "sampling/sampling_logp_difference/max": 1.0407252311706543, "sampling/sampling_logp_difference/mean": 0.031181257218122482, "step": 405, "step_time": 15.879743326004245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 360.75, "completions/mean_terminated_length": 360.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.051042694831267, "epoch": 0.00812, "frac_reward_zero_std": 0.0, "grad_norm": 0.24192388355731964, "kl": 0.7769069038331509, "learning_rate": 7.093935075921465e-06, "loss": -0.0786, "num_tokens": 19844036.0, "reward": 1.6875, "reward_std": 0.8917239904403687, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.8957785964012146, "sampling/importance_sampling_ratio/max": 1.0469380617141724, "sampling/importance_sampling_ratio/mean": 0.992263674736023, "sampling/importance_sampling_ratio/min": 0.26129621267318726, "sampling/sampling_logp_difference/max": 1.3514559268951416, "sampling/sampling_logp_difference/mean": 0.01741056703031063, "step": 406, "step_time": 15.066514829013613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 339.375, "completions/mean_terminated_length": 339.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.054846562910825014, "epoch": 0.00814, "frac_reward_zero_std": 0.0, "grad_norm": 0.1827344298362732, "kl": 3.7017474183812737, "learning_rate": 7.0806592968971624e-06, "loss": -0.0524, "num_tokens": 19895113.0, "reward": 1.625, "reward_std": 0.8386746644973755, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.9069623351097107, "sampling/importance_sampling_ratio/max": 1.1315696239471436, "sampling/importance_sampling_ratio/mean": 1.0170458555221558, "sampling/importance_sampling_ratio/min": 0.61131352186203, "sampling/sampling_logp_difference/max": 0.492124080657959, "sampling/sampling_logp_difference/mean": 0.012506188824772835, "step": 407, "step_time": 13.892732247972162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 338.1875, "completions/mean_terminated_length": 338.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06914574559777975, "epoch": 0.00816, "frac_reward_zero_std": 0.0, "grad_norm": 0.19219359755516052, "kl": 0.7822239585220814, "learning_rate": 7.067372560915205e-06, "loss": -0.0755, "num_tokens": 19944953.0, "reward": 1.65625, "reward_std": 0.7987453937530518, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.8273305296897888, "sampling/importance_sampling_ratio/max": 1.1841861009597778, "sampling/importance_sampling_ratio/mean": 1.0259826183319092, "sampling/importance_sampling_ratio/min": 0.969458818435669, "sampling/sampling_logp_difference/max": 0.12475016713142395, "sampling/sampling_logp_difference/mean": 0.00799620896577835, "step": 408, "step_time": 14.343620349027333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 316.0625, "completions/mean_terminated_length": 325.7419128417969, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1290102646453306, "epoch": 0.00818, "frac_reward_zero_std": 0.0, "grad_norm": 0.8014764785766602, "kl": 6.350228000432253, "learning_rate": 7.054075043236623e-06, "loss": -0.0285, "num_tokens": 19995234.0, "reward": 1.46875, "reward_std": 1.136181116104126, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.1635422706604004, "sampling/importance_sampling_ratio/max": 1.0849379301071167, "sampling/importance_sampling_ratio/mean": 0.9895661473274231, "sampling/importance_sampling_ratio/min": 0.610192596912384, "sampling/sampling_logp_difference/max": 0.4940352439880371, "sampling/sampling_logp_difference/mean": 0.017016444355249405, "step": 409, "step_time": 13.219443665962899 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 220.5625, "completions/mean_terminated_length": 227.16128540039062, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09832175681367517, "epoch": 0.0082, "frac_reward_zero_std": 0.0, "grad_norm": 0.22410769760608673, "kl": 3.9329519160091877, "learning_rate": 7.040766919264664e-06, "loss": -0.0268, "num_tokens": 20042305.0, "reward": 1.03125, "reward_std": 0.9010239839553833, "rewards/rollout_reward_func/mean": 1.03125, "rewards/rollout_reward_func/std": 0.9327162504196167, "sampling/importance_sampling_ratio/max": 1.0567063093185425, "sampling/importance_sampling_ratio/mean": 0.922265887260437, "sampling/importance_sampling_ratio/min": 0.26068127155303955, "sampling/sampling_logp_difference/max": 1.3573598861694336, "sampling/sampling_logp_difference/mean": 0.04878237470984459, "step": 410, "step_time": 12.531523333018413 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.03125, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 310.875, "completions/mean_terminated_length": 320.3870849609375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11570011358708143, "epoch": 0.00822, "frac_reward_zero_std": 0.0, "grad_norm": 0.48577985167503357, "kl": 3.878637148067355, "learning_rate": 7.027448364542479e-06, "loss": -0.0102, "num_tokens": 20092492.0, "reward": 1.4375, "reward_std": 0.8430415391921997, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9136068224906921, "sampling/importance_sampling_ratio/max": 1.436326026916504, "sampling/importance_sampling_ratio/mean": 1.0094215869903564, "sampling/importance_sampling_ratio/min": 0.32537537813186646, "sampling/sampling_logp_difference/max": 1.1227169036865234, "sampling/sampling_logp_difference/mean": 0.023349016904830933, "step": 411, "step_time": 14.874941909001791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 293.375, "completions/mean_terminated_length": 293.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.057414278388023376, "epoch": 0.00824, "frac_reward_zero_std": 0.0, "grad_norm": 0.2626599669456482, "kl": 1.5605944339185953, "learning_rate": 7.01411955475081e-06, "loss": -0.0352, "num_tokens": 20142645.0, "reward": 1.53125, "reward_std": 0.7457714080810547, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.8025915622711182, "sampling/importance_sampling_ratio/max": 1.0773483514785767, "sampling/importance_sampling_ratio/mean": 0.9825709462165833, "sampling/importance_sampling_ratio/min": 0.21452949941158295, "sampling/sampling_logp_difference/max": 1.5407490730285645, "sampling/sampling_logp_difference/mean": 0.02620694786310196, "step": 412, "step_time": 13.985488432037528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1211.0, "completions/max_terminated_length": 1211.0, "completions/mean_length": 285.125, "completions/mean_terminated_length": 285.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1252192386891693, "epoch": 0.00826, "frac_reward_zero_std": 0.0, "grad_norm": 0.26714327931404114, "kl": 0.8070348799228668, "learning_rate": 7.000780665705665e-06, "loss": -0.0458, "num_tokens": 20190741.0, "reward": 1.28125, "reward_std": 0.8600857257843018, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.8884337544441223, "sampling/importance_sampling_ratio/max": 1.1325948238372803, "sampling/importance_sampling_ratio/mean": 0.9806170463562012, "sampling/importance_sampling_ratio/min": 0.33450302481651306, "sampling/sampling_logp_difference/max": 1.1065421104431152, "sampling/sampling_logp_difference/mean": 0.028341183438897133, "step": 413, "step_time": 14.214297995014931 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 308.03125, "completions/mean_terminated_length": 308.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06275781139265746, "epoch": 0.00828, "frac_reward_zero_std": 0.0, "grad_norm": 0.3393501043319702, "kl": 0.7716346047818661, "learning_rate": 6.987431873356011e-06, "loss": -0.0517, "num_tokens": 20240535.0, "reward": 1.78125, "reward_std": 0.6768368482589722, "rewards/rollout_reward_func/mean": 1.78125, "rewards/rollout_reward_func/std": 0.7063935995101929, "sampling/importance_sampling_ratio/max": 1.136718511581421, "sampling/importance_sampling_ratio/mean": 1.0052387714385986, "sampling/importance_sampling_ratio/min": 0.2480352371931076, "sampling/sampling_logp_difference/max": 0.9178956747055054, "sampling/sampling_logp_difference/mean": 0.01902475208044052, "step": 414, "step_time": 15.666271780020907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 344.3125, "completions/mean_terminated_length": 344.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06271873624064028, "epoch": 0.0083, "frac_reward_zero_std": 0.0, "grad_norm": 0.12716588377952576, "kl": 0.6197800887748599, "learning_rate": 6.97407335378144e-06, "loss": -0.0611, "num_tokens": 20291289.0, "reward": 1.65625, "reward_std": 0.8321393728256226, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.8654431700706482, "sampling/importance_sampling_ratio/max": 1.0624135732650757, "sampling/importance_sampling_ratio/mean": 1.0274181365966797, "sampling/importance_sampling_ratio/min": 1.0020767450332642, "sampling/sampling_logp_difference/max": 0.059509895741939545, "sampling/sampling_logp_difference/mean": 0.007581651210784912, "step": 415, "step_time": 14.287351879975176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 414.125, "completions/mean_terminated_length": 414.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07234500627964735, "epoch": 0.00832, "frac_reward_zero_std": 0.0, "grad_norm": 0.5090242624282837, "kl": 0.8026974815875292, "learning_rate": 6.960705283189857e-06, "loss": -0.0746, "num_tokens": 20345304.0, "reward": 1.75, "reward_std": 0.9365485906600952, "rewards/rollout_reward_func/mean": 1.75, "rewards/rollout_reward_func/std": 0.9158109426498413, "sampling/importance_sampling_ratio/max": 1.1114124059677124, "sampling/importance_sampling_ratio/mean": 0.9888192415237427, "sampling/importance_sampling_ratio/min": 0.4450640380382538, "sampling/sampling_logp_difference/max": 0.8094854354858398, "sampling/sampling_logp_difference/mean": 0.02056647464632988, "step": 416, "step_time": 14.024509964059689 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.020833333488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 313.375, "completions/mean_terminated_length": 313.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07047661836259067, "epoch": 0.00834, "frac_reward_zero_std": 0.0, "grad_norm": 0.14763323962688446, "kl": 1.0244530700147152, "learning_rate": 6.94732783791515e-06, "loss": -0.0722, "num_tokens": 20395943.0, "reward": 1.34375, "reward_std": 1.09218168258667, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.180742621421814, "sampling/importance_sampling_ratio/max": 1.281393051147461, "sampling/importance_sampling_ratio/mean": 0.9869333505630493, "sampling/importance_sampling_ratio/min": 0.4422774016857147, "sampling/sampling_logp_difference/max": 0.815683126449585, "sampling/sampling_logp_difference/mean": 0.026790954172611237, "step": 417, "step_time": 15.9139576429734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 284.59375, "completions/mean_terminated_length": 284.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06915555451996624, "epoch": 0.00836, "frac_reward_zero_std": 0.0, "grad_norm": 0.3290942311286926, "kl": 1.3998677525669336, "learning_rate": 6.933941194414866e-06, "loss": -0.0615, "num_tokens": 20444783.0, "reward": 1.3125, "reward_std": 1.0295413732528687, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.0606601238250732, "sampling/importance_sampling_ratio/max": 1.2126996517181396, "sampling/importance_sampling_ratio/mean": 0.9880244731903076, "sampling/importance_sampling_ratio/min": 0.38371679186820984, "sampling/sampling_logp_difference/max": 0.9595072269439697, "sampling/sampling_logp_difference/mean": 0.026356248185038567, "step": 418, "step_time": 13.58709024003474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 838.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 241.875, "completions/mean_terminated_length": 241.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0728802738012746, "epoch": 0.00838, "frac_reward_zero_std": 0.0, "grad_norm": 0.3221737742424011, "kl": 3.513235479593277, "learning_rate": 6.920545529267882e-06, "loss": -0.0725, "num_tokens": 20491852.0, "reward": 1.1875, "reward_std": 1.1068898439407349, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.1198358535766602, "sampling/importance_sampling_ratio/max": 2.1024179458618164, "sampling/importance_sampling_ratio/mean": 1.01142156124115, "sampling/importance_sampling_ratio/min": 0.37603604793548584, "sampling/sampling_logp_difference/max": 0.9780936241149902, "sampling/sampling_logp_difference/mean": 0.03414604812860489, "step": 419, "step_time": 12.149869143016986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2025.0, "completions/max_terminated_length": 2025.0, "completions/mean_length": 415.15625, "completions/mean_terminated_length": 415.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07774914114270359, "epoch": 0.0084, "frac_reward_zero_std": 0.0, "grad_norm": 0.4862137734889984, "kl": 2.1473714765161276, "learning_rate": 6.907141019172076e-06, "loss": -0.0177, "num_tokens": 20546466.0, "reward": 1.53125, "reward_std": 0.9099595546722412, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.949851393699646, "sampling/importance_sampling_ratio/max": 1.0752414464950562, "sampling/importance_sampling_ratio/mean": 0.9978874325752258, "sampling/importance_sampling_ratio/min": 0.4818542003631592, "sampling/sampling_logp_difference/max": 0.7416019439697266, "sampling/sampling_logp_difference/mean": 0.014209765009582043, "step": 420, "step_time": 17.68969084796845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 386.59375, "completions/mean_terminated_length": 386.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07869170769117773, "epoch": 0.00842, "frac_reward_zero_std": 0.0, "grad_norm": 0.9726786017417908, "kl": 3.7160073672421277, "learning_rate": 6.893727840941997e-06, "loss": -0.0393, "num_tokens": 20598998.0, "reward": 1.3125, "reward_std": 0.9801014065742493, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.0606601238250732, "sampling/importance_sampling_ratio/max": 1.4154627323150635, "sampling/importance_sampling_ratio/mean": 0.9867216944694519, "sampling/importance_sampling_ratio/min": 0.20354434847831726, "sampling/sampling_logp_difference/max": 1.5986597537994385, "sampling/sampling_logp_difference/mean": 0.04575711861252785, "step": 421, "step_time": 14.494834436976817 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 295.375, "completions/mean_terminated_length": 295.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08312376006506383, "epoch": 0.00844, "frac_reward_zero_std": 0.0, "grad_norm": 0.5055375695228577, "kl": 1.035505022853613, "learning_rate": 6.880306171506535e-06, "loss": -0.0305, "num_tokens": 20648390.0, "reward": 1.34375, "reward_std": 1.0755670070648193, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.0658745765686035, "sampling/importance_sampling_ratio/max": 1.1092897653579712, "sampling/importance_sampling_ratio/mean": 0.9503019452095032, "sampling/importance_sampling_ratio/min": 1.910289981754687e-11, "sampling/sampling_logp_difference/max": 24.491905212402344, "sampling/sampling_logp_difference/mean": 0.23473785817623138, "step": 422, "step_time": 12.881233660969883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 820.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 214.59375, "completions/mean_terminated_length": 214.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08017971739172935, "epoch": 0.00846, "frac_reward_zero_std": 0.0, "grad_norm": 0.3805738091468811, "kl": 1.0813879668712616, "learning_rate": 6.866876187906582e-06, "loss": -0.0464, "num_tokens": 20693874.0, "reward": 1.375, "reward_std": 0.6933960318565369, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.7931155562400818, "sampling/importance_sampling_ratio/max": 1.1252421140670776, "sampling/importance_sampling_ratio/mean": 1.0030213594436646, "sampling/importance_sampling_ratio/min": 0.5591307878494263, "sampling/sampling_logp_difference/max": 0.581371545791626, "sampling/sampling_logp_difference/mean": 0.018962882459163666, "step": 423, "step_time": 12.501532886992209 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 307.9375, "completions/mean_terminated_length": 307.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07684741180855781, "epoch": 0.00848, "frac_reward_zero_std": 0.0, "grad_norm": 0.6582745313644409, "kl": 1.6276474660262465, "learning_rate": 6.8534380672927e-06, "loss": -0.0556, "num_tokens": 20742975.0, "reward": 1.125, "reward_std": 1.1407177448272705, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 1.1845782995224, "sampling/importance_sampling_ratio/max": 1.1306796073913574, "sampling/importance_sampling_ratio/mean": 0.9604613780975342, "sampling/importance_sampling_ratio/min": 0.2614985704421997, "sampling/sampling_logp_difference/max": 1.3487623929977417, "sampling/sampling_logp_difference/mean": 0.04070751741528511, "step": 424, "step_time": 13.738173962003202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 250.4375, "completions/mean_terminated_length": 250.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10787395935039967, "epoch": 0.0085, "frac_reward_zero_std": 0.0, "grad_norm": 0.3617130517959595, "kl": 3.213782448321581, "learning_rate": 6.839991986922785e-06, "loss": -0.0683, "num_tokens": 20790197.0, "reward": 1.15625, "reward_std": 1.2525452375411987, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.221035361289978, "sampling/importance_sampling_ratio/max": 1.1719225645065308, "sampling/importance_sampling_ratio/mean": 0.9104937314987183, "sampling/importance_sampling_ratio/min": 2.5110086454560587e-08, "sampling/sampling_logp_difference/max": 17.443025588989258, "sampling/sampling_logp_difference/mean": 0.20038476586341858, "step": 425, "step_time": 15.057119838005747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.020833333488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 355.375, "completions/mean_terminated_length": 355.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06876552570611238, "epoch": 0.00852, "frac_reward_zero_std": 0.0, "grad_norm": 0.345507949590683, "kl": 3.3127465546131134, "learning_rate": 6.826538124159727e-06, "loss": -0.0443, "num_tokens": 20841595.0, "reward": 1.4375, "reward_std": 0.7944362163543701, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.8775883316993713, "sampling/importance_sampling_ratio/max": 1.172499656677246, "sampling/importance_sampling_ratio/mean": 1.001413345336914, "sampling/importance_sampling_ratio/min": 0.2179529368877411, "sampling/sampling_logp_difference/max": 1.5292696952819824, "sampling/sampling_logp_difference/mean": 0.02230917662382126, "step": 426, "step_time": 14.37751269598084 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 263.46875, "completions/mean_terminated_length": 263.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06555705075152218, "epoch": 0.00854, "frac_reward_zero_std": 0.0, "grad_norm": 0.13431426882743835, "kl": 3.160032218322158, "learning_rate": 6.813076656469068e-06, "loss": -0.0618, "num_tokens": 20890206.0, "reward": 1.5, "reward_std": 0.9522165060043335, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9837387204170227, "sampling/importance_sampling_ratio/max": 1.061876654624939, "sampling/importance_sampling_ratio/mean": 0.9994890689849854, "sampling/importance_sampling_ratio/min": 0.5452803373336792, "sampling/sampling_logp_difference/max": 0.6152143478393555, "sampling/sampling_logp_difference/mean": 0.015544107183814049, "step": 427, "step_time": 13.428406363003887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 262.375, "completions/mean_terminated_length": 262.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06514066562522203, "epoch": 0.00856, "frac_reward_zero_std": 0.0, "grad_norm": 0.06738400459289551, "kl": 0.8773789033293724, "learning_rate": 6.799607761416671e-06, "loss": -0.0602, "num_tokens": 20937888.0, "reward": 1.5, "reward_std": 0.8680364489555359, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9503819346427917, "sampling/importance_sampling_ratio/max": 1.1241341829299927, "sampling/importance_sampling_ratio/mean": 1.0133163928985596, "sampling/importance_sampling_ratio/min": 0.4439276456832886, "sampling/sampling_logp_difference/max": 0.8121514320373535, "sampling/sampling_logp_difference/mean": 0.01740408129990101, "step": 428, "step_time": 13.792926269961754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0017361111240461469, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017361111240461469, "completions/clipped_ratio": 0.0, "completions/max_length": 1382.0, "completions/max_terminated_length": 1382.0, "completions/mean_length": 345.90625, "completions/mean_terminated_length": 345.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07498855330049992, "epoch": 0.00858, "frac_reward_zero_std": 0.0, "grad_norm": 0.07330093532800674, "kl": 0.6259181248024106, "learning_rate": 6.786131616666364e-06, "loss": -0.068, "num_tokens": 20989239.0, "reward": 1.6875, "reward_std": 0.677844762802124, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.7378040552139282, "sampling/importance_sampling_ratio/max": 1.147524356842041, "sampling/importance_sampling_ratio/mean": 0.9965670108795166, "sampling/importance_sampling_ratio/min": 2.315621607694981e-18, "sampling/sampling_logp_difference/max": 25.355070114135742, "sampling/sampling_logp_difference/mean": 0.2974022924900055, "step": 429, "step_time": 15.313179292992572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 285.71875, "completions/mean_terminated_length": 285.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.056241588084958494, "epoch": 0.0086, "frac_reward_zero_std": 0.0, "grad_norm": 0.29757779836654663, "kl": 0.7140652015805244, "learning_rate": 6.772648399977606e-06, "loss": -0.044, "num_tokens": 21038503.0, "reward": 1.4375, "reward_std": 0.7768235206604004, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.8775883316993713, "sampling/importance_sampling_ratio/max": 1.0908395051956177, "sampling/importance_sampling_ratio/mean": 1.0058863162994385, "sampling/importance_sampling_ratio/min": 0.48749667406082153, "sampling/sampling_logp_difference/max": 0.7272474765777588, "sampling/sampling_logp_difference/mean": 0.013463521376252174, "step": 430, "step_time": 13.830508546045166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 380.40625, "completions/mean_terminated_length": 380.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10205891984514892, "epoch": 0.00862, "frac_reward_zero_std": 0.0, "grad_norm": 0.37402358651161194, "kl": 2.1911431960761547, "learning_rate": 6.759158289203141e-06, "loss": -0.0719, "num_tokens": 21091370.0, "reward": 1.65625, "reward_std": 0.9354231953620911, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 1.130214810371399, "sampling/importance_sampling_ratio/mean": 0.9784994125366211, "sampling/importance_sampling_ratio/min": 0.2601413130760193, "sampling/sampling_logp_difference/max": 1.36344575881958, "sampling/sampling_logp_difference/mean": 0.02662779949605465, "step": 431, "step_time": 16.101705120978295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 333.03125, "completions/mean_terminated_length": 333.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0704459483968094, "epoch": 0.00864, "frac_reward_zero_std": 0.0, "grad_norm": 0.5276067852973938, "kl": 0.7269503809511662, "learning_rate": 6.745661462286648e-06, "loss": -0.0913, "num_tokens": 21141703.0, "reward": 1.71875, "reward_std": 0.9396492838859558, "rewards/rollout_reward_func/mean": 1.71875, "rewards/rollout_reward_func/std": 0.9240295886993408, "sampling/importance_sampling_ratio/max": 1.1756020784378052, "sampling/importance_sampling_ratio/mean": 1.0209901332855225, "sampling/importance_sampling_ratio/min": 0.5173784494400024, "sampling/sampling_logp_difference/max": 0.668635368347168, "sampling/sampling_logp_difference/mean": 0.014798149466514587, "step": 432, "step_time": 15.603944514019531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 315.3125, "completions/mean_terminated_length": 315.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06436925276648253, "epoch": 0.00866, "frac_reward_zero_std": 0.25, "grad_norm": 0.1429034024477005, "kl": 1.9535130448639393, "learning_rate": 6.7321580972603996e-06, "loss": -0.0416, "num_tokens": 21193455.0, "reward": 1.46875, "reward_std": 0.9662595391273499, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.1354798078536987, "sampling/importance_sampling_ratio/max": 1.0872983932495117, "sampling/importance_sampling_ratio/mean": 0.9721660614013672, "sampling/importance_sampling_ratio/min": 0.4979541003704071, "sampling/sampling_logp_difference/max": 0.7474174499511719, "sampling/sampling_logp_difference/mean": 0.02240510657429695, "step": 433, "step_time": 12.418835983029567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 390.375, "completions/mean_terminated_length": 390.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05938173457980156, "epoch": 0.00868, "frac_reward_zero_std": 0.0, "grad_norm": 0.10264012962579727, "kl": 2.1290777176618576, "learning_rate": 6.718648372242909e-06, "loss": -0.1004, "num_tokens": 21245654.0, "reward": 1.78125, "reward_std": 1.0867363214492798, "rewards/rollout_reward_func/mean": 1.78125, "rewards/rollout_reward_func/std": 1.156590461730957, "sampling/importance_sampling_ratio/max": 1.4063764810562134, "sampling/importance_sampling_ratio/mean": 1.027975082397461, "sampling/importance_sampling_ratio/min": 0.4281769394874573, "sampling/sampling_logp_difference/max": 0.8481135368347168, "sampling/sampling_logp_difference/mean": 0.01772662065923214, "step": 434, "step_time": 16.105282681019162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 238.625, "completions/mean_terminated_length": 238.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0630050360923633, "epoch": 0.0087, "frac_reward_zero_std": 0.0, "grad_norm": 0.4854274094104767, "kl": 0.6356697250157595, "learning_rate": 6.705132465436579e-06, "loss": -0.0313, "num_tokens": 21292924.0, "reward": 1.46875, "reward_std": 0.6912998557090759, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.7177192568778992, "sampling/importance_sampling_ratio/max": 1.1114959716796875, "sampling/importance_sampling_ratio/mean": 1.0148463249206543, "sampling/importance_sampling_ratio/min": 0.7095881104469299, "sampling/sampling_logp_difference/max": 0.354095458984375, "sampling/sampling_logp_difference/mean": 0.010468724183738232, "step": 435, "step_time": 13.172275184988393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 273.03125, "completions/mean_terminated_length": 273.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06568894581869245, "epoch": 0.00872, "frac_reward_zero_std": 0.0, "grad_norm": 0.21332716941833496, "kl": 1.2211805675178766, "learning_rate": 6.691610555125361e-06, "loss": -0.0559, "num_tokens": 21341614.0, "reward": 1.4375, "reward_std": 0.898113489151001, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9482581615447998, "sampling/importance_sampling_ratio/max": 1.1713348627090454, "sampling/importance_sampling_ratio/mean": 1.0015771389007568, "sampling/importance_sampling_ratio/min": 0.37877994775772095, "sampling/sampling_logp_difference/max": 0.9709253311157227, "sampling/sampling_logp_difference/mean": 0.027644414454698563, "step": 436, "step_time": 15.122571986998082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 315.75, "completions/mean_terminated_length": 315.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.055186140816658735, "epoch": 0.00874, "frac_reward_zero_std": 0.0, "grad_norm": 0.10899345576763153, "kl": 0.9099633889272809, "learning_rate": 6.678082819672389e-06, "loss": -0.0551, "num_tokens": 21391745.0, "reward": 1.5625, "reward_std": 0.9281114339828491, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.9482581615447998, "sampling/importance_sampling_ratio/max": 1.1034425497055054, "sampling/importance_sampling_ratio/mean": 1.0141355991363525, "sampling/importance_sampling_ratio/min": 0.6141790747642517, "sampling/sampling_logp_difference/max": 0.4931621551513672, "sampling/sampling_logp_difference/mean": 0.011391401290893555, "step": 437, "step_time": 14.501030878018355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 408.03125, "completions/mean_terminated_length": 408.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06608565896749496, "epoch": 0.00876, "frac_reward_zero_std": 0.0, "grad_norm": 0.1561840921640396, "kl": 0.6371904332190752, "learning_rate": 6.664549437517642e-06, "loss": -0.0709, "num_tokens": 21445612.0, "reward": 1.90625, "reward_std": 0.9168749451637268, "rewards/rollout_reward_func/mean": 1.90625, "rewards/rollout_reward_func/std": 0.892960786819458, "sampling/importance_sampling_ratio/max": 1.311112642288208, "sampling/importance_sampling_ratio/mean": 1.0525370836257935, "sampling/importance_sampling_ratio/min": 1.0035860538482666, "sampling/sampling_logp_difference/max": 0.23233374953269958, "sampling/sampling_logp_difference/mean": 0.012254012748599052, "step": 438, "step_time": 13.649306228981004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 2079.0, "completions/max_terminated_length": 2079.0, "completions/mean_length": 421.625, "completions/mean_terminated_length": 421.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08135636383667588, "epoch": 0.00878, "frac_reward_zero_std": 0.0, "grad_norm": 0.5638837814331055, "kl": 1.677250262349844, "learning_rate": 6.651010587175577e-06, "loss": -0.0431, "num_tokens": 21500455.0, "reward": 1.53125, "reward_std": 1.0088657140731812, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 1.0467884540557861, "sampling/importance_sampling_ratio/max": 1.082666039466858, "sampling/importance_sampling_ratio/mean": 0.9830026626586914, "sampling/importance_sampling_ratio/min": 0.414383202791214, "sampling/sampling_logp_difference/max": 0.9271960258483887, "sampling/sampling_logp_difference/mean": 0.02183719351887703, "step": 439, "step_time": 17.445313347008778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1409.0, "completions/max_terminated_length": 1409.0, "completions/mean_length": 335.4375, "completions/mean_terminated_length": 335.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05371509585529566, "epoch": 0.0088, "frac_reward_zero_std": 0.0, "grad_norm": 0.081868015229702, "kl": 0.8344623297452927, "learning_rate": 6.637466447232784e-06, "loss": -0.032, "num_tokens": 21551442.0, "reward": 1.5625, "reward_std": 0.760758101940155, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.84002685546875, "sampling/importance_sampling_ratio/max": 1.2201048135757446, "sampling/importance_sampling_ratio/mean": 1.0331605672836304, "sampling/importance_sampling_ratio/min": 1.0023375749588013, "sampling/sampling_logp_difference/max": 0.12558455765247345, "sampling/sampling_logp_difference/mean": 0.008842837065458298, "step": 440, "step_time": 15.385816872003488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 282.34375, "completions/mean_terminated_length": 282.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08249389659613371, "epoch": 0.00882, "frac_reward_zero_std": 0.0, "grad_norm": 0.9205237627029419, "kl": 0.689533936791122, "learning_rate": 6.623917196345622e-06, "loss": -0.0475, "num_tokens": 21600584.0, "reward": 1.5, "reward_std": 0.8301036357879639, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.8424234986305237, "sampling/importance_sampling_ratio/max": 1.0816195011138916, "sampling/importance_sampling_ratio/mean": 1.013765573501587, "sampling/importance_sampling_ratio/min": 0.869009256362915, "sampling/sampling_logp_difference/max": 0.14041388034820557, "sampling/sampling_logp_difference/mean": 0.00873644184321165, "step": 441, "step_time": 13.706705386008252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1310.0, "completions/max_terminated_length": 1310.0, "completions/mean_length": 273.90625, "completions/mean_terminated_length": 273.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06474345934111625, "epoch": 0.00884, "frac_reward_zero_std": 0.0, "grad_norm": 1.2033511400222778, "kl": 5.881407762877643, "learning_rate": 6.610363013237871e-06, "loss": -0.0487, "num_tokens": 21649123.0, "reward": 1.5, "reward_std": 0.8307148218154907, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.8798826932907104, "sampling/importance_sampling_ratio/max": 1.0680922269821167, "sampling/importance_sampling_ratio/mean": 0.9746274948120117, "sampling/importance_sampling_ratio/min": 0.26035913825035095, "sampling/sampling_logp_difference/max": 1.3455910682678223, "sampling/sampling_logp_difference/mean": 0.028834134340286255, "step": 442, "step_time": 14.880684342002496 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 287.96875, "completions/mean_terminated_length": 287.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06984263635240495, "epoch": 0.00886, "frac_reward_zero_std": 0.0, "grad_norm": 0.22789452970027924, "kl": 1.190170057117939, "learning_rate": 6.59680407669837e-06, "loss": -0.0539, "num_tokens": 21697730.0, "reward": 1.375, "reward_std": 0.9928832054138184, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.008032202720642, "sampling/importance_sampling_ratio/max": 1.1410115957260132, "sampling/importance_sampling_ratio/mean": 0.9983597993850708, "sampling/importance_sampling_ratio/min": 0.3809761106967926, "sampling/sampling_logp_difference/max": 0.9650247097015381, "sampling/sampling_logp_difference/mean": 0.02288585901260376, "step": 443, "step_time": 13.781198749988107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 259.6875, "completions/mean_terminated_length": 259.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06393871211912483, "epoch": 0.00888, "frac_reward_zero_std": 0.0, "grad_norm": 0.47664010524749756, "kl": 1.2118823267519474, "learning_rate": 6.583240565578657e-06, "loss": -0.0633, "num_tokens": 21745767.0, "reward": 1.46875, "reward_std": 0.7416049242019653, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.8793096542358398, "sampling/importance_sampling_ratio/max": 1.0501857995986938, "sampling/importance_sampling_ratio/mean": 0.9543295502662659, "sampling/importance_sampling_ratio/min": 0.29431459307670593, "sampling/sampling_logp_difference/max": 1.224409818649292, "sampling/sampling_logp_difference/mean": 0.0385059118270874, "step": 444, "step_time": 13.34988146899559 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1418.0, "completions/max_terminated_length": 1418.0, "completions/mean_length": 374.875, "completions/mean_terminated_length": 374.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06341001903638244, "epoch": 0.0089, "frac_reward_zero_std": 0.0, "grad_norm": 0.10112548619508743, "kl": 1.5895256604999304, "learning_rate": 6.569672658790611e-06, "loss": -0.0889, "num_tokens": 21797977.0, "reward": 1.65625, "reward_std": 0.9691790342330933, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 1.035167932510376, "sampling/importance_sampling_ratio/max": 1.0943137407302856, "sampling/importance_sampling_ratio/mean": 0.9773273468017578, "sampling/importance_sampling_ratio/min": 0.36161890625953674, "sampling/sampling_logp_difference/max": 1.0278944969177246, "sampling/sampling_logp_difference/mean": 0.023605382069945335, "step": 445, "step_time": 15.520706446041004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 389.53125, "completions/mean_terminated_length": 389.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06675707595422864, "epoch": 0.00892, "frac_reward_zero_std": 0.0, "grad_norm": 0.48004642128944397, "kl": 3.444618422538042, "learning_rate": 6.556100535304097e-06, "loss": -0.0667, "num_tokens": 21851041.0, "reward": 1.46875, "reward_std": 1.178049087524414, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.1909435987472534, "sampling/importance_sampling_ratio/max": 1.3409615755081177, "sampling/importance_sampling_ratio/mean": 1.0018236637115479, "sampling/importance_sampling_ratio/min": 0.511696994304657, "sampling/sampling_logp_difference/max": 0.6699979305267334, "sampling/sampling_logp_difference/mean": 0.018221773207187653, "step": 446, "step_time": 15.637539645977085 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 336.125, "completions/mean_terminated_length": 336.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05471942643634975, "epoch": 0.00894, "frac_reward_zero_std": 0.0, "grad_norm": 0.10275297611951828, "kl": 1.9446740942075849, "learning_rate": 6.542524374144598e-06, "loss": -0.0921, "num_tokens": 21900927.0, "reward": 1.59375, "reward_std": 1.1050667762756348, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.160071611404419, "sampling/importance_sampling_ratio/max": 1.183933138847351, "sampling/importance_sampling_ratio/mean": 0.9713013172149658, "sampling/importance_sampling_ratio/min": 0.33387044072151184, "sampling/sampling_logp_difference/max": 1.0970282554626465, "sampling/sampling_logp_difference/mean": 0.03250851109623909, "step": 447, "step_time": 13.873197365974193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 242.3125, "completions/mean_terminated_length": 242.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08009049901738763, "epoch": 0.00896, "frac_reward_zero_std": 0.0, "grad_norm": 0.37950900197029114, "kl": 1.5850320812314749, "learning_rate": 6.528944354390855e-06, "loss": -0.067, "num_tokens": 21946751.0, "reward": 1.15625, "reward_std": 1.1730315685272217, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 1.2727763652801514, "sampling/importance_sampling_ratio/max": 1.5663334131240845, "sampling/importance_sampling_ratio/mean": 1.0022976398468018, "sampling/importance_sampling_ratio/min": 0.37666746973991394, "sampling/sampling_logp_difference/max": 0.9764337539672852, "sampling/sampling_logp_difference/mean": 0.033502496778964996, "step": 448, "step_time": 14.071949056044105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 257.0625, "completions/mean_terminated_length": 257.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07682976149953902, "epoch": 0.00898, "frac_reward_zero_std": 0.0, "grad_norm": 0.2660537362098694, "kl": 2.2239083368331194, "learning_rate": 6.515360655172512e-06, "loss": -0.0465, "num_tokens": 21994233.0, "reward": 1.1875, "reward_std": 1.0729880332946777, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.0906493663787842, "sampling/importance_sampling_ratio/max": 1.1922820806503296, "sampling/importance_sampling_ratio/mean": 0.9909559488296509, "sampling/importance_sampling_ratio/min": 0.3820347487926483, "sampling/sampling_logp_difference/max": 0.9619150161743164, "sampling/sampling_logp_difference/mean": 0.02956731803715229, "step": 449, "step_time": 13.546009486002731 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 287.6875, "completions/mean_terminated_length": 287.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06422560242936015, "epoch": 0.009, "frac_reward_zero_std": 0.0, "grad_norm": 0.11046067625284195, "kl": 1.4699750375002623, "learning_rate": 6.501773455667742e-06, "loss": -0.0442, "num_tokens": 22043123.0, "reward": 1.4375, "reward_std": 0.9033769369125366, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9136068224906921, "sampling/importance_sampling_ratio/max": 1.1541030406951904, "sampling/importance_sampling_ratio/mean": 1.0136189460754395, "sampling/importance_sampling_ratio/min": 0.4863405227661133, "sampling/sampling_logp_difference/max": 0.7204599380493164, "sampling/sampling_logp_difference/mean": 0.016011638566851616, "step": 450, "step_time": 14.313535854991642 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1310.0, "completions/max_terminated_length": 1310.0, "completions/mean_length": 373.90625, "completions/mean_terminated_length": 373.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05076137441210449, "epoch": 0.00902, "frac_reward_zero_std": 0.0, "grad_norm": 0.21791765093803406, "kl": 0.739462535828352, "learning_rate": 6.488182935100893e-06, "loss": -0.0588, "num_tokens": 22096296.0, "reward": 1.875, "reward_std": 0.7833450436592102, "rewards/rollout_reward_func/mean": 1.875, "rewards/rollout_reward_func/std": 0.7931155562400818, "sampling/importance_sampling_ratio/max": 1.222494125366211, "sampling/importance_sampling_ratio/mean": 1.0113414525985718, "sampling/importance_sampling_ratio/min": 0.4121992886066437, "sampling/sampling_logp_difference/max": 0.9143033027648926, "sampling/sampling_logp_difference/mean": 0.015160317532718182, "step": 451, "step_time": 15.352716118999524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 271.65625, "completions/mean_terminated_length": 271.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05895353632513434, "epoch": 0.00904, "frac_reward_zero_std": 0.0, "grad_norm": 0.03629952296614647, "kl": 1.5075587686151266, "learning_rate": 6.474589272740116e-06, "loss": -0.0611, "num_tokens": 22144299.0, "reward": 1.4375, "reward_std": 0.9077951908111572, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9482581615447998, "sampling/importance_sampling_ratio/max": 1.1094876527786255, "sampling/importance_sampling_ratio/mean": 1.0066062211990356, "sampling/importance_sampling_ratio/min": 0.47816887497901917, "sampling/sampling_logp_difference/max": 0.7378420829772949, "sampling/sampling_logp_difference/mean": 0.013233162462711334, "step": 452, "step_time": 13.333865939028328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 404.8125, "completions/mean_terminated_length": 404.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06318458588793874, "epoch": 0.00906, "frac_reward_zero_std": 0.0, "grad_norm": 0.5052416920661926, "kl": 1.124704647809267, "learning_rate": 6.46099264789501e-06, "loss": -0.0628, "num_tokens": 22197838.0, "reward": 1.5, "reward_std": 1.0774788856506348, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.0776318311691284, "sampling/importance_sampling_ratio/max": 1.1836997270584106, "sampling/importance_sampling_ratio/mean": 0.9156104922294617, "sampling/importance_sampling_ratio/min": 0.1386173665523529, "sampling/sampling_logp_difference/max": 1.9857301712036133, "sampling/sampling_logp_difference/mean": 0.05305805057287216, "step": 453, "step_time": 15.372839547024341 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 301.15625, "completions/mean_terminated_length": 301.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.060577905271202326, "epoch": 0.00908, "frac_reward_zero_std": 0.0, "grad_norm": 0.570548415184021, "kl": 3.8317542504519224, "learning_rate": 6.447393239914247e-06, "loss": -0.055, "num_tokens": 22246705.0, "reward": 1.25, "reward_std": 1.0199705362319946, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.0472698211669922, "sampling/importance_sampling_ratio/max": 1.2676328420639038, "sampling/importance_sampling_ratio/mean": 0.9783234000205994, "sampling/importance_sampling_ratio/min": 0.3975785970687866, "sampling/sampling_logp_difference/max": 0.9252262115478516, "sampling/sampling_logp_difference/mean": 0.03210502862930298, "step": 454, "step_time": 14.323905048004235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 375.78125, "completions/mean_terminated_length": 375.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07253911206498742, "epoch": 0.0091, "frac_reward_zero_std": 0.0, "grad_norm": 0.41189709305763245, "kl": 0.6800583638250828, "learning_rate": 6.433791228183211e-06, "loss": -0.0624, "num_tokens": 22299388.0, "reward": 1.53125, "reward_std": 1.0592429637908936, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 1.0467884540557861, "sampling/importance_sampling_ratio/max": 1.7396628856658936, "sampling/importance_sampling_ratio/mean": 1.0117305517196655, "sampling/importance_sampling_ratio/min": 0.5601499676704407, "sampling/sampling_logp_difference/max": 0.5878134965896606, "sampling/sampling_logp_difference/mean": 0.020849112421274185, "step": 455, "step_time": 13.636222442029975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1337.0, "completions/max_terminated_length": 1337.0, "completions/mean_length": 323.90625, "completions/mean_terminated_length": 323.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.057039862498641014, "epoch": 0.00912, "frac_reward_zero_std": 0.0, "grad_norm": 0.04551361873745918, "kl": 0.6120191141963005, "learning_rate": 6.42018679212163e-06, "loss": -0.0617, "num_tokens": 22348931.0, "reward": 1.5625, "reward_std": 0.883499264717102, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.9136068224906921, "sampling/importance_sampling_ratio/max": 1.1610394716262817, "sampling/importance_sampling_ratio/mean": 1.030522346496582, "sampling/importance_sampling_ratio/min": 1.0033904314041138, "sampling/sampling_logp_difference/max": 0.14646758139133453, "sampling/sampling_logp_difference/mean": 0.008224360644817352, "step": 456, "step_time": 14.689362557022832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 220.09375, "completions/mean_terminated_length": 220.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07580272061750293, "epoch": 0.00914, "frac_reward_zero_std": 0.0, "grad_norm": 0.6025184392929077, "kl": 0.685421185567975, "learning_rate": 6.406580111181216e-06, "loss": -0.051, "num_tokens": 22395418.0, "reward": 1.34375, "reward_std": 0.8636828064918518, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 1.1466573476791382, "sampling/importance_sampling_ratio/mean": 1.0033115148544312, "sampling/importance_sampling_ratio/min": 0.41807350516319275, "sampling/sampling_logp_difference/max": 0.5944864749908447, "sampling/sampling_logp_difference/mean": 0.020813077688217163, "step": 457, "step_time": 13.503452001998085 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 378.4375, "completions/mean_terminated_length": 378.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07338765077292919, "epoch": 0.00916, "frac_reward_zero_std": 0.0, "grad_norm": 0.9459490776062012, "kl": 1.1215592566877604, "learning_rate": 6.392971364843287e-06, "loss": -0.0556, "num_tokens": 22448447.0, "reward": 1.5, "reward_std": 1.040791392326355, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.0472698211669922, "sampling/importance_sampling_ratio/max": 1.099439024925232, "sampling/importance_sampling_ratio/mean": 0.9952383041381836, "sampling/importance_sampling_ratio/min": 0.16307228803634644, "sampling/sampling_logp_difference/max": 1.829698085784912, "sampling/sampling_logp_difference/mean": 0.02245154045522213, "step": 458, "step_time": 14.001850486019976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 242.84375, "completions/mean_terminated_length": 242.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0637617486063391, "epoch": 0.00918, "frac_reward_zero_std": 0.0, "grad_norm": 0.15840432047843933, "kl": 1.2517505399882793, "learning_rate": 6.379360732616405e-06, "loss": -0.0629, "num_tokens": 22495383.0, "reward": 1.40625, "reward_std": 0.9724266529083252, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9791166186332703, "sampling/importance_sampling_ratio/max": 1.093833088874817, "sampling/importance_sampling_ratio/mean": 0.9938198328018188, "sampling/importance_sampling_ratio/min": 0.488789439201355, "sampling/sampling_logp_difference/max": 0.7414236068725586, "sampling/sampling_logp_difference/mean": 0.02169463038444519, "step": 459, "step_time": 13.682289628020953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1643.0, "completions/max_terminated_length": 1643.0, "completions/mean_length": 242.1875, "completions/mean_terminated_length": 242.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0675401147454977, "epoch": 0.0092, "frac_reward_zero_std": 0.0, "grad_norm": 0.09352073818445206, "kl": 1.5017788913100958, "learning_rate": 6.365748394034013e-06, "loss": -0.0619, "num_tokens": 22541628.0, "reward": 1.25, "reward_std": 0.857441782951355, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9503819346427917, "sampling/importance_sampling_ratio/max": 1.1906869411468506, "sampling/importance_sampling_ratio/mean": 1.0141849517822266, "sampling/importance_sampling_ratio/min": 0.37824326753616333, "sampling/sampling_logp_difference/max": 0.9721274375915527, "sampling/sampling_logp_difference/mean": 0.020014526322484016, "step": 460, "step_time": 15.98796270301682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 272.0625, "completions/mean_terminated_length": 272.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07072029681876302, "epoch": 0.00922, "frac_reward_zero_std": 0.0, "grad_norm": 0.20142929255962372, "kl": 0.7870849445462227, "learning_rate": 6.352134528652057e-06, "loss": -0.058, "num_tokens": 22589991.0, "reward": 1.46875, "reward_std": 0.9096556901931763, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.949851393699646, "sampling/importance_sampling_ratio/max": 1.1925395727157593, "sampling/importance_sampling_ratio/mean": 1.0355302095413208, "sampling/importance_sampling_ratio/min": 0.9905286431312561, "sampling/sampling_logp_difference/max": 0.1719588041305542, "sampling/sampling_logp_difference/mean": 0.010543136857450008, "step": 461, "step_time": 13.519866200978868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1625.0, "completions/max_terminated_length": 1625.0, "completions/mean_length": 295.5625, "completions/mean_terminated_length": 295.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08115232363343239, "epoch": 0.00924, "frac_reward_zero_std": 0.0, "grad_norm": 0.1986130177974701, "kl": 2.195191290229559, "learning_rate": 6.3385193160466255e-06, "loss": -0.062, "num_tokens": 22640101.0, "reward": 1.5, "reward_std": 0.9094313979148865, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.1071614027023315, "sampling/importance_sampling_ratio/max": 1.1805000305175781, "sampling/importance_sampling_ratio/mean": 0.9944171905517578, "sampling/importance_sampling_ratio/min": 0.34162354469299316, "sampling/sampling_logp_difference/max": 1.074117660522461, "sampling/sampling_logp_difference/mean": 0.02387671172618866, "step": 462, "step_time": 16.680368583998643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 793.0, "completions/max_terminated_length": 793.0, "completions/mean_length": 227.96875, "completions/mean_terminated_length": 227.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06492766086012125, "epoch": 0.00926, "frac_reward_zero_std": 0.0, "grad_norm": 0.11425802111625671, "kl": 0.8311294112354517, "learning_rate": 6.324902935811576e-06, "loss": -0.0393, "num_tokens": 22686695.0, "reward": 1.28125, "reward_std": 0.7827985286712646, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.7718588709831238, "sampling/importance_sampling_ratio/max": 1.2707387208938599, "sampling/importance_sampling_ratio/mean": 1.0165098905563354, "sampling/importance_sampling_ratio/min": 0.34436532855033875, "sampling/sampling_logp_difference/max": 1.0670154094696045, "sampling/sampling_logp_difference/mean": 0.02124207280576229, "step": 463, "step_time": 12.004438496005605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 384.1875, "completions/mean_terminated_length": 384.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.069333974737674, "epoch": 0.00928, "frac_reward_zero_std": 0.0, "grad_norm": 0.14804692566394806, "kl": 0.8760755080729723, "learning_rate": 6.311285567556168e-06, "loss": -0.0578, "num_tokens": 22740351.0, "reward": 1.28125, "reward_std": 1.2685456275939941, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.3255400657653809, "sampling/importance_sampling_ratio/max": 1.147806167602539, "sampling/importance_sampling_ratio/mean": 1.0066943168640137, "sampling/importance_sampling_ratio/min": 0.4779960513114929, "sampling/sampling_logp_difference/max": 0.7407441139221191, "sampling/sampling_logp_difference/mean": 0.014071325771510601, "step": 464, "step_time": 14.42178633401636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 290.46875, "completions/mean_terminated_length": 290.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0639915312640369, "epoch": 0.0093, "frac_reward_zero_std": 0.0, "grad_norm": 0.24928054213523865, "kl": 2.638353059068322, "learning_rate": 6.297667390902694e-06, "loss": -0.0607, "num_tokens": 22789941.0, "reward": 1.1875, "reward_std": 1.2244212627410889, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.203154444694519, "sampling/importance_sampling_ratio/max": 1.1035828590393066, "sampling/importance_sampling_ratio/mean": 0.987730085849762, "sampling/importance_sampling_ratio/min": 0.37054067850112915, "sampling/sampling_logp_difference/max": 0.9929556846618652, "sampling/sampling_logp_difference/mean": 0.023783281445503235, "step": 465, "step_time": 14.56138713497785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1832.0, "completions/max_terminated_length": 1832.0, "completions/mean_length": 341.46875, "completions/mean_terminated_length": 341.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06317741516977549, "epoch": 0.00932, "frac_reward_zero_std": 0.0, "grad_norm": 0.06477497518062592, "kl": 0.6968811564147472, "learning_rate": 6.284048585484113e-06, "loss": -0.0787, "num_tokens": 22840404.0, "reward": 1.78125, "reward_std": 0.845369815826416, "rewards/rollout_reward_func/mean": 1.78125, "rewards/rollout_reward_func/std": 0.8700898885726929, "sampling/importance_sampling_ratio/max": 1.1868640184402466, "sampling/importance_sampling_ratio/mean": 1.0342392921447754, "sampling/importance_sampling_ratio/min": 1.0061311721801758, "sampling/sampling_logp_difference/max": 0.17115843296051025, "sampling/sampling_logp_difference/mean": 0.008991203270852566, "step": 466, "step_time": 16.47752326200134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 287.71875, "completions/mean_terminated_length": 287.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08801390673033893, "epoch": 0.00934, "frac_reward_zero_std": 0.0, "grad_norm": 0.1583954244852066, "kl": 3.340424921363592, "learning_rate": 6.270429330941671e-06, "loss": -0.0697, "num_tokens": 22889742.0, "reward": 1.5, "reward_std": 0.9585972428321838, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.0160009860992432, "sampling/importance_sampling_ratio/max": 1.1784000396728516, "sampling/importance_sampling_ratio/mean": 0.9603805541992188, "sampling/importance_sampling_ratio/min": 0.43562954664230347, "sampling/sampling_logp_difference/max": 0.8308303356170654, "sampling/sampling_logp_difference/mean": 0.03477958217263222, "step": 467, "step_time": 13.48231214399857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1310.0, "completions/max_terminated_length": 1310.0, "completions/mean_length": 405.96875, "completions/mean_terminated_length": 405.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.075011414475739, "epoch": 0.00936, "frac_reward_zero_std": 0.0, "grad_norm": 0.531211256980896, "kl": 0.7840764950960875, "learning_rate": 6.2568098069225436e-06, "loss": -0.0732, "num_tokens": 22943615.0, "reward": 1.4375, "reward_std": 1.170825481414795, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.1622419357299805, "sampling/importance_sampling_ratio/max": 1.1584091186523438, "sampling/importance_sampling_ratio/mean": 0.9402605295181274, "sampling/importance_sampling_ratio/min": 0.30742478370666504, "sampling/sampling_logp_difference/max": 1.2459596395492554, "sampling/sampling_logp_difference/mean": 0.03651365637779236, "step": 468, "step_time": 15.560310052023851 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 324.71875, "completions/mean_terminated_length": 324.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05633753852453083, "epoch": 0.00938, "frac_reward_zero_std": 0.0, "grad_norm": 0.06845758855342865, "kl": 1.0743025951087475, "learning_rate": 6.243190193077457e-06, "loss": -0.0677, "num_tokens": 22993974.0, "reward": 1.5625, "reward_std": 0.9801400899887085, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.2524210214614868, "sampling/importance_sampling_ratio/mean": 1.0101855993270874, "sampling/importance_sampling_ratio/min": 0.3864140212535858, "sampling/sampling_logp_difference/max": 0.9508402347564697, "sampling/sampling_logp_difference/mean": 0.015468901954591274, "step": 469, "step_time": 13.883836760986014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 302.71875, "completions/mean_terminated_length": 302.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06647023325785995, "epoch": 0.0094, "frac_reward_zero_std": 0.0, "grad_norm": 0.05918964371085167, "kl": 0.7074726074934006, "learning_rate": 6.2295706690583325e-06, "loss": -0.038, "num_tokens": 23043836.0, "reward": 1.40625, "reward_std": 0.8969102501869202, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9108441472053528, "sampling/importance_sampling_ratio/max": 1.1236289739608765, "sampling/importance_sampling_ratio/mean": 1.027857780456543, "sampling/importance_sampling_ratio/min": 1.0056477785110474, "sampling/sampling_logp_difference/max": 0.11121366918087006, "sampling/sampling_logp_difference/mean": 0.00792483426630497, "step": 470, "step_time": 14.10890599299455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 254.9375, "completions/mean_terminated_length": 254.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06922103860415518, "epoch": 0.00942, "frac_reward_zero_std": 0.0, "grad_norm": 0.2086547315120697, "kl": 0.7934078238904476, "learning_rate": 6.215951414515888e-06, "loss": -0.029, "num_tokens": 23090637.0, "reward": 1.34375, "reward_std": 0.6102004051208496, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.6530017256736755, "sampling/importance_sampling_ratio/max": 1.4060214757919312, "sampling/importance_sampling_ratio/mean": 1.0265928506851196, "sampling/importance_sampling_ratio/min": 0.5629007816314697, "sampling/sampling_logp_difference/max": 0.6168222427368164, "sampling/sampling_logp_difference/mean": 0.017222275957465172, "step": 471, "step_time": 14.272567929991055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 269.40625, "completions/mean_terminated_length": 269.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07596896728500724, "epoch": 0.00944, "frac_reward_zero_std": 0.0, "grad_norm": 0.19587042927742004, "kl": 0.9251693338155746, "learning_rate": 6.202332609097308e-06, "loss": -0.0508, "num_tokens": 23139096.0, "reward": 1.1875, "reward_std": 1.1884679794311523, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.1760376691818237, "sampling/importance_sampling_ratio/max": 1.1228491067886353, "sampling/importance_sampling_ratio/mean": 1.00180983543396, "sampling/importance_sampling_ratio/min": 0.5576131939888, "sampling/sampling_logp_difference/max": 0.5841107368469238, "sampling/sampling_logp_difference/mean": 0.01944294571876526, "step": 472, "step_time": 13.534603946027346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 256.75, "completions/mean_terminated_length": 256.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07434304221533239, "epoch": 0.00946, "frac_reward_zero_std": 0.0, "grad_norm": 0.2595062553882599, "kl": 0.9200648628175259, "learning_rate": 6.188714432443833e-06, "loss": -0.0558, "num_tokens": 23186844.0, "reward": 1.4375, "reward_std": 0.8341206312179565, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.8775883316993713, "sampling/importance_sampling_ratio/max": 1.1232616901397705, "sampling/importance_sampling_ratio/mean": 1.00001859664917, "sampling/importance_sampling_ratio/min": 0.5317307114601135, "sampling/sampling_logp_difference/max": 0.6316750049591064, "sampling/sampling_logp_difference/mean": 0.019682273268699646, "step": 473, "step_time": 14.126548002008349 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 354.8125, "completions/mean_terminated_length": 354.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0636695132125169, "epoch": 0.00948, "frac_reward_zero_std": 0.0, "grad_norm": 0.141702800989151, "kl": 2.8501334730535746, "learning_rate": 6.175097064188427e-06, "loss": -0.053, "num_tokens": 23238940.0, "reward": 1.5, "reward_std": 1.0686612129211426, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.0472698211669922, "sampling/importance_sampling_ratio/max": 1.104523777961731, "sampling/importance_sampling_ratio/mean": 0.9985552430152893, "sampling/importance_sampling_ratio/min": 0.4038442373275757, "sampling/sampling_logp_difference/max": 0.9115257263183594, "sampling/sampling_logp_difference/mean": 0.020999625325202942, "step": 474, "step_time": 14.20624098496046 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1598.0, "completions/max_terminated_length": 1598.0, "completions/mean_length": 278.9375, "completions/mean_terminated_length": 278.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0869070990011096, "epoch": 0.0095, "frac_reward_zero_std": 0.0, "grad_norm": 1.0391080379486084, "kl": 0.6426119580864906, "learning_rate": 6.161480683953376e-06, "loss": -0.0677, "num_tokens": 23288368.0, "reward": 1.375, "reward_std": 1.1817001104354858, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.2115039825439453, "sampling/importance_sampling_ratio/max": 1.09040105342865, "sampling/importance_sampling_ratio/mean": 1.017608404159546, "sampling/importance_sampling_ratio/min": 0.7308170795440674, "sampling/sampling_logp_difference/max": 0.3137493133544922, "sampling/sampling_logp_difference/mean": 0.013720659539103508, "step": 475, "step_time": 16.061217015972943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 388.125, "completions/mean_terminated_length": 388.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05429676023777574, "epoch": 0.00952, "frac_reward_zero_std": 0.0, "grad_norm": 0.04265559837222099, "kl": 1.1594327297061682, "learning_rate": 6.147865471347945e-06, "loss": -0.0484, "num_tokens": 23340784.0, "reward": 1.40625, "reward_std": 1.1942355632781982, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.2144126892089844, "sampling/importance_sampling_ratio/max": 1.0855991840362549, "sampling/importance_sampling_ratio/mean": 1.010496735572815, "sampling/importance_sampling_ratio/min": 0.4909416735172272, "sampling/sampling_logp_difference/max": 0.7112860679626465, "sampling/sampling_logp_difference/mean": 0.013172764331102371, "step": 476, "step_time": 14.514826419006567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1670.0, "completions/max_terminated_length": 1670.0, "completions/mean_length": 346.8125, "completions/mean_terminated_length": 346.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06844092311803252, "epoch": 0.00954, "frac_reward_zero_std": 0.0, "grad_norm": 0.49338892102241516, "kl": 0.6703840866684914, "learning_rate": 6.134251605965988e-06, "loss": -0.043, "num_tokens": 23392376.0, "reward": 1.34375, "reward_std": 0.8105394840240479, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.9708451628684998, "sampling/importance_sampling_ratio/max": 1.117914080619812, "sampling/importance_sampling_ratio/mean": 1.0259592533111572, "sampling/importance_sampling_ratio/min": 0.842060923576355, "sampling/sampling_logp_difference/max": 0.17439520359039307, "sampling/sampling_logp_difference/mean": 0.009620074182748795, "step": 477, "step_time": 17.18232743801491 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1274.0, "completions/max_terminated_length": 1274.0, "completions/mean_length": 425.03125, "completions/mean_terminated_length": 425.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04918978363275528, "epoch": 0.00956, "frac_reward_zero_std": 0.0, "grad_norm": 0.39748772978782654, "kl": 2.2991502061486244, "learning_rate": 6.1206392673835955e-06, "loss": -0.0565, "num_tokens": 23447847.0, "reward": 1.6875, "reward_std": 1.126867651939392, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 1.1760376691818237, "sampling/importance_sampling_ratio/max": 1.0916820764541626, "sampling/importance_sampling_ratio/mean": 0.9965099096298218, "sampling/importance_sampling_ratio/min": 0.336323618888855, "sampling/sampling_logp_difference/max": 1.089576244354248, "sampling/sampling_logp_difference/mean": 0.01626656763255596, "step": 478, "step_time": 14.80315724601678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 865.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 323.1875, "completions/mean_terminated_length": 323.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06008997582830489, "epoch": 0.00958, "frac_reward_zero_std": 0.0, "grad_norm": 0.22471988201141357, "kl": 1.2411922067403793, "learning_rate": 6.1070286351567154e-06, "loss": -0.0469, "num_tokens": 23498909.0, "reward": 1.46875, "reward_std": 1.0703740119934082, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.0467884540557861, "sampling/importance_sampling_ratio/max": 1.0990291833877563, "sampling/importance_sampling_ratio/mean": 0.9837429523468018, "sampling/importance_sampling_ratio/min": 0.38367336988449097, "sampling/sampling_logp_difference/max": 0.9579775333404541, "sampling/sampling_logp_difference/mean": 0.021667297929525375, "step": 479, "step_time": 12.803804419978405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 216.875, "completions/mean_terminated_length": 216.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06690309126861393, "epoch": 0.0096, "frac_reward_zero_std": 0.0, "grad_norm": 0.30698955059051514, "kl": 2.9969784282147884, "learning_rate": 6.093419888818785e-06, "loss": -0.0526, "num_tokens": 23545041.0, "reward": 1.1875, "reward_std": 0.9407845735549927, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.1760376691818237, "sampling/importance_sampling_ratio/max": 1.1517219543457031, "sampling/importance_sampling_ratio/mean": 0.984228253364563, "sampling/importance_sampling_ratio/min": 0.4785599112510681, "sampling/sampling_logp_difference/max": 0.736842155456543, "sampling/sampling_logp_difference/mean": 0.029164545238018036, "step": 480, "step_time": 13.63705414099968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 287.6875, "completions/mean_terminated_length": 287.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.057086979039013386, "epoch": 0.00962, "frac_reward_zero_std": 0.0, "grad_norm": 0.1731041669845581, "kl": 5.036657739430666, "learning_rate": 6.0798132078783714e-06, "loss": -0.0423, "num_tokens": 23593471.0, "reward": 1.25, "reward_std": 0.7932263016700745, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9503819346427917, "sampling/importance_sampling_ratio/max": 1.087975263595581, "sampling/importance_sampling_ratio/mean": 0.9655698537826538, "sampling/importance_sampling_ratio/min": 0.48071733117103577, "sampling/sampling_logp_difference/max": 0.7325301170349121, "sampling/sampling_logp_difference/mean": 0.03134101629257202, "step": 481, "step_time": 13.688181453951984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1607.0, "completions/max_terminated_length": 1607.0, "completions/mean_length": 287.5625, "completions/mean_terminated_length": 287.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.048452231334522367, "epoch": 0.00964, "frac_reward_zero_std": 0.0, "grad_norm": 0.04186068847775459, "kl": 1.3830047603696585, "learning_rate": 6.0662087718167915e-06, "loss": -0.0675, "num_tokens": 23641921.0, "reward": 1.46875, "reward_std": 0.9145076870918274, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.9832262396812439, "sampling/importance_sampling_ratio/max": 1.0801013708114624, "sampling/importance_sampling_ratio/mean": 0.9997960925102234, "sampling/importance_sampling_ratio/min": 0.38387230038642883, "sampling/sampling_logp_difference/max": 0.9574835300445557, "sampling/sampling_logp_difference/mean": 0.016065189614892006, "step": 482, "step_time": 16.5434720430203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 820.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 258.28125, "completions/mean_terminated_length": 258.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05551329790614545, "epoch": 0.00966, "frac_reward_zero_std": 0.0, "grad_norm": 0.08053102344274521, "kl": 1.1703959358856082, "learning_rate": 6.0526067600857556e-06, "loss": -0.046, "num_tokens": 23690658.0, "reward": 1.40625, "reward_std": 0.9601868391036987, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.011526346206665, "sampling/importance_sampling_ratio/max": 1.1231260299682617, "sampling/importance_sampling_ratio/mean": 1.0086675882339478, "sampling/importance_sampling_ratio/min": 0.45189368724823, "sampling/sampling_logp_difference/max": 0.7939262390136719, "sampling/sampling_logp_difference/mean": 0.015315910801291466, "step": 483, "step_time": 12.284153991015046 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 275.3125, "completions/mean_terminated_length": 275.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04666323750279844, "epoch": 0.00968, "frac_reward_zero_std": 0.0, "grad_norm": 0.09118732810020447, "kl": 2.3878975957632065, "learning_rate": 6.039007352104992e-06, "loss": -0.0609, "num_tokens": 23739310.0, "reward": 1.5, "reward_std": 0.823853075504303, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9158109426498413, "sampling/importance_sampling_ratio/max": 1.130614995956421, "sampling/importance_sampling_ratio/mean": 1.001272201538086, "sampling/importance_sampling_ratio/min": 0.37314704060554504, "sampling/sampling_logp_difference/max": 0.9858236312866211, "sampling/sampling_logp_difference/mean": 0.015612808987498283, "step": 484, "step_time": 14.388398664988927 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 874.0, "completions/max_terminated_length": 874.0, "completions/mean_length": 254.375, "completions/mean_terminated_length": 254.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.059141550213098526, "epoch": 0.0097, "frac_reward_zero_std": 0.0, "grad_norm": 0.06179508566856384, "kl": 0.7104571145027876, "learning_rate": 6.025410727259885e-06, "loss": -0.0432, "num_tokens": 23787815.0, "reward": 1.40625, "reward_std": 0.9915584921836853, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.011526346206665, "sampling/importance_sampling_ratio/max": 1.1179795265197754, "sampling/importance_sampling_ratio/mean": 1.0085954666137695, "sampling/importance_sampling_ratio/min": 0.5837541222572327, "sampling/sampling_logp_difference/max": 0.5383858680725098, "sampling/sampling_logp_difference/mean": 0.011215927079319954, "step": 485, "step_time": 12.77167118692887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 267.25, "completions/mean_terminated_length": 267.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05410638893954456, "epoch": 0.00972, "frac_reward_zero_std": 0.25, "grad_norm": 0.06172558292746544, "kl": 2.4864428620785475, "learning_rate": 6.01181706489911e-06, "loss": -0.0293, "num_tokens": 23838080.0, "reward": 1.46875, "reward_std": 0.7644376158714294, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.915260374546051, "sampling/importance_sampling_ratio/max": 1.0952601432800293, "sampling/importance_sampling_ratio/mean": 0.9911171197891235, "sampling/importance_sampling_ratio/min": 0.4847448170185089, "sampling/sampling_logp_difference/max": 0.735116720199585, "sampling/sampling_logp_difference/mean": 0.01625274308025837, "step": 486, "step_time": 13.329795310011832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 407.9375, "completions/mean_terminated_length": 407.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05473025538958609, "epoch": 0.00974, "frac_reward_zero_std": 0.0, "grad_norm": 0.11383280158042908, "kl": 1.6924950294196606, "learning_rate": 5.99822654433226e-06, "loss": -0.0482, "num_tokens": 23891909.0, "reward": 1.625, "reward_std": 0.8550988435745239, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.8706690073013306, "sampling/importance_sampling_ratio/max": 1.1619420051574707, "sampling/importance_sampling_ratio/mean": 1.0126657485961914, "sampling/importance_sampling_ratio/min": 0.3336850106716156, "sampling/sampling_logp_difference/max": 1.0975341796875, "sampling/sampling_logp_difference/mean": 0.017199810594320297, "step": 487, "step_time": 14.450369653015514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1535.0, "completions/max_terminated_length": 1535.0, "completions/mean_length": 282.15625, "completions/mean_terminated_length": 282.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.055443365359678864, "epoch": 0.00976, "frac_reward_zero_std": 0.0, "grad_norm": 0.5018998384475708, "kl": 0.6937058418989182, "learning_rate": 5.984639344827491e-06, "loss": -0.0554, "num_tokens": 23941681.0, "reward": 1.53125, "reward_std": 0.7990366220474243, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.841825008392334, "sampling/importance_sampling_ratio/max": 1.081470251083374, "sampling/importance_sampling_ratio/mean": 1.0040227174758911, "sampling/importance_sampling_ratio/min": 0.4473101496696472, "sampling/sampling_logp_difference/max": 0.8044416904449463, "sampling/sampling_logp_difference/mean": 0.013071482069790363, "step": 488, "step_time": 15.69637168297777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 234.96875, "completions/mean_terminated_length": 234.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07409748225472867, "epoch": 0.00978, "frac_reward_zero_std": 0.0, "grad_norm": 0.29610905051231384, "kl": 0.6573834978044033, "learning_rate": 5.971055645609147e-06, "loss": -0.0464, "num_tokens": 23988864.0, "reward": 1.40625, "reward_std": 0.7948984503746033, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.8370214104652405, "sampling/importance_sampling_ratio/max": 1.186734676361084, "sampling/importance_sampling_ratio/mean": 1.0231568813323975, "sampling/importance_sampling_ratio/min": 0.7681976556777954, "sampling/sampling_logp_difference/max": 0.2651025056838989, "sampling/sampling_logp_difference/mean": 0.011315693147480488, "step": 489, "step_time": 13.26571771001909 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 265.0625, "completions/mean_terminated_length": 265.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05500504677183926, "epoch": 0.0098, "frac_reward_zero_std": 0.0, "grad_norm": 0.26487699151039124, "kl": 0.6674336045980453, "learning_rate": 5.957475625855404e-06, "loss": -0.051, "num_tokens": 24036786.0, "reward": 1.28125, "reward_std": 1.0460567474365234, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.0846248865127563, "sampling/importance_sampling_ratio/max": 1.0959770679473877, "sampling/importance_sampling_ratio/mean": 0.994857668876648, "sampling/importance_sampling_ratio/min": 0.58502197265625, "sampling/sampling_logp_difference/max": 0.5361063480377197, "sampling/sampling_logp_difference/mean": 0.01588468998670578, "step": 490, "step_time": 14.259237201011274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 265.75, "completions/mean_terminated_length": 265.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05786513816565275, "epoch": 0.00982, "frac_reward_zero_std": 0.0, "grad_norm": 0.07412518560886383, "kl": 0.8142491560429335, "learning_rate": 5.943899464695905e-06, "loss": -0.074, "num_tokens": 24085394.0, "reward": 1.6875, "reward_std": 0.6754930019378662, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.692703902721405, "sampling/importance_sampling_ratio/max": 1.2227506637573242, "sampling/importance_sampling_ratio/mean": 1.0249691009521484, "sampling/importance_sampling_ratio/min": 1.0045998096466064, "sampling/sampling_logp_difference/max": 0.19607135653495789, "sampling/sampling_logp_difference/mean": 0.007475786376744509, "step": 491, "step_time": 15.885513138025999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 226.8125, "completions/mean_terminated_length": 226.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04979927837848663, "epoch": 0.00984, "frac_reward_zero_std": 0.0, "grad_norm": 0.1422267109155655, "kl": 1.4423873219639063, "learning_rate": 5.930327341209392e-06, "loss": -0.0425, "num_tokens": 24133333.0, "reward": 1.34375, "reward_std": 0.8870099782943726, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.9019467830657959, "sampling/importance_sampling_ratio/max": 1.076332926750183, "sampling/importance_sampling_ratio/mean": 1.0056655406951904, "sampling/importance_sampling_ratio/min": 0.5433318018913269, "sampling/sampling_logp_difference/max": 0.6098670959472656, "sampling/sampling_logp_difference/mean": 0.011795156635344028, "step": 492, "step_time": 12.354239872045582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1706.0, "completions/max_terminated_length": 1706.0, "completions/mean_length": 375.5, "completions/mean_terminated_length": 375.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06024106778204441, "epoch": 0.00986, "frac_reward_zero_std": 0.0, "grad_norm": 0.18152596056461334, "kl": 0.6529789948835969, "learning_rate": 5.916759434421345e-06, "loss": -0.0507, "num_tokens": 24185451.0, "reward": 1.46875, "reward_std": 0.9868423938751221, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.0467884540557861, "sampling/importance_sampling_ratio/max": 1.185685396194458, "sampling/importance_sampling_ratio/mean": 1.0091133117675781, "sampling/importance_sampling_ratio/min": 0.40429195761680603, "sampling/sampling_logp_difference/max": 0.9246453046798706, "sampling/sampling_logp_difference/mean": 0.016719866544008255, "step": 493, "step_time": 16.829493016019114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 207.0625, "completions/mean_terminated_length": 207.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07991513563320041, "epoch": 0.00988, "frac_reward_zero_std": 0.0, "grad_norm": 0.2603641748428345, "kl": 1.1950923204421997, "learning_rate": 5.903195923301632e-06, "loss": -0.0568, "num_tokens": 24230535.0, "reward": 1.25, "reward_std": 0.9174618124961853, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9503819346427917, "sampling/importance_sampling_ratio/max": 1.106600046157837, "sampling/importance_sampling_ratio/mean": 0.9932041168212891, "sampling/importance_sampling_ratio/min": 0.4043042063713074, "sampling/sampling_logp_difference/max": 0.9056158065795898, "sampling/sampling_logp_difference/mean": 0.024355903267860413, "step": 494, "step_time": 13.583737705979729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1499.0, "completions/max_terminated_length": 1499.0, "completions/mean_length": 342.15625, "completions/mean_terminated_length": 342.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0556188952177763, "epoch": 0.0099, "frac_reward_zero_std": 0.0, "grad_norm": 0.1430864781141281, "kl": 0.7456785459071398, "learning_rate": 5.88963698676213e-06, "loss": -0.0687, "num_tokens": 24281394.0, "reward": 1.625, "reward_std": 0.8774920105934143, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.8706690073013306, "sampling/importance_sampling_ratio/max": 1.1654044389724731, "sampling/importance_sampling_ratio/mean": 1.006744146347046, "sampling/importance_sampling_ratio/min": 0.6109700798988342, "sampling/sampling_logp_difference/max": 0.4951763153076172, "sampling/sampling_logp_difference/mean": 0.013619610108435154, "step": 495, "step_time": 15.350016831012908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 240.59375, "completions/mean_terminated_length": 240.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04523323546163738, "epoch": 0.00992, "frac_reward_zero_std": 0.0, "grad_norm": 0.02966424636542797, "kl": 0.7440181095153093, "learning_rate": 5.87608280365438e-06, "loss": -0.0364, "num_tokens": 24327760.0, "reward": 1.34375, "reward_std": 0.8667654991149902, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.0035220384597778, "sampling/importance_sampling_ratio/max": 1.0639219284057617, "sampling/importance_sampling_ratio/mean": 1.019451379776001, "sampling/importance_sampling_ratio/min": 1.0059040784835815, "sampling/sampling_logp_difference/max": 0.06202142685651779, "sampling/sampling_logp_difference/mean": 0.006053936202079058, "step": 496, "step_time": 14.562214206976932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2484.0, "completions/max_terminated_length": 2484.0, "completions/mean_length": 448.4375, "completions/mean_terminated_length": 448.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05081927147693932, "epoch": 0.00994, "frac_reward_zero_std": 0.0, "grad_norm": 0.2932394742965698, "kl": 1.8424465842545033, "learning_rate": 5.862533552767218e-06, "loss": -0.112, "num_tokens": 24382789.0, "reward": 1.75, "reward_std": 1.1751246452331543, "rewards/rollout_reward_func/mean": 1.75, "rewards/rollout_reward_func/std": 1.1359236240386963, "sampling/importance_sampling_ratio/max": 1.1245445013046265, "sampling/importance_sampling_ratio/mean": 1.010493516921997, "sampling/importance_sampling_ratio/min": 0.4303305447101593, "sampling/sampling_logp_difference/max": 0.8658051490783691, "sampling/sampling_logp_difference/mean": 0.014254504814743996, "step": 497, "step_time": 20.25270610001462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 261.09375, "completions/mean_terminated_length": 261.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05825067055411637, "epoch": 0.00996, "frac_reward_zero_std": 0.0, "grad_norm": 0.0603882260620594, "kl": 0.6762855900451541, "learning_rate": 5.848989412824425e-06, "loss": -0.053, "num_tokens": 24431463.0, "reward": 1.40625, "reward_std": 0.9147768020629883, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9455967545509338, "sampling/importance_sampling_ratio/max": 1.1775329113006592, "sampling/importance_sampling_ratio/mean": 1.0285567045211792, "sampling/importance_sampling_ratio/min": 1.0016405582427979, "sampling/sampling_logp_difference/max": 0.16344526410102844, "sampling/sampling_logp_difference/mean": 0.008108926936984062, "step": 498, "step_time": 13.291593185058446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 212.6875, "completions/mean_terminated_length": 212.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05605421820655465, "epoch": 0.00998, "frac_reward_zero_std": 0.0, "grad_norm": 0.21934592723846436, "kl": 2.5827599186450243, "learning_rate": 5.8354505624823585e-06, "loss": -0.0486, "num_tokens": 24477844.0, "reward": 1.0625, "reward_std": 1.1135666370391846, "rewards/rollout_reward_func/mean": 1.0625, "rewards/rollout_reward_func/std": 1.2164862155914307, "sampling/importance_sampling_ratio/max": 1.069158911705017, "sampling/importance_sampling_ratio/mean": 0.9816351532936096, "sampling/importance_sampling_ratio/min": 0.4519496560096741, "sampling/sampling_logp_difference/max": 0.7942256927490234, "sampling/sampling_logp_difference/mean": 0.02105456404387951, "step": 499, "step_time": 14.594587703060824 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 357.0625, "completions/mean_terminated_length": 357.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05098555970471352, "epoch": 0.01, "frac_reward_zero_std": 0.0, "grad_norm": 0.16997012495994568, "kl": 0.87129345536232, "learning_rate": 5.821917180327612e-06, "loss": -0.0494, "num_tokens": 24530129.0, "reward": 1.5625, "reward_std": 0.9460138082504272, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.9816871285438538, "sampling/importance_sampling_ratio/max": 1.0918132066726685, "sampling/importance_sampling_ratio/mean": 1.0003740787506104, "sampling/importance_sampling_ratio/min": 0.349516361951828, "sampling/sampling_logp_difference/max": 1.0581598281860352, "sampling/sampling_logp_difference/mean": 0.014081384986639023, "step": 500, "step_time": 13.960927583015291 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 282.1875, "completions/mean_terminated_length": 282.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04993079858832061, "epoch": 0.01002, "frac_reward_zero_std": 0.0, "grad_norm": 0.027138056233525276, "kl": 0.7615616843104362, "learning_rate": 5.808389444874641e-06, "loss": -0.0535, "num_tokens": 24579006.0, "reward": 1.59375, "reward_std": 0.761160671710968, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.756024181842804, "sampling/importance_sampling_ratio/max": 1.045803189277649, "sampling/importance_sampling_ratio/mean": 1.0067065954208374, "sampling/importance_sampling_ratio/min": 0.7208585739135742, "sampling/sampling_logp_difference/max": 0.32730793952941895, "sampling/sampling_logp_difference/mean": 0.007416464388370514, "step": 501, "step_time": 13.66727350599831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 244.875, "completions/mean_terminated_length": 244.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06462878221645951, "epoch": 0.01004, "frac_reward_zero_std": 0.0, "grad_norm": 0.34291985630989075, "kl": 4.46508114784956, "learning_rate": 5.794867534563422e-06, "loss": -0.0602, "num_tokens": 24625450.0, "reward": 1.28125, "reward_std": 0.9574502110481262, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.9913944005966187, "sampling/importance_sampling_ratio/max": 1.0792968273162842, "sampling/importance_sampling_ratio/mean": 0.9721320271492004, "sampling/importance_sampling_ratio/min": 0.37466147541999817, "sampling/sampling_logp_difference/max": 0.9816036224365234, "sampling/sampling_logp_difference/mean": 0.029746949672698975, "step": 502, "step_time": 14.453715487994486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 305.375, "completions/mean_terminated_length": 305.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.056456235935911536, "epoch": 0.01006, "frac_reward_zero_std": 0.0, "grad_norm": 0.06336680054664612, "kl": 0.5839304374530911, "learning_rate": 5.781351627757093e-06, "loss": -0.0346, "num_tokens": 24675639.0, "reward": 1.4375, "reward_std": 0.8738956451416016, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.0757592916488647, "sampling/importance_sampling_ratio/max": 1.0968070030212402, "sampling/importance_sampling_ratio/mean": 1.0225951671600342, "sampling/importance_sampling_ratio/min": 1.0009516477584839, "sampling/sampling_logp_difference/max": 0.09242911636829376, "sampling/sampling_logp_difference/mean": 0.006327519193291664, "step": 503, "step_time": 14.624674671998946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1571.0, "completions/max_terminated_length": 1571.0, "completions/mean_length": 303.4375, "completions/mean_terminated_length": 303.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05792485876008868, "epoch": 0.01008, "frac_reward_zero_std": 0.0, "grad_norm": 0.15558280050754547, "kl": 1.480612140148878, "learning_rate": 5.767841902739602e-06, "loss": -0.0647, "num_tokens": 24724291.0, "reward": 1.1875, "reward_std": 1.2899363040924072, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.2556324005126953, "sampling/importance_sampling_ratio/max": 1.0853991508483887, "sampling/importance_sampling_ratio/mean": 1.0032583475112915, "sampling/importance_sampling_ratio/min": 0.37441593408584595, "sampling/sampling_logp_difference/max": 0.9824016094207764, "sampling/sampling_logp_difference/mean": 0.016480915248394012, "step": 504, "step_time": 15.521406412983197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 366.4375, "completions/mean_terminated_length": 366.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0428816310595721, "epoch": 0.0101, "frac_reward_zero_std": 0.0, "grad_norm": 0.11629009246826172, "kl": 0.6535941623151302, "learning_rate": 5.754338537713353e-06, "loss": -0.0832, "num_tokens": 24776981.0, "reward": 1.96875, "reward_std": 0.7293993830680847, "rewards/rollout_reward_func/mean": 1.96875, "rewards/rollout_reward_func/std": 0.7398506999015808, "sampling/importance_sampling_ratio/max": 1.3470836877822876, "sampling/importance_sampling_ratio/mean": 1.0287712812423706, "sampling/importance_sampling_ratio/min": 1.0014194250106812, "sampling/sampling_logp_difference/max": 0.2818075120449066, "sampling/sampling_logp_difference/mean": 0.006728395354002714, "step": 505, "step_time": 16.335138534021098 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 321.96875, "completions/mean_terminated_length": 321.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04943880648352206, "epoch": 0.01012, "frac_reward_zero_std": 0.0, "grad_norm": 0.024209877476096153, "kl": 0.6243277806788683, "learning_rate": 5.740841710796861e-06, "loss": -0.0634, "num_tokens": 24826392.0, "reward": 1.5, "reward_std": 0.9922661781311035, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9837387204170227, "sampling/importance_sampling_ratio/max": 1.0557805299758911, "sampling/importance_sampling_ratio/mean": 1.0190091133117676, "sampling/importance_sampling_ratio/min": 1.0005053281784058, "sampling/sampling_logp_difference/max": 0.054283976554870605, "sampling/sampling_logp_difference/mean": 0.0053757429122924805, "step": 506, "step_time": 14.021311894975952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 303.15625, "completions/mean_terminated_length": 303.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06157241214532405, "epoch": 0.01014, "frac_reward_zero_std": 0.0, "grad_norm": 0.12063601613044739, "kl": 1.8023562375456095, "learning_rate": 5.727351600022396e-06, "loss": -0.0685, "num_tokens": 24875606.0, "reward": 1.5625, "reward_std": 0.8604230880737305, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.8775883316993713, "sampling/importance_sampling_ratio/max": 1.110426664352417, "sampling/importance_sampling_ratio/mean": 1.0049645900726318, "sampling/importance_sampling_ratio/min": 0.3301292657852173, "sampling/sampling_logp_difference/max": 1.1082558631896973, "sampling/sampling_logp_difference/mean": 0.018916942179203033, "step": 507, "step_time": 14.25623611699848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1301.0, "completions/max_terminated_length": 1301.0, "completions/mean_length": 356.21875, "completions/mean_terminated_length": 356.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04433771898038685, "epoch": 0.01016, "frac_reward_zero_std": 0.0, "grad_norm": 0.06371816247701645, "kl": 0.673696294426918, "learning_rate": 5.7138683833336385e-06, "loss": -0.0817, "num_tokens": 24927959.0, "reward": 1.6875, "reward_std": 0.9213650226593018, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.9979817867279053, "sampling/importance_sampling_ratio/max": 1.136757731437683, "sampling/importance_sampling_ratio/mean": 1.0242295265197754, "sampling/importance_sampling_ratio/min": 1.0014981031417847, "sampling/sampling_logp_difference/max": 0.12039585411548615, "sampling/sampling_logp_difference/mean": 0.006338212639093399, "step": 508, "step_time": 15.146361787992646 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 354.6875, "completions/mean_terminated_length": 354.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0504665900953114, "epoch": 0.01018, "frac_reward_zero_std": 0.0, "grad_norm": 0.05736808106303215, "kl": 0.7612782930955291, "learning_rate": 5.70039223858333e-06, "loss": -0.0669, "num_tokens": 24980731.0, "reward": 1.65625, "reward_std": 0.9230955839157104, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 1.0035220384597778, "sampling/importance_sampling_ratio/max": 1.1082857847213745, "sampling/importance_sampling_ratio/mean": 1.0057852268218994, "sampling/importance_sampling_ratio/min": 0.5640212893486023, "sampling/sampling_logp_difference/max": 0.5725376605987549, "sampling/sampling_logp_difference/mean": 0.010230633430182934, "step": 509, "step_time": 13.85823105397867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 385.03125, "completions/mean_terminated_length": 385.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.052477234858088195, "epoch": 0.0102, "frac_reward_zero_std": 0.0, "grad_norm": 0.09378521144390106, "kl": 0.7924843914806843, "learning_rate": 5.686923343530932e-06, "loss": -0.0581, "num_tokens": 25033733.0, "reward": 1.59375, "reward_std": 0.925920844078064, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.9791166186332703, "sampling/importance_sampling_ratio/max": 1.072269082069397, "sampling/importance_sampling_ratio/mean": 0.9964228272438049, "sampling/importance_sampling_ratio/min": 0.27280759811401367, "sampling/sampling_logp_difference/max": 1.3492887020111084, "sampling/sampling_logp_difference/mean": 0.015636326745152473, "step": 510, "step_time": 15.731957643001806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 284.28125, "completions/mean_terminated_length": 284.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05024354043416679, "epoch": 0.01022, "frac_reward_zero_std": 0.0, "grad_norm": 0.05659446865320206, "kl": 2.219845501706004, "learning_rate": 5.673461875840275e-06, "loss": -0.0625, "num_tokens": 25083221.0, "reward": 1.4375, "reward_std": 1.0763310194015503, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.1053389310836792, "sampling/importance_sampling_ratio/max": 1.0839896202087402, "sampling/importance_sampling_ratio/mean": 1.0213775634765625, "sampling/importance_sampling_ratio/min": 0.8830721974372864, "sampling/sampling_logp_difference/max": 0.12425947189331055, "sampling/sampling_logp_difference/mean": 0.008161716163158417, "step": 511, "step_time": 13.90104088402586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 329.1875, "completions/mean_terminated_length": 329.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.043870126944966614, "epoch": 0.01024, "frac_reward_zero_std": 0.0, "grad_norm": 0.05100803077220917, "kl": 0.7624031621962786, "learning_rate": 5.6600080130772166e-06, "loss": -0.0664, "num_tokens": 25134683.0, "reward": 1.875, "reward_std": 0.7224016189575195, "rewards/rollout_reward_func/mean": 1.875, "rewards/rollout_reward_func/std": 0.7513428926467896, "sampling/importance_sampling_ratio/max": 1.1333376169204712, "sampling/importance_sampling_ratio/mean": 1.0215727090835571, "sampling/importance_sampling_ratio/min": 1.0023460388183594, "sampling/sampling_logp_difference/max": 0.12516336143016815, "sampling/sampling_logp_difference/mean": 0.005974264815449715, "step": 512, "step_time": 13.526431539008627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 302.4375, "completions/mean_terminated_length": 302.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06246385769918561, "epoch": 0.01026, "frac_reward_zero_std": 0.0, "grad_norm": 0.19918647408485413, "kl": 1.4805419370532036, "learning_rate": 5.646561932707302e-06, "loss": -0.0667, "num_tokens": 25183880.0, "reward": 1.46875, "reward_std": 0.8979780077934265, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.949851393699646, "sampling/importance_sampling_ratio/max": 1.095615029335022, "sampling/importance_sampling_ratio/mean": 1.0009491443634033, "sampling/importance_sampling_ratio/min": 0.428063303232193, "sampling/sampling_logp_difference/max": 0.8681530952453613, "sampling/sampling_logp_difference/mean": 0.017709309235215187, "step": 513, "step_time": 14.60255530901486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 267.03125, "completions/mean_terminated_length": 267.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.051986103877425194, "epoch": 0.01028, "frac_reward_zero_std": 0.0, "grad_norm": 0.13221876323223114, "kl": 0.7101787626743317, "learning_rate": 5.633123812093419e-06, "loss": -0.0552, "num_tokens": 25231893.0, "reward": 1.53125, "reward_std": 0.7438260316848755, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.761339008808136, "sampling/importance_sampling_ratio/max": 1.0482888221740723, "sampling/importance_sampling_ratio/mean": 1.0172955989837646, "sampling/importance_sampling_ratio/min": 0.9691065549850464, "sampling/sampling_logp_difference/max": 0.04712396860122681, "sampling/sampling_logp_difference/mean": 0.005793025717139244, "step": 514, "step_time": 13.539264980965527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 222.0625, "completions/mean_terminated_length": 222.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04943054378964007, "epoch": 0.0103, "frac_reward_zero_std": 0.0, "grad_norm": 0.02702224813401699, "kl": 1.3164726700633764, "learning_rate": 5.619693828493467e-06, "loss": -0.0596, "num_tokens": 25277536.0, "reward": 1.40625, "reward_std": 0.846801221370697, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9108441472053528, "sampling/importance_sampling_ratio/max": 1.0593903064727783, "sampling/importance_sampling_ratio/mean": 0.9950811266899109, "sampling/importance_sampling_ratio/min": 0.29872044920921326, "sampling/sampling_logp_difference/max": 1.2081561088562012, "sampling/sampling_logp_difference/mean": 0.017886992543935776, "step": 515, "step_time": 13.274098764028167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1670.0, "completions/max_terminated_length": 1670.0, "completions/mean_length": 296.375, "completions/mean_terminated_length": 296.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04996308358386159, "epoch": 0.01032, "frac_reward_zero_std": 0.0, "grad_norm": 0.0627775490283966, "kl": 2.4382962863892317, "learning_rate": 5.606272159058005e-06, "loss": -0.0676, "num_tokens": 25326860.0, "reward": 1.5625, "reward_std": 0.9547792673110962, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.0429593324661255, "sampling/importance_sampling_ratio/mean": 1.0067893266677856, "sampling/importance_sampling_ratio/min": 0.6104435324668884, "sampling/sampling_logp_difference/max": 0.49356794357299805, "sampling/sampling_logp_difference/mean": 0.010332666337490082, "step": 516, "step_time": 16.827174444013508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 202.25, "completions/mean_terminated_length": 202.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06431660824455321, "epoch": 0.01034, "frac_reward_zero_std": 0.0, "grad_norm": 0.32675644755363464, "kl": 1.3722603227943182, "learning_rate": 5.5928589808279266e-06, "loss": -0.0581, "num_tokens": 25371603.0, "reward": 1.375, "reward_std": 0.7684495449066162, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.8327955007553101, "sampling/importance_sampling_ratio/max": 1.0593417882919312, "sampling/importance_sampling_ratio/mean": 0.9862761497497559, "sampling/importance_sampling_ratio/min": 0.43198728561401367, "sampling/sampling_logp_difference/max": 0.8393704891204834, "sampling/sampling_logp_difference/mean": 0.021776143461465836, "step": 517, "step_time": 12.31949826098571 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 250.4375, "completions/mean_terminated_length": 250.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08180417877156287, "epoch": 0.01036, "frac_reward_zero_std": 0.0, "grad_norm": 0.30815014243125916, "kl": 0.8816915545612574, "learning_rate": 5.5794544707321184e-06, "loss": -0.0568, "num_tokens": 25419431.0, "reward": 1.28125, "reward_std": 1.0303415060043335, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.0234153270721436, "sampling/importance_sampling_ratio/max": 1.0999314785003662, "sampling/importance_sampling_ratio/mean": 0.9666001796722412, "sampling/importance_sampling_ratio/min": 5.703796102096703e-09, "sampling/sampling_logp_difference/max": 18.744720458984375, "sampling/sampling_logp_difference/mean": 0.18676698207855225, "step": 518, "step_time": 14.766480344012962 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1661.0, "completions/max_terminated_length": 1661.0, "completions/mean_length": 296.96875, "completions/mean_terminated_length": 296.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.062321150093339384, "epoch": 0.01038, "frac_reward_zero_std": 0.0, "grad_norm": 0.06268885731697083, "kl": 1.7694702483713627, "learning_rate": 5.566058805585135e-06, "loss": -0.0666, "num_tokens": 25468180.0, "reward": 1.3125, "reward_std": 1.024105191230774, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.0906493663787842, "sampling/importance_sampling_ratio/max": 1.1616342067718506, "sampling/importance_sampling_ratio/mean": 1.0071724653244019, "sampling/importance_sampling_ratio/min": 0.3745763897895813, "sampling/sampling_logp_difference/max": 0.9821503162384033, "sampling/sampling_logp_difference/mean": 0.017708472907543182, "step": 519, "step_time": 16.578356965997955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 330.21875, "completions/mean_terminated_length": 330.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.061064021894708276, "epoch": 0.0104, "frac_reward_zero_std": 0.0, "grad_norm": 0.1284235715866089, "kl": 1.0584113206714392, "learning_rate": 5.55267216208485e-06, "loss": -0.0533, "num_tokens": 25519512.0, "reward": 1.59375, "reward_std": 0.9421594142913818, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.0734140872955322, "sampling/importance_sampling_ratio/max": 1.0753841400146484, "sampling/importance_sampling_ratio/mean": 1.0044829845428467, "sampling/importance_sampling_ratio/min": 0.4889681339263916, "sampling/sampling_logp_difference/max": 0.7154660224914551, "sampling/sampling_logp_difference/mean": 0.013236596249043941, "step": 520, "step_time": 15.498959638047381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 250.125, "completions/mean_terminated_length": 250.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05262994731310755, "epoch": 0.01042, "frac_reward_zero_std": 0.0, "grad_norm": 0.29036635160446167, "kl": 0.7523059993982315, "learning_rate": 5.539294716810144e-06, "loss": -0.0483, "num_tokens": 25567664.0, "reward": 1.46875, "reward_std": 0.8483964204788208, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.8793096542358398, "sampling/importance_sampling_ratio/max": 1.084110975265503, "sampling/importance_sampling_ratio/mean": 1.005819320678711, "sampling/importance_sampling_ratio/min": 0.5485065579414368, "sampling/sampling_logp_difference/max": 0.6005423069000244, "sampling/sampling_logp_difference/mean": 0.011509617790579796, "step": 521, "step_time": 14.076516401997651 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 252.3125, "completions/mean_terminated_length": 252.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07037584646604955, "epoch": 0.01044, "frac_reward_zero_std": 0.0, "grad_norm": 0.18852895498275757, "kl": 0.8371746297925711, "learning_rate": 5.525926646218561e-06, "loss": -0.0498, "num_tokens": 25613874.0, "reward": 1.4375, "reward_std": 0.7727276086807251, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.8007053136825562, "sampling/importance_sampling_ratio/max": 1.1499203443527222, "sampling/importance_sampling_ratio/mean": 1.0193638801574707, "sampling/importance_sampling_ratio/min": 0.5760675072669983, "sampling/sampling_logp_difference/max": 0.5529723167419434, "sampling/sampling_logp_difference/mean": 0.01587546244263649, "step": 522, "step_time": 14.007102468982339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 365.9375, "completions/mean_terminated_length": 365.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0525537688517943, "epoch": 0.01046, "frac_reward_zero_std": 0.0, "grad_norm": 0.0725618228316307, "kl": 0.7993987640365958, "learning_rate": 5.512568126643991e-06, "loss": -0.0562, "num_tokens": 25666495.0, "reward": 1.59375, "reward_std": 0.875164806842804, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.8747119307518005, "sampling/importance_sampling_ratio/max": 1.112173080444336, "sampling/importance_sampling_ratio/mean": 1.022926926612854, "sampling/importance_sampling_ratio/min": 1.000780701637268, "sampling/sampling_logp_difference/max": 0.10553433001041412, "sampling/sampling_logp_difference/mean": 0.006222185678780079, "step": 523, "step_time": 13.981385472987313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1607.0, "completions/max_terminated_length": 1607.0, "completions/mean_length": 339.125, "completions/mean_terminated_length": 339.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0597864156588912, "epoch": 0.01048, "frac_reward_zero_std": 0.0, "grad_norm": 0.20401230454444885, "kl": 1.8376051532104611, "learning_rate": 5.499219334294335e-06, "loss": -0.0775, "num_tokens": 25716211.0, "reward": 1.4375, "reward_std": 1.075290322303772, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.1622419357299805, "sampling/importance_sampling_ratio/max": 1.103743076324463, "sampling/importance_sampling_ratio/mean": 0.9708000421524048, "sampling/importance_sampling_ratio/min": 0.2958933413028717, "sampling/sampling_logp_difference/max": 1.2177283763885498, "sampling/sampling_logp_difference/mean": 0.029932580888271332, "step": 524, "step_time": 16.22228739799175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1328.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 239.03125, "completions/mean_terminated_length": 239.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05149866302963346, "epoch": 0.0105, "frac_reward_zero_std": 0.0, "grad_norm": 0.3158223330974579, "kl": 4.431758103892207, "learning_rate": 5.485880445249192e-06, "loss": -0.061, "num_tokens": 25763569.0, "reward": 1.4375, "reward_std": 0.9554010033607483, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9816871285438538, "sampling/importance_sampling_ratio/max": 1.3001508712768555, "sampling/importance_sampling_ratio/mean": 0.9977188110351562, "sampling/importance_sampling_ratio/min": 0.539094865322113, "sampling/sampling_logp_difference/max": 0.6177544593811035, "sampling/sampling_logp_difference/mean": 0.018455766141414642, "step": 525, "step_time": 14.813234948043828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1598.0, "completions/max_terminated_length": 1598.0, "completions/mean_length": 467.59375, "completions/mean_terminated_length": 467.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10324837011285126, "epoch": 0.01052, "frac_reward_zero_std": 0.0, "grad_norm": 0.39742982387542725, "kl": 1.3156514018774033, "learning_rate": 5.472551635457521e-06, "loss": -0.0597, "num_tokens": 25818965.0, "reward": 1.5, "reward_std": 1.1857529878616333, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.1639753580093384, "sampling/importance_sampling_ratio/max": 1.3324869871139526, "sampling/importance_sampling_ratio/mean": 0.9798476099967957, "sampling/importance_sampling_ratio/min": 0.27038657665252686, "sampling/sampling_logp_difference/max": 1.3142857551574707, "sampling/sampling_logp_difference/mean": 0.02699112705886364, "step": 526, "step_time": 15.94192234199727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 267.3125, "completions/mean_terminated_length": 267.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05276556091848761, "epoch": 0.01054, "frac_reward_zero_std": 0.0, "grad_norm": 0.16392534971237183, "kl": 0.7545105870813131, "learning_rate": 5.459233080735336e-06, "loss": -0.0584, "num_tokens": 25866778.0, "reward": 1.09375, "reward_std": 1.4236716032028198, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.4670438766479492, "sampling/importance_sampling_ratio/max": 1.0878568887710571, "sampling/importance_sampling_ratio/mean": 0.9863501787185669, "sampling/importance_sampling_ratio/min": 0.43196359276771545, "sampling/sampling_logp_difference/max": 0.8393714427947998, "sampling/sampling_logp_difference/mean": 0.018998391926288605, "step": 527, "step_time": 14.11726524400001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 372.9375, "completions/mean_terminated_length": 372.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05164441850502044, "epoch": 0.01056, "frac_reward_zero_std": 0.0, "grad_norm": 0.1368335634469986, "kl": 2.202199053019285, "learning_rate": 5.4459249567633776e-06, "loss": -0.0536, "num_tokens": 25918452.0, "reward": 1.40625, "reward_std": 1.217861533164978, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.18755304813385, "sampling/importance_sampling_ratio/max": 1.191821813583374, "sampling/importance_sampling_ratio/mean": 0.9951108694076538, "sampling/importance_sampling_ratio/min": 0.4298063814640045, "sampling/sampling_logp_difference/max": 0.8443422317504883, "sampling/sampling_logp_difference/mean": 0.021199239417910576, "step": 528, "step_time": 15.395525110012386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 255.34375, "completions/mean_terminated_length": 255.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06042054854333401, "epoch": 0.01058, "frac_reward_zero_std": 0.0, "grad_norm": 0.33327722549438477, "kl": 2.0983848981559277, "learning_rate": 5.432627439084797e-06, "loss": -0.0696, "num_tokens": 25966397.0, "reward": 1.1875, "reward_std": 1.2408065795898438, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.281065583229065, "sampling/importance_sampling_ratio/max": 1.1557977199554443, "sampling/importance_sampling_ratio/mean": 0.9790440797805786, "sampling/importance_sampling_ratio/min": 0.29930585622787476, "sampling/sampling_logp_difference/max": 1.2062840461730957, "sampling/sampling_logp_difference/mean": 0.025997020304203033, "step": 529, "step_time": 15.052932121005142 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 311.0, "completions/mean_terminated_length": 311.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06705114804208279, "epoch": 0.0106, "frac_reward_zero_std": 0.0, "grad_norm": 0.15815512835979462, "kl": 1.2262510564178228, "learning_rate": 5.419340703102839e-06, "loss": -0.0505, "num_tokens": 26015619.0, "reward": 1.5625, "reward_std": 0.9220276474952698, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.9136068224906921, "sampling/importance_sampling_ratio/max": 1.1014342308044434, "sampling/importance_sampling_ratio/mean": 0.9487708210945129, "sampling/importance_sampling_ratio/min": 3.414816873714699e-09, "sampling/sampling_logp_difference/max": 18.529165267944336, "sampling/sampling_logp_difference/mean": 0.19160321354866028, "step": 530, "step_time": 14.39595950099465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 337.96875, "completions/mean_terminated_length": 337.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05739062954671681, "epoch": 0.01062, "frac_reward_zero_std": 0.0, "grad_norm": 0.14782465994358063, "kl": 1.4115401450544596, "learning_rate": 5.406064924078536e-06, "loss": -0.0593, "num_tokens": 26066343.0, "reward": 1.375, "reward_std": 1.1559804677963257, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.1570261716842651, "sampling/importance_sampling_ratio/max": 1.082648515701294, "sampling/importance_sampling_ratio/mean": 1.010502815246582, "sampling/importance_sampling_ratio/min": 0.6092374324798584, "sampling/sampling_logp_difference/max": 0.49550867080688477, "sampling/sampling_logp_difference/mean": 0.010332038626074791, "step": 531, "step_time": 13.711903288014582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2133.0, "completions/max_terminated_length": 2133.0, "completions/mean_length": 258.4375, "completions/mean_terminated_length": 258.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04377466323785484, "epoch": 0.01064, "frac_reward_zero_std": 0.0, "grad_norm": 0.21200737357139587, "kl": 1.122314341366291, "learning_rate": 5.392800277128386e-06, "loss": -0.0434, "num_tokens": 26112999.0, "reward": 1.4375, "reward_std": 0.8580430746078491, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9136068224906921, "sampling/importance_sampling_ratio/max": 1.0647929906845093, "sampling/importance_sampling_ratio/mean": 0.9680566787719727, "sampling/importance_sampling_ratio/min": 0.15892678499221802, "sampling/sampling_logp_difference/max": 1.856536865234375, "sampling/sampling_logp_difference/mean": 0.031013716012239456, "step": 532, "step_time": 18.644646123982966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 255.15625, "completions/mean_terminated_length": 255.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.042870525852777064, "epoch": 0.01066, "frac_reward_zero_std": 0.0, "grad_norm": 0.11554234474897385, "kl": 0.7030113004148006, "learning_rate": 5.3795469372220525e-06, "loss": -0.0508, "num_tokens": 26161303.0, "reward": 1.59375, "reward_std": 0.8381726145744324, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.8747119307518005, "sampling/importance_sampling_ratio/max": 1.0883076190948486, "sampling/importance_sampling_ratio/mean": 1.018310785293579, "sampling/importance_sampling_ratio/min": 0.997432291507721, "sampling/sampling_logp_difference/max": 0.08464425802230835, "sampling/sampling_logp_difference/mean": 0.005618062801659107, "step": 533, "step_time": 14.130116630971315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 307.625, "completions/mean_terminated_length": 307.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0496963553596288, "epoch": 0.01068, "frac_reward_zero_std": 0.0, "grad_norm": 0.27096736431121826, "kl": 4.436789985746145, "learning_rate": 5.366305079180043e-06, "loss": -0.0544, "num_tokens": 26211713.0, "reward": 1.375, "reward_std": 1.0334365367889404, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.0395407676696777, "sampling/importance_sampling_ratio/max": 1.0941848754882812, "sampling/importance_sampling_ratio/mean": 0.9713655710220337, "sampling/importance_sampling_ratio/min": 0.3311862647533417, "sampling/sampling_logp_difference/max": 1.1049847602844238, "sampling/sampling_logp_difference/mean": 0.023829089477658272, "step": 534, "step_time": 13.620953157980694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 385.90625, "completions/mean_terminated_length": 385.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06869043211918324, "epoch": 0.0107, "frac_reward_zero_std": 0.0, "grad_norm": 1.5241724252700806, "kl": 0.6827614028006792, "learning_rate": 5.35307487767142e-06, "loss": -0.0774, "num_tokens": 26264949.0, "reward": 1.5625, "reward_std": 0.9493417739868164, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.9816871285438538, "sampling/importance_sampling_ratio/max": 1.1860058307647705, "sampling/importance_sampling_ratio/mean": 1.004124402999878, "sampling/importance_sampling_ratio/min": 0.7061329483985901, "sampling/sampling_logp_difference/max": 0.35461652278900146, "sampling/sampling_logp_difference/mean": 0.01179687213152647, "step": 535, "step_time": 14.088903085968923 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1270.0, "completions/max_terminated_length": 1270.0, "completions/mean_length": 307.59375, "completions/mean_terminated_length": 307.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.051454957108944654, "epoch": 0.01072, "frac_reward_zero_std": 0.0, "grad_norm": 0.5474433898925781, "kl": 1.4458179529756308, "learning_rate": 5.339856507211485e-06, "loss": -0.0521, "num_tokens": 26316100.0, "reward": 1.5625, "reward_std": 0.8933137655258179, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.0573080778121948, "sampling/importance_sampling_ratio/mean": 0.9787353277206421, "sampling/importance_sampling_ratio/min": 0.398989200592041, "sampling/sampling_logp_difference/max": 0.9186568260192871, "sampling/sampling_logp_difference/mean": 0.017806988209486008, "step": 536, "step_time": 15.435965967015363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 243.5625, "completions/mean_terminated_length": 243.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.03536455146968365, "epoch": 0.01074, "frac_reward_zero_std": 0.0, "grad_norm": 0.02315039001405239, "kl": 0.7222655396908522, "learning_rate": 5.326650142159479e-06, "loss": -0.0341, "num_tokens": 26362374.0, "reward": 1.3125, "reward_std": 0.7727276086807251, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.8206016421318054, "sampling/importance_sampling_ratio/max": 1.0555977821350098, "sampling/importance_sampling_ratio/mean": 1.0152208805084229, "sampling/importance_sampling_ratio/min": 1.0047518014907837, "sampling/sampling_logp_difference/max": 0.04582517594099045, "sampling/sampling_logp_difference/mean": 0.004667545203119516, "step": 537, "step_time": 13.724069442978362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 360.15625, "completions/mean_terminated_length": 360.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04291023802943528, "epoch": 0.01076, "frac_reward_zero_std": 0.0, "grad_norm": 0.12907084822654724, "kl": 0.7210882026702166, "learning_rate": 5.313455956716286e-06, "loss": -0.0333, "num_tokens": 26414737.0, "reward": 1.5625, "reward_std": 0.6616166830062866, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.6690146923065186, "sampling/importance_sampling_ratio/max": 1.373033881187439, "sampling/importance_sampling_ratio/mean": 1.0326502323150635, "sampling/importance_sampling_ratio/min": 1.000519871711731, "sampling/sampling_logp_difference/max": 0.16883090138435364, "sampling/sampling_logp_difference/mean": 0.00808272697031498, "step": 538, "step_time": 14.763313264993485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 802.0, "completions/max_terminated_length": 802.0, "completions/mean_length": 233.28125, "completions/mean_terminated_length": 233.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.034923289669677615, "epoch": 0.01078, "frac_reward_zero_std": 0.0, "grad_norm": 0.07633934170007706, "kl": 1.3797003459185362, "learning_rate": 5.3002741249221305e-06, "loss": -0.0585, "num_tokens": 26461624.0, "reward": 1.46875, "reward_std": 0.9121211171150208, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.949851393699646, "sampling/importance_sampling_ratio/max": 1.0344045162200928, "sampling/importance_sampling_ratio/mean": 0.9864422082901001, "sampling/importance_sampling_ratio/min": 0.17288237810134888, "sampling/sampling_logp_difference/max": 1.81699800491333, "sampling/sampling_logp_difference/mean": 0.021108832210302353, "step": 539, "step_time": 12.485736528979032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 251.15625, "completions/mean_terminated_length": 251.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.044498660834506154, "epoch": 0.0108, "frac_reward_zero_std": 0.0, "grad_norm": 0.12358521670103073, "kl": 2.67305526137352, "learning_rate": 5.287104820654283e-06, "loss": -0.0282, "num_tokens": 26509872.0, "reward": 1.1875, "reward_std": 0.9384939074516296, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 0.931093692779541, "sampling/importance_sampling_ratio/max": 1.10910165309906, "sampling/importance_sampling_ratio/mean": 1.0040202140808105, "sampling/importance_sampling_ratio/min": 0.5389928221702576, "sampling/sampling_logp_difference/max": 0.6180357933044434, "sampling/sampling_logp_difference/mean": 0.011501410976052284, "step": 540, "step_time": 13.666482448999886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 263.8125, "completions/mean_terminated_length": 263.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04918763169553131, "epoch": 0.01082, "frac_reward_zero_std": 0.0, "grad_norm": 0.0864177793264389, "kl": 0.6976499315351248, "learning_rate": 5.273948217624771e-06, "loss": -0.0392, "num_tokens": 26557602.0, "reward": 1.4375, "reward_std": 0.7981966733932495, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.8007053136825562, "sampling/importance_sampling_ratio/max": 1.2055563926696777, "sampling/importance_sampling_ratio/mean": 1.0262634754180908, "sampling/importance_sampling_ratio/min": 1.0009711980819702, "sampling/sampling_logp_difference/max": 0.18195772171020508, "sampling/sampling_logp_difference/mean": 0.007737189531326294, "step": 541, "step_time": 16.073237111006165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1571.0, "completions/max_terminated_length": 1571.0, "completions/mean_length": 361.15625, "completions/mean_terminated_length": 361.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05236098321620375, "epoch": 0.01084, "frac_reward_zero_std": 0.0, "grad_norm": 0.9345636367797852, "kl": 0.7522961124777794, "learning_rate": 5.260804489378083e-06, "loss": -0.1044, "num_tokens": 26608880.0, "reward": 1.78125, "reward_std": 0.8138851523399353, "rewards/rollout_reward_func/mean": 1.78125, "rewards/rollout_reward_func/std": 0.8321900367736816, "sampling/importance_sampling_ratio/max": 1.9208312034606934, "sampling/importance_sampling_ratio/mean": 1.0594648122787476, "sampling/importance_sampling_ratio/min": 0.9434375166893005, "sampling/sampling_logp_difference/max": 0.48158442974090576, "sampling/sampling_logp_difference/mean": 0.015042450278997421, "step": 542, "step_time": 16.31532746301673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 327.15625, "completions/mean_terminated_length": 327.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.03967329754959792, "epoch": 0.01086, "frac_reward_zero_std": 0.0, "grad_norm": 0.07902982831001282, "kl": 2.945635505951941, "learning_rate": 5.2476738092888805e-06, "loss": -0.0763, "num_tokens": 26659231.0, "reward": 1.59375, "reward_std": 0.988669753074646, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.0429292917251587, "sampling/importance_sampling_ratio/max": 1.2500206232070923, "sampling/importance_sampling_ratio/mean": 1.0112019777297974, "sampling/importance_sampling_ratio/min": 0.5405921339988708, "sampling/sampling_logp_difference/max": 0.6151280403137207, "sampling/sampling_logp_difference/mean": 0.011162715964019299, "step": 543, "step_time": 15.309275598963723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 212.65625, "completions/mean_terminated_length": 212.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.03769327246118337, "epoch": 0.01088, "frac_reward_zero_std": 0.0, "grad_norm": 0.021280504763126373, "kl": 0.7246777303516865, "learning_rate": 5.234556350559705e-06, "loss": -0.0457, "num_tokens": 26704694.0, "reward": 1.3125, "reward_std": 0.8900679349899292, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.931093692779541, "sampling/importance_sampling_ratio/max": 1.0826703310012817, "sampling/importance_sampling_ratio/mean": 1.0158612728118896, "sampling/importance_sampling_ratio/min": 1.0029031038284302, "sampling/sampling_logp_difference/max": 0.0794183537364006, "sampling/sampling_logp_difference/mean": 0.005045770667493343, "step": 544, "step_time": 14.38434533495456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1499.0, "completions/max_terminated_length": 1499.0, "completions/mean_length": 295.40625, "completions/mean_terminated_length": 295.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.03788271255325526, "epoch": 0.0109, "frac_reward_zero_std": 0.0, "grad_norm": 0.6288876533508301, "kl": 10.481778303161263, "learning_rate": 5.221452286218712e-06, "loss": -0.0619, "num_tokens": 26753827.0, "reward": 1.28125, "reward_std": 1.2665190696716309, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.2504031658172607, "sampling/importance_sampling_ratio/max": 1.05752694606781, "sampling/importance_sampling_ratio/mean": 0.9510358572006226, "sampling/importance_sampling_ratio/min": 0.25948265194892883, "sampling/sampling_logp_difference/max": 1.3490934371948242, "sampling/sampling_logp_difference/mean": 0.032558564096689224, "step": 545, "step_time": 15.203593145983177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 210.59375, "completions/mean_terminated_length": 210.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04728252405766398, "epoch": 0.01092, "frac_reward_zero_std": 0.0, "grad_norm": 0.041854556649923325, "kl": 1.8096843771636486, "learning_rate": 5.2083617891173625e-06, "loss": -0.0634, "num_tokens": 26797512.0, "reward": 1.34375, "reward_std": 0.7870134115219116, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 1.0367662906646729, "sampling/importance_sampling_ratio/mean": 0.9950569868087769, "sampling/importance_sampling_ratio/min": 0.37500637769699097, "sampling/sampling_logp_difference/max": 0.9808850288391113, "sampling/sampling_logp_difference/mean": 0.014425428584218025, "step": 546, "step_time": 13.41178483296244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1607.0, "completions/max_terminated_length": 1607.0, "completions/mean_length": 338.65625, "completions/mean_terminated_length": 338.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.042072139331139624, "epoch": 0.01094, "frac_reward_zero_std": 0.0, "grad_norm": 0.2350781261920929, "kl": 0.7662234492599964, "learning_rate": 5.195285031928168e-06, "loss": -0.0468, "num_tokens": 26849655.0, "reward": 1.65625, "reward_std": 0.7114962339401245, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.7452808618545532, "sampling/importance_sampling_ratio/max": 1.2780810594558716, "sampling/importance_sampling_ratio/mean": 1.0217349529266357, "sampling/importance_sampling_ratio/min": 1.003886103630066, "sampling/sampling_logp_difference/max": 0.20815229415893555, "sampling/sampling_logp_difference/mean": 0.0057370918802917, "step": 547, "step_time": 16.67633700699662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 338.5, "completions/mean_terminated_length": 338.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13747725437860936, "epoch": 0.01096, "frac_reward_zero_std": 0.0, "grad_norm": 0.4717409610748291, "kl": 3.185236958786845, "learning_rate": 5.182222187142388e-06, "loss": -0.0397, "num_tokens": 26901077.0, "reward": 1.53125, "reward_std": 0.7651960849761963, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.841825008392334, "sampling/importance_sampling_ratio/max": 1.0638220310211182, "sampling/importance_sampling_ratio/mean": 0.9757282137870789, "sampling/importance_sampling_ratio/min": 0.18224398791790009, "sampling/sampling_logp_difference/max": 1.5936570167541504, "sampling/sampling_logp_difference/mean": 0.02345426194369793, "step": 548, "step_time": 14.03227550603333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1679.0, "completions/max_terminated_length": 1679.0, "completions/mean_length": 325.4375, "completions/mean_terminated_length": 325.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.051760223461315036, "epoch": 0.01098, "frac_reward_zero_std": 0.0, "grad_norm": 0.3531344532966614, "kl": 0.7026428021490574, "learning_rate": 5.169173427067784e-06, "loss": -0.0747, "num_tokens": 26951770.0, "reward": 1.65625, "reward_std": 0.8536139726638794, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.9708451628684998, "sampling/importance_sampling_ratio/max": 1.234411358833313, "sampling/importance_sampling_ratio/mean": 1.0188387632369995, "sampling/importance_sampling_ratio/min": 0.8220882415771484, "sampling/sampling_logp_difference/max": 0.2024691104888916, "sampling/sampling_logp_difference/mean": 0.008433694019913673, "step": 549, "step_time": 16.215017914000782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 820.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 170.90625, "completions/mean_terminated_length": 170.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05089723889250308, "epoch": 0.011, "frac_reward_zero_std": 0.0, "grad_norm": 0.04381244629621506, "kl": 1.4628117457032204, "learning_rate": 5.156138923826317e-06, "loss": -0.0459, "num_tokens": 26996457.0, "reward": 1.21875, "reward_std": 0.9519853591918945, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.0390558242797852, "sampling/importance_sampling_ratio/max": 1.069899320602417, "sampling/importance_sampling_ratio/mean": 0.9978997111320496, "sampling/importance_sampling_ratio/min": 0.3884580433368683, "sampling/sampling_logp_difference/max": 0.9456110000610352, "sampling/sampling_logp_difference/mean": 0.015390846878290176, "step": 550, "step_time": 12.838283206016058 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 228.125, "completions/mean_terminated_length": 228.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05098956951405853, "epoch": 0.01102, "frac_reward_zero_std": 0.0, "grad_norm": 0.20205150544643402, "kl": 1.9499027840793133, "learning_rate": 5.143118849351898e-06, "loss": -0.0743, "num_tokens": 27042435.0, "reward": 1.3125, "reward_std": 0.9499588012695312, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.029797911643982, "sampling/importance_sampling_ratio/max": 1.0394108295440674, "sampling/importance_sampling_ratio/mean": 0.9472538232803345, "sampling/importance_sampling_ratio/min": 0.2904457449913025, "sampling/sampling_logp_difference/max": 1.237825870513916, "sampling/sampling_logp_difference/mean": 0.03841017931699753, "step": 551, "step_time": 15.262356988998363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 280.625, "completions/mean_terminated_length": 280.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05893095815554261, "epoch": 0.01104, "frac_reward_zero_std": 0.0, "grad_norm": 0.5474704504013062, "kl": 2.093672029674053, "learning_rate": 5.1301133753881115e-06, "loss": -0.0361, "num_tokens": 27092686.0, "reward": 1.25, "reward_std": 1.188399076461792, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.2181423902511597, "sampling/importance_sampling_ratio/max": 1.1693679094314575, "sampling/importance_sampling_ratio/mean": 1.0161464214324951, "sampling/importance_sampling_ratio/min": 0.7849123477935791, "sampling/sampling_logp_difference/max": 0.24232721328735352, "sampling/sampling_logp_difference/mean": 0.00929209403693676, "step": 552, "step_time": 12.352952995919622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 283.03125, "completions/mean_terminated_length": 283.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04782107216306031, "epoch": 0.01106, "frac_reward_zero_std": 0.0, "grad_norm": 0.1797013282775879, "kl": 1.879569873213768, "learning_rate": 5.1171226734859505e-06, "loss": -0.0684, "num_tokens": 27140509.0, "reward": 1.46875, "reward_std": 1.0319900512695312, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.0155048370361328, "sampling/importance_sampling_ratio/max": 1.0958178043365479, "sampling/importance_sampling_ratio/mean": 0.9846041202545166, "sampling/importance_sampling_ratio/min": 0.38114190101623535, "sampling/sampling_logp_difference/max": 0.9646539688110352, "sampling/sampling_logp_difference/mean": 0.021327538415789604, "step": 553, "step_time": 14.418588185988483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 273.5, "completions/mean_terminated_length": 273.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.049470493104308844, "epoch": 0.01108, "frac_reward_zero_std": 0.0, "grad_norm": 0.28726425766944885, "kl": 0.7001199331134558, "learning_rate": 5.1041469150015535e-06, "loss": -0.0189, "num_tokens": 27189417.0, "reward": 1.15625, "reward_std": 0.9401702880859375, "rewards/rollout_reward_func/mean": 1.15625, "rewards/rollout_reward_func/std": 0.9540871381759644, "sampling/importance_sampling_ratio/max": 1.2051515579223633, "sampling/importance_sampling_ratio/mean": 0.9910358786582947, "sampling/importance_sampling_ratio/min": 0.434942364692688, "sampling/sampling_logp_difference/max": 0.8325183391571045, "sampling/sampling_logp_difference/mean": 0.02191191166639328, "step": 554, "step_time": 13.657956080016447 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1535.0, "completions/max_terminated_length": 1535.0, "completions/mean_length": 306.03125, "completions/mean_terminated_length": 306.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04770445206668228, "epoch": 0.0111, "frac_reward_zero_std": 0.0, "grad_norm": 0.1662331372499466, "kl": 1.8398576341569424, "learning_rate": 5.091186271093949e-06, "loss": -0.0499, "num_tokens": 27240453.0, "reward": 1.59375, "reward_std": 0.8918143510818481, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.011526346206665, "sampling/importance_sampling_ratio/max": 1.0620007514953613, "sampling/importance_sampling_ratio/mean": 0.9856340885162354, "sampling/importance_sampling_ratio/min": 0.4229898750782013, "sampling/sampling_logp_difference/max": 0.8602776527404785, "sampling/sampling_logp_difference/mean": 0.014707392081618309, "step": 555, "step_time": 15.60890909601585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 253.375, "completions/mean_terminated_length": 253.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04430479777511209, "epoch": 0.01112, "frac_reward_zero_std": 0.0, "grad_norm": 0.2760268449783325, "kl": 0.6729493597522378, "learning_rate": 5.078240912722787e-06, "loss": -0.0491, "num_tokens": 27288825.0, "reward": 1.625, "reward_std": 0.614006519317627, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.6599119901657104, "sampling/importance_sampling_ratio/max": 1.1560375690460205, "sampling/importance_sampling_ratio/mean": 1.0194264650344849, "sampling/importance_sampling_ratio/min": 1.0013983249664307, "sampling/sampling_logp_difference/max": 0.14428648352622986, "sampling/sampling_logp_difference/mean": 0.005693025887012482, "step": 556, "step_time": 13.729664138998487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 347.53125, "completions/mean_terminated_length": 347.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04772258666343987, "epoch": 0.01114, "frac_reward_zero_std": 0.0, "grad_norm": 0.3046954572200775, "kl": 1.036294300109148, "learning_rate": 5.065311010646101e-06, "loss": -0.0791, "num_tokens": 27340292.0, "reward": 1.5625, "reward_std": 1.1021817922592163, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 1.0757592916488647, "sampling/importance_sampling_ratio/max": 1.1224939823150635, "sampling/importance_sampling_ratio/mean": 0.9956333637237549, "sampling/importance_sampling_ratio/min": 0.18728338181972504, "sampling/sampling_logp_difference/max": 1.6896581649780273, "sampling/sampling_logp_difference/mean": 0.02038806676864624, "step": 557, "step_time": 15.57096115098102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1346.0, "completions/max_terminated_length": 1346.0, "completions/mean_length": 281.0625, "completions/mean_terminated_length": 281.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04764269827865064, "epoch": 0.01116, "frac_reward_zero_std": 0.0, "grad_norm": 0.28248974680900574, "kl": 3.5651475209742785, "learning_rate": 5.052396735418037e-06, "loss": -0.0759, "num_tokens": 27388173.0, "reward": 1.46875, "reward_std": 1.0528347492218018, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.1635422706604004, "sampling/importance_sampling_ratio/max": 1.0958105325698853, "sampling/importance_sampling_ratio/mean": 1.0114107131958008, "sampling/importance_sampling_ratio/min": 0.6083773970603943, "sampling/sampling_logp_difference/max": 0.4969358444213867, "sampling/sampling_logp_difference/mean": 0.011184444651007652, "step": 558, "step_time": 15.254605092966813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 303.125, "completions/mean_terminated_length": 303.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04940228583291173, "epoch": 0.01118, "frac_reward_zero_std": 0.0, "grad_norm": 0.1906096339225769, "kl": 1.1824469156563282, "learning_rate": 5.039498257386617e-06, "loss": -0.095, "num_tokens": 27436743.0, "reward": 1.6875, "reward_std": 0.9452994465827942, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.931093692779541, "sampling/importance_sampling_ratio/max": 1.136138677597046, "sampling/importance_sampling_ratio/mean": 1.0048527717590332, "sampling/importance_sampling_ratio/min": 0.4786671996116638, "sampling/sampling_logp_difference/max": 0.7367067337036133, "sampling/sampling_logp_difference/mean": 0.01276198960840702, "step": 559, "step_time": 15.103679279010976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2524.0, "completions/max_terminated_length": 2524.0, "completions/mean_length": 426.625, "completions/mean_terminated_length": 426.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04866787535138428, "epoch": 0.0112, "frac_reward_zero_std": 0.0, "grad_norm": 0.14157229661941528, "kl": 0.8031516838818789, "learning_rate": 5.026615746691483e-06, "loss": -0.0772, "num_tokens": 27491165.0, "reward": 1.65625, "reward_std": 1.0356981754302979, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 1.180742621421814, "sampling/importance_sampling_ratio/max": 1.232375144958496, "sampling/importance_sampling_ratio/mean": 0.997700572013855, "sampling/importance_sampling_ratio/min": 0.2756166458129883, "sampling/sampling_logp_difference/max": 1.2886991500854492, "sampling/sampling_logp_difference/mean": 0.01960805431008339, "step": 560, "step_time": 20.138625787003548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 252.96875, "completions/mean_terminated_length": 252.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04789076920133084, "epoch": 0.01122, "frac_reward_zero_std": 0.0, "grad_norm": 0.0327153205871582, "kl": 2.1721367184072733, "learning_rate": 5.013749373261662e-06, "loss": -0.0397, "num_tokens": 27538979.0, "reward": 1.28125, "reward_std": 0.9633897542953491, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.9583041071891785, "sampling/importance_sampling_ratio/max": 1.3083194494247437, "sampling/importance_sampling_ratio/mean": 1.0142402648925781, "sampling/importance_sampling_ratio/min": 0.6113470196723938, "sampling/sampling_logp_difference/max": 0.49222755432128906, "sampling/sampling_logp_difference/mean": 0.012155972421169281, "step": 561, "step_time": 14.417534939013422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 184.0, "completions/mean_terminated_length": 184.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.050273405853658915, "epoch": 0.01124, "frac_reward_zero_std": 0.0, "grad_norm": 0.08837810158729553, "kl": 2.111183892004192, "learning_rate": 5.0008993068133165e-06, "loss": -0.0463, "num_tokens": 27583861.0, "reward": 1.125, "reward_std": 0.9530547261238098, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.9755064845085144, "sampling/importance_sampling_ratio/max": 1.1128450632095337, "sampling/importance_sampling_ratio/mean": 0.9892511963844299, "sampling/importance_sampling_ratio/min": 0.33854079246520996, "sampling/sampling_logp_difference/max": 1.083113431930542, "sampling/sampling_logp_difference/mean": 0.024198902770876884, "step": 562, "step_time": 13.584570332008298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2515.0, "completions/max_terminated_length": 2515.0, "completions/mean_length": 356.65625, "completions/mean_terminated_length": 356.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04998483415693045, "epoch": 0.01126, "frac_reward_zero_std": 0.0, "grad_norm": 0.7168739438056946, "kl": 1.1451443005353212, "learning_rate": 4.98806571684751e-06, "loss": -0.1156, "num_tokens": 27634693.0, "reward": 1.59375, "reward_std": 0.8957716226577759, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 1.0429292917251587, "sampling/importance_sampling_ratio/max": 1.9415220022201538, "sampling/importance_sampling_ratio/mean": 1.0121941566467285, "sampling/importance_sampling_ratio/min": 0.4252904951572418, "sampling/sampling_logp_difference/max": 0.8651552200317383, "sampling/sampling_logp_difference/mean": 0.02631467953324318, "step": 563, "step_time": 20.272851682981127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 201.25, "completions/mean_terminated_length": 201.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.056077928049489856, "epoch": 0.01128, "frac_reward_zero_std": 0.0, "grad_norm": 0.32129383087158203, "kl": 1.3836487252265215, "learning_rate": 4.975248772647969e-06, "loss": -0.0404, "num_tokens": 27680131.0, "reward": 1.28125, "reward_std": 0.966189980506897, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.9583041071891785, "sampling/importance_sampling_ratio/max": 1.1152478456497192, "sampling/importance_sampling_ratio/mean": 0.9801700115203857, "sampling/importance_sampling_ratio/min": 0.2700822055339813, "sampling/sampling_logp_difference/max": 1.3090720176696777, "sampling/sampling_logp_difference/mean": 0.02330346591770649, "step": 564, "step_time": 11.992738738990738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 811.0, "completions/max_terminated_length": 811.0, "completions/mean_length": 237.5, "completions/mean_terminated_length": 237.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.03896765201352537, "epoch": 0.0113, "frac_reward_zero_std": 0.0, "grad_norm": 0.07975989580154419, "kl": 2.918852038681507, "learning_rate": 4.962448643278852e-06, "loss": -0.0493, "num_tokens": 27727276.0, "reward": 1.40625, "reward_std": 0.8908449411392212, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9108441472053528, "sampling/importance_sampling_ratio/max": 1.204797625541687, "sampling/importance_sampling_ratio/mean": 0.9755691289901733, "sampling/importance_sampling_ratio/min": 0.25980260968208313, "sampling/sampling_logp_difference/max": 1.3478703498840332, "sampling/sampling_logp_difference/mean": 0.02882034331560135, "step": 565, "step_time": 12.044477605973952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 766.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 267.96875, "completions/mean_terminated_length": 267.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05095998418983072, "epoch": 0.01132, "frac_reward_zero_std": 0.0, "grad_norm": 0.37138399481773376, "kl": 0.7105943653732538, "learning_rate": 4.949665497582519e-06, "loss": -0.0432, "num_tokens": 27776103.0, "reward": 1.5625, "reward_std": 0.7356451749801636, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.8007053136825562, "sampling/importance_sampling_ratio/max": 1.0537348985671997, "sampling/importance_sampling_ratio/mean": 1.0098004341125488, "sampling/importance_sampling_ratio/min": 0.8538264632225037, "sampling/sampling_logp_difference/max": 0.16486573219299316, "sampling/sampling_logp_difference/mean": 0.006347172893583775, "step": 566, "step_time": 12.098341830991558 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 319.0, "completions/mean_terminated_length": 319.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04263381357304752, "epoch": 0.01134, "frac_reward_zero_std": 0.0, "grad_norm": 0.0708019882440567, "kl": 2.380285686813295, "learning_rate": 4.936899504177303e-06, "loss": -0.0464, "num_tokens": 27826215.0, "reward": 1.46875, "reward_std": 0.9766610860824585, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.9832262396812439, "sampling/importance_sampling_ratio/max": 1.0809569358825684, "sampling/importance_sampling_ratio/mean": 1.0057094097137451, "sampling/importance_sampling_ratio/min": 0.6110399961471558, "sampling/sampling_logp_difference/max": 0.49254798889160156, "sampling/sampling_logp_difference/mean": 0.010419935919344425, "step": 567, "step_time": 15.893353378007305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 354.125, "completions/mean_terminated_length": 354.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04627755330875516, "epoch": 0.01136, "frac_reward_zero_std": 0.0, "grad_norm": 0.11004626005887985, "kl": 0.837645161896944, "learning_rate": 4.9241508314552856e-06, "loss": -0.0568, "num_tokens": 27877199.0, "reward": 1.53125, "reward_std": 0.8277777433395386, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.841825008392334, "sampling/importance_sampling_ratio/max": 1.7272523641586304, "sampling/importance_sampling_ratio/mean": 1.0196751356124878, "sampling/importance_sampling_ratio/min": 0.2423471361398697, "sampling/sampling_logp_difference/max": 1.4543511867523193, "sampling/sampling_logp_difference/mean": 0.021981215104460716, "step": 568, "step_time": 15.478983994020382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 366.5, "completions/mean_terminated_length": 366.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.039799833903089166, "epoch": 0.01138, "frac_reward_zero_std": 0.0, "grad_norm": 0.016467392444610596, "kl": 0.7182892374694347, "learning_rate": 4.911419647580079e-06, "loss": -0.0487, "num_tokens": 27929630.0, "reward": 1.65625, "reward_std": 0.7212116718292236, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.7006621360778809, "sampling/importance_sampling_ratio/max": 1.058200716972351, "sampling/importance_sampling_ratio/mean": 1.016961932182312, "sampling/importance_sampling_ratio/min": 1.0051319599151611, "sampling/sampling_logp_difference/max": 0.056637369096279144, "sampling/sampling_logp_difference/mean": 0.004567008465528488, "step": 569, "step_time": 13.559530658007134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 226.28125, "completions/mean_terminated_length": 226.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.049264705274254084, "epoch": 0.0114, "frac_reward_zero_std": 0.0, "grad_norm": 1.381972074508667, "kl": 6.279066637158394, "learning_rate": 4.898706120484606e-06, "loss": -0.0305, "num_tokens": 27975573.0, "reward": 1.09375, "reward_std": 1.0087909698486328, "rewards/rollout_reward_func/mean": 1.09375, "rewards/rollout_reward_func/std": 1.0273478031158447, "sampling/importance_sampling_ratio/max": 1.077020525932312, "sampling/importance_sampling_ratio/mean": 0.9393138885498047, "sampling/importance_sampling_ratio/min": 0.20259152352809906, "sampling/sampling_logp_difference/max": 1.6145105361938477, "sampling/sampling_logp_difference/mean": 0.04668797552585602, "step": 570, "step_time": 14.48001557198586 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 297.625, "completions/mean_terminated_length": 297.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05608301528263837, "epoch": 0.01142, "frac_reward_zero_std": 0.0, "grad_norm": 0.43621641397476196, "kl": 1.7442483436316252, "learning_rate": 4.886010417868881e-06, "loss": -0.0513, "num_tokens": 28025457.0, "reward": 1.53125, "reward_std": 0.9370410442352295, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 1.0467884540557861, "sampling/importance_sampling_ratio/max": 1.1500650644302368, "sampling/importance_sampling_ratio/mean": 0.9180441498756409, "sampling/importance_sampling_ratio/min": 4.99728348979378e-11, "sampling/sampling_logp_difference/max": 23.91929054260254, "sampling/sampling_logp_difference/mean": 0.23971116542816162, "step": 571, "step_time": 13.823248783999588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 215.875, "completions/mean_terminated_length": 215.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04933591419830918, "epoch": 0.01144, "frac_reward_zero_std": 0.0, "grad_norm": 0.2924031913280487, "kl": 0.7404092829674482, "learning_rate": 4.873332707197804e-06, "loss": -0.059, "num_tokens": 28070864.0, "reward": 1.28125, "reward_std": 0.9493400454521179, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.0234153270721436, "sampling/importance_sampling_ratio/max": 1.1265618801116943, "sampling/importance_sampling_ratio/mean": 1.0091698169708252, "sampling/importance_sampling_ratio/min": 0.5648952126502991, "sampling/sampling_logp_difference/max": 0.5711460113525391, "sampling/sampling_logp_difference/mean": 0.012703105807304382, "step": 572, "step_time": 12.273507017016527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1899.0, "completions/max_terminated_length": 1899.0, "completions/mean_length": 281.625, "completions/mean_terminated_length": 281.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.057002809597179294, "epoch": 0.01146, "frac_reward_zero_std": 0.0, "grad_norm": 0.4531663954257965, "kl": 0.7088623642921448, "learning_rate": 4.860673155698953e-06, "loss": -0.0793, "num_tokens": 28118416.0, "reward": 1.46875, "reward_std": 1.083348274230957, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.1354798078536987, "sampling/importance_sampling_ratio/max": 1.2971839904785156, "sampling/importance_sampling_ratio/mean": 1.0287117958068848, "sampling/importance_sampling_ratio/min": 0.9024885892868042, "sampling/sampling_logp_difference/max": 0.25751256942749023, "sampling/sampling_logp_difference/mean": 0.010088244453072548, "step": 573, "step_time": 17.22952969004109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 340.125, "completions/mean_terminated_length": 340.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05128414696082473, "epoch": 0.01148, "frac_reward_zero_std": 0.0, "grad_norm": 0.13964799046516418, "kl": 0.8232049494981766, "learning_rate": 4.848031930360365e-06, "loss": -0.0442, "num_tokens": 28169215.0, "reward": 1.46875, "reward_std": 0.8749716281890869, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.915260374546051, "sampling/importance_sampling_ratio/max": 1.0801031589508057, "sampling/importance_sampling_ratio/mean": 1.021833896636963, "sampling/importance_sampling_ratio/min": 1.005933165550232, "sampling/sampling_logp_difference/max": 0.062384605407714844, "sampling/sampling_logp_difference/mean": 0.006214193068444729, "step": 574, "step_time": 14.394694768998306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 271.53125, "completions/mean_terminated_length": 271.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04239384888205677, "epoch": 0.0115, "frac_reward_zero_std": 0.0, "grad_norm": 0.04701114445924759, "kl": 0.7685212008655071, "learning_rate": 4.835409197928351e-06, "loss": -0.0402, "num_tokens": 28216300.0, "reward": 1.40625, "reward_std": 0.7494902610778809, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.797551691532135, "sampling/importance_sampling_ratio/max": 1.0828006267547607, "sampling/importance_sampling_ratio/mean": 1.0205228328704834, "sampling/importance_sampling_ratio/min": 1.0008091926574707, "sampling/sampling_logp_difference/max": 0.06906203925609589, "sampling/sampling_logp_difference/mean": 0.006018606014549732, "step": 575, "step_time": 14.422360647993628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 245.375, "completions/mean_terminated_length": 245.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.03541055123787373, "epoch": 0.01152, "frac_reward_zero_std": 0.0, "grad_norm": 0.020719725638628006, "kl": 0.6196196349337697, "learning_rate": 4.822805124905282e-06, "loss": -0.0674, "num_tokens": 28263529.0, "reward": 1.46875, "reward_std": 0.9343355894088745, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.9832262396812439, "sampling/importance_sampling_ratio/max": 1.1135623455047607, "sampling/importance_sampling_ratio/mean": 1.0163750648498535, "sampling/importance_sampling_ratio/min": 1.0029717683792114, "sampling/sampling_logp_difference/max": 0.10282225906848907, "sampling/sampling_logp_difference/mean": 0.004893001168966293, "step": 576, "step_time": 13.607693618992926 }, { "clip_ratio/high_max": 0.005681818351149559, "clip_ratio/high_mean": 0.0028409091755747795, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01065340917557478, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 243.625, "completions/mean_terminated_length": 243.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10248988727107644, "epoch": 0.01154, "frac_reward_zero_std": 0.0, "grad_norm": 0.37072697281837463, "kl": 0.9222978502511978, "learning_rate": 4.810219877547406e-06, "loss": -0.064, "num_tokens": 28311212.0, "reward": 1.40625, "reward_std": 0.9472306966781616, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9455967545509338, "sampling/importance_sampling_ratio/max": 1.071657657623291, "sampling/importance_sampling_ratio/mean": 0.9994951486587524, "sampling/importance_sampling_ratio/min": 0.4910520315170288, "sampling/sampling_logp_difference/max": 0.7156476974487305, "sampling/sampling_logp_difference/mean": 0.012446967884898186, "step": 577, "step_time": 13.31073089002166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1409.0, "completions/max_terminated_length": 1409.0, "completions/mean_length": 392.875, "completions/mean_terminated_length": 392.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.03937035799026489, "epoch": 0.01156, "frac_reward_zero_std": 0.0, "grad_norm": 0.054898519068956375, "kl": 0.8009003512561321, "learning_rate": 4.797653621862637e-06, "loss": -0.0722, "num_tokens": 28364272.0, "reward": 1.84375, "reward_std": 0.892972469329834, "rewards/rollout_reward_func/mean": 1.84375, "rewards/rollout_reward_func/std": 0.919655442237854, "sampling/importance_sampling_ratio/max": 1.095548391342163, "sampling/importance_sampling_ratio/mean": 1.02171790599823, "sampling/importance_sampling_ratio/min": 1.0072609186172485, "sampling/sampling_logp_difference/max": 0.08374310284852982, "sampling/sampling_logp_difference/mean": 0.005386988632380962, "step": 578, "step_time": 15.992077134971623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 300.15625, "completions/mean_terminated_length": 300.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04762755590490997, "epoch": 0.01158, "frac_reward_zero_std": 0.0, "grad_norm": 0.15081076323986053, "kl": 1.4845580160617828, "learning_rate": 4.785106523608388e-06, "loss": -0.0532, "num_tokens": 28414197.0, "reward": 1.40625, "reward_std": 0.9790077209472656, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.011526346206665, "sampling/importance_sampling_ratio/max": 1.1063469648361206, "sampling/importance_sampling_ratio/mean": 0.9902609586715698, "sampling/importance_sampling_ratio/min": 0.4764520525932312, "sampling/sampling_logp_difference/max": 0.7477169036865234, "sampling/sampling_logp_difference/mean": 0.01775233820080757, "step": 579, "step_time": 13.672508533010841 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 315.90625, "completions/mean_terminated_length": 315.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04714487644378096, "epoch": 0.0116, "frac_reward_zero_std": 0.0, "grad_norm": 0.11118737608194351, "kl": 1.4444379936903715, "learning_rate": 4.7725787482893645e-06, "loss": -0.0435, "num_tokens": 28464639.0, "reward": 1.40625, "reward_std": 0.9176375865936279, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.0429292917251587, "sampling/importance_sampling_ratio/max": 1.0647755861282349, "sampling/importance_sampling_ratio/mean": 0.9854443073272705, "sampling/importance_sampling_ratio/min": 0.443263441324234, "sampling/sampling_logp_difference/max": 0.8136858940124512, "sampling/sampling_logp_difference/mean": 0.019151391461491585, "step": 580, "step_time": 14.376741512969602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 2142.0, "completions/max_terminated_length": 2142.0, "completions/mean_length": 362.375, "completions/mean_terminated_length": 362.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.043085628538392484, "epoch": 0.01162, "frac_reward_zero_std": 0.0, "grad_norm": 0.2527630031108856, "kl": 1.140782231464982, "learning_rate": 4.760070461155393e-06, "loss": -0.0827, "num_tokens": 28517635.0, "reward": 1.75, "reward_std": 0.9039424657821655, "rewards/rollout_reward_func/mean": 1.75, "rewards/rollout_reward_func/std": 0.9837387204170227, "sampling/importance_sampling_ratio/max": 1.2848817110061646, "sampling/importance_sampling_ratio/mean": 1.0095584392547607, "sampling/importance_sampling_ratio/min": 0.2706436812877655, "sampling/sampling_logp_difference/max": 1.309140920639038, "sampling/sampling_logp_difference/mean": 0.018767578527331352, "step": 581, "step_time": 19.500424261015723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 377.875, "completions/mean_terminated_length": 377.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05428939242847264, "epoch": 0.01164, "frac_reward_zero_std": 0.0, "grad_norm": 0.3058634400367737, "kl": 0.6973444409668446, "learning_rate": 4.7475818271992335e-06, "loss": -0.0737, "num_tokens": 28570038.0, "reward": 1.8125, "reward_std": 0.7930147051811218, "rewards/rollout_reward_func/mean": 1.8125, "rewards/rollout_reward_func/std": 0.7803018093109131, "sampling/importance_sampling_ratio/max": 1.1184077262878418, "sampling/importance_sampling_ratio/mean": 0.9949795603752136, "sampling/importance_sampling_ratio/min": 0.5299368500709534, "sampling/sampling_logp_difference/max": 0.6394507884979248, "sampling/sampling_logp_difference/mean": 0.01495174691081047, "step": 582, "step_time": 14.46694412099896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 257.15625, "completions/mean_terminated_length": 257.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04693469719495624, "epoch": 0.01166, "frac_reward_zero_std": 0.0, "grad_norm": 0.06603390723466873, "kl": 0.7103128917515278, "learning_rate": 4.735113011154413e-06, "loss": -0.0478, "num_tokens": 28619667.0, "reward": 1.65625, "reward_std": 0.6215572357177734, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.6015772223472595, "sampling/importance_sampling_ratio/max": 1.1967873573303223, "sampling/importance_sampling_ratio/mean": 1.0283514261245728, "sampling/importance_sampling_ratio/min": 1.0029264688491821, "sampling/sampling_logp_difference/max": 0.1603095531463623, "sampling/sampling_logp_difference/mean": 0.007811025716364384, "step": 583, "step_time": 13.375173298016307 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1598.0, "completions/max_terminated_length": 1598.0, "completions/mean_length": 398.78125, "completions/mean_terminated_length": 398.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04677172051742673, "epoch": 0.01168, "frac_reward_zero_std": 0.0, "grad_norm": 0.22494415938854218, "kl": 0.8433660753071308, "learning_rate": 4.722664177493042e-06, "loss": -0.0447, "num_tokens": 28672946.0, "reward": 1.59375, "reward_std": 0.7085968255996704, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.756024181842804, "sampling/importance_sampling_ratio/max": 1.5394256114959717, "sampling/importance_sampling_ratio/mean": 1.0055830478668213, "sampling/importance_sampling_ratio/min": 0.5811366438865662, "sampling/sampling_logp_difference/max": 0.7380056381225586, "sampling/sampling_logp_difference/mean": 0.016991043463349342, "step": 584, "step_time": 17.01011104004283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 411.4375, "completions/mean_terminated_length": 411.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.043781903805211186, "epoch": 0.0117, "frac_reward_zero_std": 0.0, "grad_norm": 0.27131181955337524, "kl": 4.533619926311076, "learning_rate": 4.710235490423655e-06, "loss": -0.0464, "num_tokens": 28727855.0, "reward": 1.53125, "reward_std": 1.137183427810669, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 1.1067060232162476, "sampling/importance_sampling_ratio/max": 1.065118670463562, "sampling/importance_sampling_ratio/mean": 1.0059785842895508, "sampling/importance_sampling_ratio/min": 0.5409020781517029, "sampling/sampling_logp_difference/max": 0.6145572662353516, "sampling/sampling_logp_difference/mean": 0.010942462831735611, "step": 585, "step_time": 14.068769189019804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 199.1875, "completions/mean_terminated_length": 199.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.052072552032768726, "epoch": 0.01172, "frac_reward_zero_std": 0.0, "grad_norm": 0.20399577915668488, "kl": 1.7691019615158439, "learning_rate": 4.697827113889034e-06, "loss": -0.0585, "num_tokens": 28772615.0, "reward": 1.125, "reward_std": 1.1078977584838867, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 1.0701221227645874, "sampling/importance_sampling_ratio/max": 1.0661931037902832, "sampling/importance_sampling_ratio/mean": 0.961740255355835, "sampling/importance_sampling_ratio/min": 0.38304615020751953, "sampling/sampling_logp_difference/max": 0.959592342376709, "sampling/sampling_logp_difference/mean": 0.03294771909713745, "step": 586, "step_time": 12.989276886961306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 305.78125, "completions/mean_terminated_length": 305.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.048484924249351025, "epoch": 0.01174, "frac_reward_zero_std": 0.0, "grad_norm": 0.27488669753074646, "kl": 0.7221025731414557, "learning_rate": 4.685439211564055e-06, "loss": -0.039, "num_tokens": 28822229.0, "reward": 1.46875, "reward_std": 0.7521332502365112, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.761339008808136, "sampling/importance_sampling_ratio/max": 1.6720702648162842, "sampling/importance_sampling_ratio/mean": 1.026247262954712, "sampling/importance_sampling_ratio/min": 0.46828266978263855, "sampling/sampling_logp_difference/max": 0.8378826379776001, "sampling/sampling_logp_difference/mean": 0.017621640115976334, "step": 587, "step_time": 15.715644123032689 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 300.4375, "completions/mean_terminated_length": 300.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04139379668049514, "epoch": 0.01176, "frac_reward_zero_std": 0.0, "grad_norm": 0.08411671221256256, "kl": 1.484500713646412, "learning_rate": 4.67307194685352e-06, "loss": -0.083, "num_tokens": 28871347.0, "reward": 1.65625, "reward_std": 0.8357104659080505, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.8273305296897888, "sampling/importance_sampling_ratio/max": 1.100260853767395, "sampling/importance_sampling_ratio/mean": 0.9781449437141418, "sampling/importance_sampling_ratio/min": 0.22036497294902802, "sampling/sampling_logp_difference/max": 1.5283644199371338, "sampling/sampling_logp_difference/mean": 0.025237035006284714, "step": 588, "step_time": 13.720544851996237 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 311.6875, "completions/mean_terminated_length": 311.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05015509470831603, "epoch": 0.01178, "frac_reward_zero_std": 0.0, "grad_norm": 0.11729273945093155, "kl": 0.9805005080997944, "learning_rate": 4.660725482890016e-06, "loss": -0.0476, "num_tokens": 28921394.0, "reward": 1.46875, "reward_std": 0.9154131412506104, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.9832262396812439, "sampling/importance_sampling_ratio/max": 1.1710877418518066, "sampling/importance_sampling_ratio/mean": 0.9978388547897339, "sampling/importance_sampling_ratio/min": 0.30753207206726074, "sampling/sampling_logp_difference/max": 1.1882398128509521, "sampling/sampling_logp_difference/mean": 0.019175559282302856, "step": 589, "step_time": 13.700935258006211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 266.46875, "completions/mean_terminated_length": 266.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04873721476178616, "epoch": 0.0118, "frac_reward_zero_std": 0.0, "grad_norm": 0.1662474274635315, "kl": 3.2656011283397675, "learning_rate": 4.648399982531745e-06, "loss": -0.0531, "num_tokens": 28970524.0, "reward": 1.21875, "reward_std": 1.1082136631011963, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.156590461730957, "sampling/importance_sampling_ratio/max": 1.0812519788742065, "sampling/importance_sampling_ratio/mean": 1.0096858739852905, "sampling/importance_sampling_ratio/min": 0.6906824111938477, "sampling/sampling_logp_difference/max": 0.37010622024536133, "sampling/sampling_logp_difference/mean": 0.0094972625374794, "step": 590, "step_time": 14.515455563014257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 299.46875, "completions/mean_terminated_length": 299.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.03751463221851736, "epoch": 0.01182, "frac_reward_zero_std": 0.0, "grad_norm": 0.01862281933426857, "kl": 0.8305587284266949, "learning_rate": 4.636095608360393e-06, "loss": -0.0693, "num_tokens": 29019234.0, "reward": 1.75, "reward_std": 0.6387276649475098, "rewards/rollout_reward_func/mean": 1.75, "rewards/rollout_reward_func/std": 0.6221709847450256, "sampling/importance_sampling_ratio/max": 1.050812840461731, "sampling/importance_sampling_ratio/mean": 1.0175074338912964, "sampling/importance_sampling_ratio/min": 1.000909686088562, "sampling/sampling_logp_difference/max": 0.048267580568790436, "sampling/sampling_logp_difference/mean": 0.004947408102452755, "step": 591, "step_time": 13.477810604992555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 282.46875, "completions/mean_terminated_length": 282.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.03993111860472709, "epoch": 0.01184, "frac_reward_zero_std": 0.0, "grad_norm": 1.204612374305725, "kl": 5.558808624744415, "learning_rate": 4.623812522678968e-06, "loss": -0.055, "num_tokens": 29068777.0, "reward": 1.21875, "reward_std": 1.146766185760498, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.1841527223587036, "sampling/importance_sampling_ratio/max": 1.4204517602920532, "sampling/importance_sampling_ratio/mean": 0.9703889489173889, "sampling/importance_sampling_ratio/min": 0.18078593909740448, "sampling/sampling_logp_difference/max": 1.7204170227050781, "sampling/sampling_logp_difference/mean": 0.03501085191965103, "step": 592, "step_time": 14.621007682013442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 236.25, "completions/mean_terminated_length": 236.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05111110187135637, "epoch": 0.01186, "frac_reward_zero_std": 0.0, "grad_norm": 0.196183443069458, "kl": 1.0043455753475428, "learning_rate": 4.611550887509677e-06, "loss": -0.0284, "num_tokens": 29115838.0, "reward": 1.25, "reward_std": 0.8485779762268066, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9158109426498413, "sampling/importance_sampling_ratio/max": 1.1818737983703613, "sampling/importance_sampling_ratio/mean": 1.003373146057129, "sampling/importance_sampling_ratio/min": 0.2951788902282715, "sampling/sampling_logp_difference/max": 1.2236125469207764, "sampling/sampling_logp_difference/mean": 0.019373694434762, "step": 593, "step_time": 13.294442479033023 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 342.84375, "completions/mean_terminated_length": 353.3870849609375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13133769063279033, "epoch": 0.01188, "frac_reward_zero_std": 0.0, "grad_norm": 0.6086904406547546, "kl": 2.349928641691804, "learning_rate": 4.599310864591776e-06, "loss": -0.0117, "num_tokens": 29168119.0, "reward": 1.625, "reward_std": 0.6752861738204956, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.7513428926467896, "sampling/importance_sampling_ratio/max": 1.2810888290405273, "sampling/importance_sampling_ratio/mean": 0.9927553534507751, "sampling/importance_sampling_ratio/min": 0.32930228114128113, "sampling/sampling_logp_difference/max": 1.1164402961730957, "sampling/sampling_logp_difference/mean": 0.029031667858362198, "step": 594, "step_time": 13.640244104957674 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 236.375, "completions/mean_terminated_length": 236.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04247415438294411, "epoch": 0.0119, "frac_reward_zero_std": 0.0, "grad_norm": 0.31959158182144165, "kl": 4.436761876568198, "learning_rate": 4.587092615379442e-06, "loss": -0.0671, "num_tokens": 29215129.0, "reward": 1.46875, "reward_std": 0.8113973140716553, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.949851393699646, "sampling/importance_sampling_ratio/max": 1.117862582206726, "sampling/importance_sampling_ratio/mean": 0.9879908561706543, "sampling/importance_sampling_ratio/min": 0.34075239300727844, "sampling/sampling_logp_difference/max": 1.076584815979004, "sampling/sampling_logp_difference/mean": 0.020965583622455597, "step": 595, "step_time": 14.41734926700883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 338.96875, "completions/mean_terminated_length": 338.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04186729632783681, "epoch": 0.01192, "frac_reward_zero_std": 0.0, "grad_norm": 0.3435642421245575, "kl": 2.353573275730014, "learning_rate": 4.574896301039641e-06, "loss": -0.0456, "num_tokens": 29266489.0, "reward": 1.40625, "reward_std": 0.8590067625045776, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9108441472053528, "sampling/importance_sampling_ratio/max": 1.1767081022262573, "sampling/importance_sampling_ratio/mean": 0.9744726419448853, "sampling/importance_sampling_ratio/min": 0.4216257631778717, "sampling/sampling_logp_difference/max": 0.8690323829650879, "sampling/sampling_logp_difference/mean": 0.025268161669373512, "step": 596, "step_time": 13.851453992989263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 402.875, "completions/mean_terminated_length": 402.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.041633856715634465, "epoch": 0.01194, "frac_reward_zero_std": 0.0, "grad_norm": 0.03863224387168884, "kl": 1.6158548705279827, "learning_rate": 4.562722082450004e-06, "loss": -0.0687, "num_tokens": 29319738.0, "reward": 1.6875, "reward_std": 0.9981072545051575, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.9979817867279053, "sampling/importance_sampling_ratio/max": 1.1092495918273926, "sampling/importance_sampling_ratio/mean": 1.0032544136047363, "sampling/importance_sampling_ratio/min": 0.48021891713142395, "sampling/sampling_logp_difference/max": 0.7335343360900879, "sampling/sampling_logp_difference/mean": 0.010884411633014679, "step": 597, "step_time": 14.56645477800339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 397.65625, "completions/mean_terminated_length": 397.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.061562062008306384, "epoch": 0.01196, "frac_reward_zero_std": 0.0, "grad_norm": 0.4641254246234894, "kl": 0.8413305953145027, "learning_rate": 4.550570120196702e-06, "loss": -0.0736, "num_tokens": 29372563.0, "reward": 1.78125, "reward_std": 0.8606635928153992, "rewards/rollout_reward_func/mean": 1.78125, "rewards/rollout_reward_func/std": 0.8321900367736816, "sampling/importance_sampling_ratio/max": 1.1437660455703735, "sampling/importance_sampling_ratio/mean": 1.0169272422790527, "sampling/importance_sampling_ratio/min": 0.48241227865219116, "sampling/sampling_logp_difference/max": 0.7418467998504639, "sampling/sampling_logp_difference/mean": 0.014053716324269772, "step": 598, "step_time": 16.262182901002234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 204.5, "completions/mean_terminated_length": 204.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05986364162527025, "epoch": 0.01198, "frac_reward_zero_std": 0.0, "grad_norm": 0.7019822001457214, "kl": 3.622422218322754, "learning_rate": 4.538440574572337e-06, "loss": -0.0688, "num_tokens": 29417187.0, "reward": 1.3125, "reward_std": 0.9731956720352173, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.9979817867279053, "sampling/importance_sampling_ratio/max": 1.1463422775268555, "sampling/importance_sampling_ratio/mean": 0.9436059594154358, "sampling/importance_sampling_ratio/min": 0.23276828229427338, "sampling/sampling_logp_difference/max": 1.4656848907470703, "sampling/sampling_logp_difference/mean": 0.047804489731788635, "step": 599, "step_time": 12.27738275694719 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 280.9375, "completions/mean_terminated_length": 280.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05462238215841353, "epoch": 0.012, "frac_reward_zero_std": 0.0, "grad_norm": 0.657209575176239, "kl": 2.350400175899267, "learning_rate": 4.5263336055738115e-06, "loss": -0.0449, "num_tokens": 29465109.0, "reward": 1.3125, "reward_std": 0.792182445526123, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.8206016421318054, "sampling/importance_sampling_ratio/max": 1.1611082553863525, "sampling/importance_sampling_ratio/mean": 0.9762046337127686, "sampling/importance_sampling_ratio/min": 0.18784254789352417, "sampling/sampling_logp_difference/max": 1.6794214248657227, "sampling/sampling_logp_difference/mean": 0.033224862068891525, "step": 600, "step_time": 13.807853289996274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1598.0, "completions/max_terminated_length": 1598.0, "completions/mean_length": 421.40625, "completions/mean_terminated_length": 421.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05296017858199775, "epoch": 0.01202, "frac_reward_zero_std": 0.0, "grad_norm": 0.3147381544113159, "kl": 0.6476240130141377, "learning_rate": 4.5142493729002316e-06, "loss": -0.0413, "num_tokens": 29519847.0, "reward": 1.75, "reward_std": 0.7446014881134033, "rewards/rollout_reward_func/mean": 1.75, "rewards/rollout_reward_func/std": 0.718421220779419, "sampling/importance_sampling_ratio/max": 1.1123205423355103, "sampling/importance_sampling_ratio/mean": 1.0168492794036865, "sampling/importance_sampling_ratio/min": 0.8335443735122681, "sampling/sampling_logp_difference/max": 0.19794097542762756, "sampling/sampling_logp_difference/mean": 0.007103838492184877, "step": 601, "step_time": 16.892152707994683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2223.0, "completions/max_terminated_length": 2223.0, "completions/mean_length": 306.21875, "completions/mean_terminated_length": 306.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05981213483028114, "epoch": 0.01204, "frac_reward_zero_std": 0.0, "grad_norm": 0.6107025146484375, "kl": 3.673478050157428, "learning_rate": 4.502188035950794e-06, "loss": -0.0612, "num_tokens": 29569874.0, "reward": 1.3125, "reward_std": 1.3206710815429688, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.3304740190505981, "sampling/importance_sampling_ratio/max": 1.0321422815322876, "sampling/importance_sampling_ratio/mean": 0.952860951423645, "sampling/importance_sampling_ratio/min": 0.24919408559799194, "sampling/sampling_logp_difference/max": 0.9015445709228516, "sampling/sampling_logp_difference/mean": 0.029621783643960953, "step": 602, "step_time": 18.92251039702387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 243.8125, "completions/mean_terminated_length": 243.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05275692488066852, "epoch": 0.01206, "frac_reward_zero_std": 0.0, "grad_norm": 0.24195881187915802, "kl": 1.1042731646448374, "learning_rate": 4.490149753822689e-06, "loss": -0.0676, "num_tokens": 29618147.0, "reward": 1.5, "reward_std": 0.9833742380142212, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9837387204170227, "sampling/importance_sampling_ratio/max": 1.0886999368667603, "sampling/importance_sampling_ratio/mean": 1.0093417167663574, "sampling/importance_sampling_ratio/min": 0.6115578413009644, "sampling/sampling_logp_difference/max": 0.49178600311279297, "sampling/sampling_logp_difference/mean": 0.010445044375956059, "step": 603, "step_time": 15.183142140987911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 312.5625, "completions/mean_terminated_length": 312.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06101666809991002, "epoch": 0.01208, "frac_reward_zero_std": 0.0, "grad_norm": 0.18174244463443756, "kl": 0.9447946846485138, "learning_rate": 4.478134685308997e-06, "loss": -0.0616, "num_tokens": 29667504.0, "reward": 1.65625, "reward_std": 0.796323299407959, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.8273305296897888, "sampling/importance_sampling_ratio/max": 1.085925817489624, "sampling/importance_sampling_ratio/mean": 1.004433512687683, "sampling/importance_sampling_ratio/min": 0.3010734021663666, "sampling/sampling_logp_difference/max": 1.2146379947662354, "sampling/sampling_logp_difference/mean": 0.018274549394845963, "step": 604, "step_time": 14.548725598026067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1670.0, "completions/max_terminated_length": 1670.0, "completions/mean_length": 343.71875, "completions/mean_terminated_length": 343.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05679389671422541, "epoch": 0.0121, "frac_reward_zero_std": 0.0, "grad_norm": 0.0354529470205307, "kl": 0.7830803133547306, "learning_rate": 4.4661429888965894e-06, "loss": -0.0689, "num_tokens": 29717652.0, "reward": 1.71875, "reward_std": 0.7433937788009644, "rewards/rollout_reward_func/mean": 1.71875, "rewards/rollout_reward_func/std": 0.7718588709831238, "sampling/importance_sampling_ratio/max": 1.0806487798690796, "sampling/importance_sampling_ratio/mean": 1.0170083045959473, "sampling/importance_sampling_ratio/min": 0.8133800625801086, "sampling/sampling_logp_difference/max": 0.22765511274337769, "sampling/sampling_logp_difference/mean": 0.008252250961959362, "step": 605, "step_time": 16.002384285005974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 224.3125, "completions/mean_terminated_length": 224.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07461326359771192, "epoch": 0.01212, "frac_reward_zero_std": 0.0, "grad_norm": 0.1464323252439499, "kl": 0.7929232940077782, "learning_rate": 4.454174822764049e-06, "loss": -0.0647, "num_tokens": 29762802.0, "reward": 1.46875, "reward_std": 0.7889829874038696, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.8025915622711182, "sampling/importance_sampling_ratio/max": 1.6428730487823486, "sampling/importance_sampling_ratio/mean": 1.0570554733276367, "sampling/importance_sampling_ratio/min": 1.0105293989181519, "sampling/sampling_logp_difference/max": 0.48249614238739014, "sampling/sampling_logp_difference/mean": 0.016019612550735474, "step": 606, "step_time": 12.648994115996175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1571.0, "completions/max_terminated_length": 1571.0, "completions/mean_length": 381.75, "completions/mean_terminated_length": 381.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06507404311560094, "epoch": 0.01214, "frac_reward_zero_std": 0.0, "grad_norm": 0.2898041903972626, "kl": 1.343998622149229, "learning_rate": 4.4422303447795816e-06, "loss": -0.045, "num_tokens": 29816259.0, "reward": 1.5, "reward_std": 0.9837794899940491, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.0160009860992432, "sampling/importance_sampling_ratio/max": 1.1287457942962646, "sampling/importance_sampling_ratio/mean": 1.0019762516021729, "sampling/importance_sampling_ratio/min": 0.5454490780830383, "sampling/sampling_logp_difference/max": 0.6155457496643066, "sampling/sampling_logp_difference/mean": 0.01754620485007763, "step": 607, "step_time": 15.994328608983778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 313.375, "completions/mean_terminated_length": 313.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06740397796966136, "epoch": 0.01216, "frac_reward_zero_std": 0.0, "grad_norm": 0.6018857955932617, "kl": 0.7697516065090895, "learning_rate": 4.430309712498924e-06, "loss": -0.0786, "num_tokens": 29866273.0, "reward": 1.71875, "reward_std": 0.7837724685668945, "rewards/rollout_reward_func/mean": 1.71875, "rewards/rollout_reward_func/std": 0.7718588709831238, "sampling/importance_sampling_ratio/max": 1.2734254598617554, "sampling/importance_sampling_ratio/mean": 1.0292562246322632, "sampling/importance_sampling_ratio/min": 0.861254096031189, "sampling/sampling_logp_difference/max": 0.2399211823940277, "sampling/sampling_logp_difference/mean": 0.010509374551475048, "step": 608, "step_time": 13.671454654046101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 226.15625, "completions/mean_terminated_length": 226.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07394240470603108, "epoch": 0.01218, "frac_reward_zero_std": 0.0, "grad_norm": 0.2469942420721054, "kl": 0.716457998380065, "learning_rate": 4.418413083163273e-06, "loss": -0.0519, "num_tokens": 29912144.0, "reward": 1.25, "reward_std": 1.0787181854248047, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.1071614027023315, "sampling/importance_sampling_ratio/max": 1.1455405950546265, "sampling/importance_sampling_ratio/mean": 1.0120530128479004, "sampling/importance_sampling_ratio/min": 0.47537025809288025, "sampling/sampling_logp_difference/max": 0.8062396049499512, "sampling/sampling_logp_difference/mean": 0.017007047310471535, "step": 609, "step_time": 12.780328791995998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1373.0, "completions/max_terminated_length": 1373.0, "completions/mean_length": 368.84375, "completions/mean_terminated_length": 368.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06637772265821695, "epoch": 0.0122, "frac_reward_zero_std": 0.0, "grad_norm": 0.6079367399215698, "kl": 1.074800199829042, "learning_rate": 4.406540613697213e-06, "loss": -0.0548, "num_tokens": 29964807.0, "reward": 1.625, "reward_std": 0.9423224925994873, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 1.0395407676696777, "sampling/importance_sampling_ratio/max": 1.2746027708053589, "sampling/importance_sampling_ratio/mean": 0.9870878458023071, "sampling/importance_sampling_ratio/min": 0.4014275074005127, "sampling/sampling_logp_difference/max": 0.9234905242919922, "sampling/sampling_logp_difference/mean": 0.02638249844312668, "step": 610, "step_time": 15.426006301000598 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 236.40625, "completions/mean_terminated_length": 236.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06042633904144168, "epoch": 0.01222, "frac_reward_zero_std": 0.0, "grad_norm": 0.0785088911652565, "kl": 1.6267925333231688, "learning_rate": 4.394692460706643e-06, "loss": -0.0484, "num_tokens": 30011373.0, "reward": 1.28125, "reward_std": 0.8527557849884033, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.8884337544441223, "sampling/importance_sampling_ratio/max": 1.1085731983184814, "sampling/importance_sampling_ratio/mean": 0.9942866563796997, "sampling/importance_sampling_ratio/min": 0.5082177519798279, "sampling/sampling_logp_difference/max": 0.737919807434082, "sampling/sampling_logp_difference/mean": 0.021475225687026978, "step": 611, "step_time": 15.67492402701464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 249.15625, "completions/mean_terminated_length": 249.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06584231229498982, "epoch": 0.01224, "frac_reward_zero_std": 0.0, "grad_norm": 0.2531028985977173, "kl": 6.179274246096611, "learning_rate": 4.382868780476709e-06, "loss": -0.0562, "num_tokens": 30059899.0, "reward": 1.1875, "reward_std": 1.2683104276657104, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.2556324005126953, "sampling/importance_sampling_ratio/max": 1.2192546129226685, "sampling/importance_sampling_ratio/mean": 0.9727973937988281, "sampling/importance_sampling_ratio/min": 0.4733879566192627, "sampling/sampling_logp_difference/max": 0.7478623390197754, "sampling/sampling_logp_difference/mean": 0.030588222667574883, "step": 612, "step_time": 13.97100176200911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 256.34375, "completions/mean_terminated_length": 256.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07122868415899575, "epoch": 0.01226, "frac_reward_zero_std": 0.0, "grad_norm": 0.26228514313697815, "kl": 0.9139906726777554, "learning_rate": 4.371069728969745e-06, "loss": -0.0464, "num_tokens": 30107565.0, "reward": 1.34375, "reward_std": 0.8717334270477295, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.0035220384597778, "sampling/importance_sampling_ratio/max": 1.122766375541687, "sampling/importance_sampling_ratio/mean": 1.0025368928909302, "sampling/importance_sampling_ratio/min": 0.4825420081615448, "sampling/sampling_logp_difference/max": 0.7395186424255371, "sampling/sampling_logp_difference/mean": 0.01828351430594921, "step": 613, "step_time": 13.901227250986267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 203.96875, "completions/mean_terminated_length": 203.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0671512521803379, "epoch": 0.01228, "frac_reward_zero_std": 0.0, "grad_norm": 0.2218661606311798, "kl": 1.1757650263607502, "learning_rate": 4.359295461823221e-06, "loss": -0.0522, "num_tokens": 30152167.0, "reward": 1.3125, "reward_std": 0.7928940057754517, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.8206016421318054, "sampling/importance_sampling_ratio/max": 1.207103967666626, "sampling/importance_sampling_ratio/mean": 1.012094259262085, "sampling/importance_sampling_ratio/min": 0.4313367009162903, "sampling/sampling_logp_difference/max": 0.840787410736084, "sampling/sampling_logp_difference/mean": 0.01871883124113083, "step": 614, "step_time": 13.13074228700134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 306.375, "completions/mean_terminated_length": 306.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.059618596686050296, "epoch": 0.0123, "frac_reward_zero_std": 0.0, "grad_norm": 0.04738689213991165, "kl": 0.8989900890737772, "learning_rate": 4.347546134347673e-06, "loss": -0.0615, "num_tokens": 30202427.0, "reward": 1.59375, "reward_std": 0.6668190956115723, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.6652370095252991, "sampling/importance_sampling_ratio/max": 1.2126702070236206, "sampling/importance_sampling_ratio/mean": 1.0050073862075806, "sampling/importance_sampling_ratio/min": 0.3099159598350525, "sampling/sampling_logp_difference/max": 1.1806223392486572, "sampling/sampling_logp_difference/mean": 0.01845197193324566, "step": 615, "step_time": 14.238142405985855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 333.90625, "completions/mean_terminated_length": 333.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06553702196106315, "epoch": 0.01232, "frac_reward_zero_std": 0.0, "grad_norm": 0.10745450109243393, "kl": 0.8113277023658156, "learning_rate": 4.33582190152468e-06, "loss": -0.0393, "num_tokens": 30253702.0, "reward": 1.46875, "reward_std": 0.8425160646438599, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.8793096542358398, "sampling/importance_sampling_ratio/max": 1.172308087348938, "sampling/importance_sampling_ratio/mean": 1.0095481872558594, "sampling/importance_sampling_ratio/min": 0.36183568835258484, "sampling/sampling_logp_difference/max": 1.0211262702941895, "sampling/sampling_logp_difference/mean": 0.017660953104496002, "step": 616, "step_time": 13.536156025962555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 284.3125, "completions/mean_terminated_length": 284.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0649743378162384, "epoch": 0.01234, "frac_reward_zero_std": 0.0, "grad_norm": 0.05924907699227333, "kl": 0.78094232827425, "learning_rate": 4.324122918004795e-06, "loss": -0.0574, "num_tokens": 30302552.0, "reward": 1.5, "reward_std": 0.9327350854873657, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9503819346427917, "sampling/importance_sampling_ratio/max": 1.214786171913147, "sampling/importance_sampling_ratio/mean": 1.0292994976043701, "sampling/importance_sampling_ratio/min": 1.008161187171936, "sampling/sampling_logp_difference/max": 0.12101498246192932, "sampling/sampling_logp_difference/mean": 0.008525034412741661, "step": 617, "step_time": 14.36820900598832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 321.4375, "completions/mean_terminated_length": 321.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06301116617396474, "epoch": 0.01236, "frac_reward_zero_std": 0.0, "grad_norm": 0.09839999675750732, "kl": 0.8916003331542015, "learning_rate": 4.3124493381055204e-06, "loss": -0.0429, "num_tokens": 30352109.0, "reward": 1.25, "reward_std": 0.8917276859283447, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.9837387204170227, "sampling/importance_sampling_ratio/max": 1.1993436813354492, "sampling/importance_sampling_ratio/mean": 0.9882416725158691, "sampling/importance_sampling_ratio/min": 0.37659454345703125, "sampling/sampling_logp_difference/max": 0.9926583766937256, "sampling/sampling_logp_difference/mean": 0.021649044007062912, "step": 618, "step_time": 14.637162668004748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 276.28125, "completions/mean_terminated_length": 276.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05005201429594308, "epoch": 0.01238, "frac_reward_zero_std": 0.0, "grad_norm": 0.09037484973669052, "kl": 1.1087535675615072, "learning_rate": 4.300801315809264e-06, "loss": -0.0306, "num_tokens": 30400666.0, "reward": 1.28125, "reward_std": 1.0060372352600098, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.0234153270721436, "sampling/importance_sampling_ratio/max": 1.0770082473754883, "sampling/importance_sampling_ratio/mean": 1.0075342655181885, "sampling/importance_sampling_ratio/min": 0.4801391363143921, "sampling/sampling_logp_difference/max": 0.7389178276062012, "sampling/sampling_logp_difference/mean": 0.013986794278025627, "step": 619, "step_time": 13.297589659006917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 216.3125, "completions/mean_terminated_length": 216.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.062155270017683506, "epoch": 0.0124, "frac_reward_zero_std": 0.0, "grad_norm": 0.06470240652561188, "kl": 0.6886183274909854, "learning_rate": 4.289179004761318e-06, "loss": -0.0638, "num_tokens": 30446299.0, "reward": 1.375, "reward_std": 0.9577416777610779, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.008032202720642, "sampling/importance_sampling_ratio/max": 1.0812385082244873, "sampling/importance_sampling_ratio/mean": 1.0052220821380615, "sampling/importance_sampling_ratio/min": 0.5189112424850464, "sampling/sampling_logp_difference/max": 0.6560463905334473, "sampling/sampling_logp_difference/mean": 0.013964138925075531, "step": 620, "step_time": 13.526654047018383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 274.78125, "completions/mean_terminated_length": 274.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.061004120390862226, "epoch": 0.01242, "frac_reward_zero_std": 0.0, "grad_norm": 0.18998093903064728, "kl": 0.9463987946510315, "learning_rate": 4.277582558267819e-06, "loss": -0.0309, "num_tokens": 30493892.0, "reward": 1.1875, "reward_std": 0.6978365182876587, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 0.692703902721405, "sampling/importance_sampling_ratio/max": 1.0656702518463135, "sampling/importance_sampling_ratio/mean": 0.9786019325256348, "sampling/importance_sampling_ratio/min": 0.43642115592956543, "sampling/sampling_logp_difference/max": 0.8358492851257324, "sampling/sampling_logp_difference/mean": 0.028210893273353577, "step": 621, "step_time": 14.921091140975477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 194.6875, "completions/mean_terminated_length": 194.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05634474894031882, "epoch": 0.01244, "frac_reward_zero_std": 0.0, "grad_norm": 0.04723816365003586, "kl": 1.0130610447376966, "learning_rate": 4.266012129293736e-06, "loss": -0.0484, "num_tokens": 30538449.0, "reward": 1.25, "reward_std": 0.8517879247665405, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 0.8424234986305237, "sampling/importance_sampling_ratio/max": 1.255550742149353, "sampling/importance_sampling_ratio/mean": 1.008002519607544, "sampling/importance_sampling_ratio/min": 0.23808327317237854, "sampling/sampling_logp_difference/max": 1.436497688293457, "sampling/sampling_logp_difference/mean": 0.02419598400592804, "step": 622, "step_time": 13.308306476988946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 315.0625, "completions/mean_terminated_length": 315.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05968285328708589, "epoch": 0.01246, "frac_reward_zero_std": 0.0, "grad_norm": 0.13280032575130463, "kl": 0.7353572566062212, "learning_rate": 4.254467870460848e-06, "loss": -0.0408, "num_tokens": 30588743.0, "reward": 1.40625, "reward_std": 0.9547842144966125, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9455967545509338, "sampling/importance_sampling_ratio/max": 1.1877961158752441, "sampling/importance_sampling_ratio/mean": 1.0294959545135498, "sampling/importance_sampling_ratio/min": 0.9705376029014587, "sampling/sampling_logp_difference/max": 0.17211467027664185, "sampling/sampling_logp_difference/mean": 0.008667686954140663, "step": 623, "step_time": 13.696209684974747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 243.5625, "completions/mean_terminated_length": 243.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05580797255970538, "epoch": 0.01248, "frac_reward_zero_std": 0.0, "grad_norm": 0.12707918882369995, "kl": 0.8855649307370186, "learning_rate": 4.2429499340457344e-06, "loss": -0.0739, "num_tokens": 30635843.0, "reward": 1.5625, "reward_std": 0.7924991250038147, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.84002685546875, "sampling/importance_sampling_ratio/max": 1.1115609407424927, "sampling/importance_sampling_ratio/mean": 1.0105518102645874, "sampling/importance_sampling_ratio/min": 0.5409652590751648, "sampling/sampling_logp_difference/max": 0.6144247055053711, "sampling/sampling_logp_difference/mean": 0.013994241133332253, "step": 624, "step_time": 14.371826135014999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 254.5, "completions/mean_terminated_length": 254.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05586596066132188, "epoch": 0.0125, "frac_reward_zero_std": 0.0, "grad_norm": 0.10071687400341034, "kl": 0.685342569835484, "learning_rate": 4.231458471977764e-06, "loss": -0.0541, "num_tokens": 30684207.0, "reward": 1.3125, "reward_std": 0.947630763053894, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.1482806205749512, "sampling/importance_sampling_ratio/max": 1.1646956205368042, "sampling/importance_sampling_ratio/mean": 1.0265765190124512, "sampling/importance_sampling_ratio/min": 1.0066580772399902, "sampling/sampling_logp_difference/max": 0.1480625569820404, "sampling/sampling_logp_difference/mean": 0.00767960399389267, "step": 625, "step_time": 13.791880740987835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 256.34375, "completions/mean_terminated_length": 256.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05657390272244811, "epoch": 0.01252, "frac_reward_zero_std": 0.0, "grad_norm": 1.1112983226776123, "kl": 4.371898812241852, "learning_rate": 4.219993635837086e-06, "loss": -0.0312, "num_tokens": 30732448.0, "reward": 1.21875, "reward_std": 1.0243430137634277, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.0075321197509766, "sampling/importance_sampling_ratio/max": 1.4410191774368286, "sampling/importance_sampling_ratio/mean": 1.0173766613006592, "sampling/importance_sampling_ratio/min": 0.33287477493286133, "sampling/sampling_logp_difference/max": 1.1018657684326172, "sampling/sampling_logp_difference/mean": 0.02079758606851101, "step": 626, "step_time": 13.65271206999023 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1742.0, "completions/max_terminated_length": 1742.0, "completions/mean_length": 261.4375, "completions/mean_terminated_length": 261.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05590021377429366, "epoch": 0.01254, "frac_reward_zero_std": 0.0, "grad_norm": 0.36767372488975525, "kl": 3.7280487436801195, "learning_rate": 4.208555576852641e-06, "loss": -0.0502, "num_tokens": 30779593.0, "reward": 1.21875, "reward_std": 0.8568439483642578, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 0.8700898885726929, "sampling/importance_sampling_ratio/max": 1.1137863397598267, "sampling/importance_sampling_ratio/mean": 0.9551399946212769, "sampling/importance_sampling_ratio/min": 0.4220494329929352, "sampling/sampling_logp_difference/max": 0.9002265930175781, "sampling/sampling_logp_difference/mean": 0.03749796003103256, "step": 627, "step_time": 16.34987409497262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1688.0, "completions/max_terminated_length": 1688.0, "completions/mean_length": 340.78125, "completions/mean_terminated_length": 340.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07550804642960429, "epoch": 0.01256, "frac_reward_zero_std": 0.0, "grad_norm": 0.339073121547699, "kl": 0.6374796889722347, "learning_rate": 4.19714444590016e-06, "loss": -0.0638, "num_tokens": 30831334.0, "reward": 1.5625, "reward_std": 0.8076189756393433, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.8775883316993713, "sampling/importance_sampling_ratio/max": 1.9039956331253052, "sampling/importance_sampling_ratio/mean": 1.0705852508544922, "sampling/importance_sampling_ratio/min": 1.0033212900161743, "sampling/sampling_logp_difference/max": 0.6268173456192017, "sampling/sampling_logp_difference/mean": 0.015842396765947342, "step": 628, "step_time": 16.11527213700174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1346.0, "completions/max_terminated_length": 1346.0, "completions/mean_length": 371.625, "completions/mean_terminated_length": 371.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05738022364675999, "epoch": 0.01258, "frac_reward_zero_std": 0.0, "grad_norm": 0.2895128130912781, "kl": 0.8024015408009291, "learning_rate": 4.185760393500174e-06, "loss": -0.0348, "num_tokens": 30884993.0, "reward": 1.8125, "reward_std": 0.7759708166122437, "rewards/rollout_reward_func/mean": 1.8125, "rewards/rollout_reward_func/std": 0.8206016421318054, "sampling/importance_sampling_ratio/max": 1.0981189012527466, "sampling/importance_sampling_ratio/mean": 1.0094176530838013, "sampling/importance_sampling_ratio/min": 0.5250579714775085, "sampling/sampling_logp_difference/max": 0.6667137145996094, "sampling/sampling_logp_difference/mean": 0.011827601119875908, "step": 629, "step_time": 15.621696344009251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 321.125, "completions/mean_terminated_length": 321.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06726813642308116, "epoch": 0.0126, "frac_reward_zero_std": 0.0, "grad_norm": 0.18706251680850983, "kl": 0.6700979210436344, "learning_rate": 4.17440356981603e-06, "loss": -0.0471, "num_tokens": 30935345.0, "reward": 1.375, "reward_std": 0.9538168907165527, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.0395407676696777, "sampling/importance_sampling_ratio/max": 1.3032041788101196, "sampling/importance_sampling_ratio/mean": 1.0389440059661865, "sampling/importance_sampling_ratio/min": 1.0030899047851562, "sampling/sampling_logp_difference/max": 0.16545546054840088, "sampling/sampling_logp_difference/mean": 0.010462379083037376, "step": 630, "step_time": 13.740573331990163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 287.65625, "completions/mean_terminated_length": 287.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06749781710095704, "epoch": 0.01262, "frac_reward_zero_std": 0.0, "grad_norm": 0.34188053011894226, "kl": 3.5385715290904045, "learning_rate": 4.163074124651908e-06, "loss": -0.0476, "num_tokens": 30985996.0, "reward": 1.375, "reward_std": 0.9530066847801208, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.9755064845085144, "sampling/importance_sampling_ratio/max": 1.2471421957015991, "sampling/importance_sampling_ratio/mean": 0.9916379451751709, "sampling/importance_sampling_ratio/min": 0.42187443375587463, "sampling/sampling_logp_difference/max": 0.8629055023193359, "sampling/sampling_logp_difference/mean": 0.02022758685052395, "step": 631, "step_time": 15.090450041971053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1742.0, "completions/max_terminated_length": 1742.0, "completions/mean_length": 316.0625, "completions/mean_terminated_length": 316.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05800894950516522, "epoch": 0.01264, "frac_reward_zero_std": 0.0, "grad_norm": 0.26068928837776184, "kl": 2.580898987594992, "learning_rate": 4.15177220745085e-06, "loss": -0.0467, "num_tokens": 31037253.0, "reward": 1.4375, "reward_std": 0.9910088777542114, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.173812985420227, "sampling/importance_sampling_ratio/mean": 1.0050257444381714, "sampling/importance_sampling_ratio/min": 0.6583209037780762, "sampling/sampling_logp_difference/max": 0.42772841453552246, "sampling/sampling_logp_difference/mean": 0.014230318367481232, "step": 632, "step_time": 16.80669640500855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1773.0, "completions/max_terminated_length": 1773.0, "completions/mean_length": 284.3125, "completions/mean_terminated_length": 284.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06686968333087862, "epoch": 0.01266, "frac_reward_zero_std": 0.0, "grad_norm": 0.22899611294269562, "kl": 2.8542810305953026, "learning_rate": 4.140497967292782e-06, "loss": -0.0579, "num_tokens": 31085900.0, "reward": 1.125, "reward_std": 1.2535009384155273, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 1.2115039825439453, "sampling/importance_sampling_ratio/max": 1.2217828035354614, "sampling/importance_sampling_ratio/mean": 1.0093481540679932, "sampling/importance_sampling_ratio/min": 0.6128396987915039, "sampling/sampling_logp_difference/max": 0.4896974563598633, "sampling/sampling_logp_difference/mean": 0.017098696902394295, "step": 633, "step_time": 16.049124577009934 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 286.84375, "completions/mean_terminated_length": 286.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06106912740506232, "epoch": 0.01268, "frac_reward_zero_std": 0.0, "grad_norm": 0.20131954550743103, "kl": 0.6858904464170337, "learning_rate": 4.1292515528925584e-06, "loss": -0.0755, "num_tokens": 31133700.0, "reward": 1.4375, "reward_std": 0.9092998504638672, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9136068224906921, "sampling/importance_sampling_ratio/max": 1.0974661111831665, "sampling/importance_sampling_ratio/mean": 1.0221502780914307, "sampling/importance_sampling_ratio/min": 0.9505645036697388, "sampling/sampling_logp_difference/max": 0.07307519763708115, "sampling/sampling_logp_difference/mean": 0.007414157968014479, "step": 634, "step_time": 15.1172788739932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1499.0, "completions/max_terminated_length": 1499.0, "completions/mean_length": 270.0, "completions/mean_terminated_length": 270.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07719157915562391, "epoch": 0.0127, "frac_reward_zero_std": 0.0, "grad_norm": 0.22059200704097748, "kl": 0.7026389446109533, "learning_rate": 4.11803311259798e-06, "loss": -0.0528, "num_tokens": 31181605.0, "reward": 1.25, "reward_std": 1.156165599822998, "rewards/rollout_reward_func/mean": 1.25, "rewards/rollout_reward_func/std": 1.1359236240386963, "sampling/importance_sampling_ratio/max": 1.3761088848114014, "sampling/importance_sampling_ratio/mean": 1.0408899784088135, "sampling/importance_sampling_ratio/min": 1.0061227083206177, "sampling/sampling_logp_difference/max": 0.3082754611968994, "sampling/sampling_logp_difference/mean": 0.011779315769672394, "step": 635, "step_time": 15.853789337998023 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1733.0, "completions/max_terminated_length": 1733.0, "completions/mean_length": 378.0, "completions/mean_terminated_length": 378.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06673988793045282, "epoch": 0.01272, "frac_reward_zero_std": 0.0, "grad_norm": 0.25583869218826294, "kl": 3.181802239269018, "learning_rate": 4.106842794387864e-06, "loss": -0.07, "num_tokens": 31233850.0, "reward": 1.53125, "reward_std": 0.9127323627471924, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 1.0467884540557861, "sampling/importance_sampling_ratio/max": 1.5145280361175537, "sampling/importance_sampling_ratio/mean": 1.0079536437988281, "sampling/importance_sampling_ratio/min": 0.16307277977466583, "sampling/sampling_logp_difference/max": 1.892707347869873, "sampling/sampling_logp_difference/mean": 0.03623726963996887, "step": 636, "step_time": 16.157285304027027 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 239.625, "completions/mean_terminated_length": 239.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05685111600905657, "epoch": 0.01274, "frac_reward_zero_std": 0.0, "grad_norm": 0.13185158371925354, "kl": 1.413423741236329, "learning_rate": 4.095680745870069e-06, "loss": -0.0614, "num_tokens": 31281153.0, "reward": 1.3125, "reward_std": 1.0680646896362305, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.0906493663787842, "sampling/importance_sampling_ratio/max": 1.0973238945007324, "sampling/importance_sampling_ratio/mean": 1.004016399383545, "sampling/importance_sampling_ratio/min": 0.42374560236930847, "sampling/sampling_logp_difference/max": 0.8680834770202637, "sampling/sampling_logp_difference/mean": 0.014467213302850723, "step": 637, "step_time": 13.726608246011892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 401.59375, "completions/mean_terminated_length": 401.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.062098873662762344, "epoch": 0.01276, "frac_reward_zero_std": 0.0, "grad_norm": 0.34058046340942383, "kl": 0.9640245549380779, "learning_rate": 4.084547114279564e-06, "loss": -0.0631, "num_tokens": 31335135.0, "reward": 1.65625, "reward_std": 0.8685724139213562, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.9370294213294983, "sampling/importance_sampling_ratio/max": 1.0628206729888916, "sampling/importance_sampling_ratio/mean": 0.9777008891105652, "sampling/importance_sampling_ratio/min": 0.33376073837280273, "sampling/sampling_logp_difference/max": 1.114989161491394, "sampling/sampling_logp_difference/mean": 0.022957008332014084, "step": 638, "step_time": 16.231635296004242 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 334.59375, "completions/mean_terminated_length": 334.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0814055276568979, "epoch": 0.01278, "frac_reward_zero_std": 0.0, "grad_norm": 0.1499270796775818, "kl": 1.0823244992643595, "learning_rate": 4.073442046476472e-06, "loss": -0.0353, "num_tokens": 31385732.0, "reward": 1.375, "reward_std": 0.8762495517730713, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.0701221227645874, "sampling/importance_sampling_ratio/max": 1.2735874652862549, "sampling/importance_sampling_ratio/mean": 0.9877982139587402, "sampling/importance_sampling_ratio/min": 3.8208880503987075e-11, "sampling/sampling_logp_difference/max": 23.96419334411621, "sampling/sampling_logp_difference/mean": 0.22130370140075684, "step": 639, "step_time": 15.332756606003386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 379.0, "completions/mean_terminated_length": 379.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08742211386561394, "epoch": 0.0128, "frac_reward_zero_std": 0.0, "grad_norm": 0.19989822804927826, "kl": 1.0389942359179258, "learning_rate": 4.062365688944147e-06, "loss": -0.0488, "num_tokens": 31439077.0, "reward": 1.625, "reward_std": 0.9064008593559265, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.941858172416687, "sampling/importance_sampling_ratio/max": 1.2678163051605225, "sampling/importance_sampling_ratio/mean": 1.0191211700439453, "sampling/importance_sampling_ratio/min": 0.643764078617096, "sampling/sampling_logp_difference/max": 0.4523344039916992, "sampling/sampling_logp_difference/mean": 0.013101737014949322, "step": 640, "step_time": 14.49540505798359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 301.40625, "completions/mean_terminated_length": 301.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06371102039702237, "epoch": 0.01282, "frac_reward_zero_std": 0.0, "grad_norm": 0.32457366585731506, "kl": 1.4212505482137203, "learning_rate": 4.051318187787231e-06, "loss": -0.0619, "num_tokens": 31488627.0, "reward": 1.625, "reward_std": 0.8219811916351318, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 0.941858172416687, "sampling/importance_sampling_ratio/max": 1.2832345962524414, "sampling/importance_sampling_ratio/mean": 1.0257691144943237, "sampling/importance_sampling_ratio/min": 0.657560408115387, "sampling/sampling_logp_difference/max": 0.43047332763671875, "sampling/sampling_logp_difference/mean": 0.013351164758205414, "step": 641, "step_time": 15.632746758987196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 313.8125, "completions/mean_terminated_length": 313.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05987959075719118, "epoch": 0.01284, "frac_reward_zero_std": 0.0, "grad_norm": 0.22514677047729492, "kl": 1.3933919249102473, "learning_rate": 4.040299688729734e-06, "loss": -0.0481, "num_tokens": 31538709.0, "reward": 1.5, "reward_std": 0.7905448079109192, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9158109426498413, "sampling/importance_sampling_ratio/max": 1.1413475275039673, "sampling/importance_sampling_ratio/mean": 1.0004206895828247, "sampling/importance_sampling_ratio/min": 0.38113757967948914, "sampling/sampling_logp_difference/max": 0.9645898342132568, "sampling/sampling_logp_difference/mean": 0.01484316773712635, "step": 642, "step_time": 13.858584091009106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 847.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 276.8125, "completions/mean_terminated_length": 276.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04988974239677191, "epoch": 0.01286, "frac_reward_zero_std": 0.0, "grad_norm": 0.04495313763618469, "kl": 1.1895230738446116, "learning_rate": 4.029310337113106e-06, "loss": -0.0445, "num_tokens": 31588296.0, "reward": 1.53125, "reward_std": 0.7622002363204956, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.8025915622711182, "sampling/importance_sampling_ratio/max": 1.2549092769622803, "sampling/importance_sampling_ratio/mean": 1.0101754665374756, "sampling/importance_sampling_ratio/min": 0.48268812894821167, "sampling/sampling_logp_difference/max": 0.736046314239502, "sampling/sampling_logp_difference/mean": 0.01370333507657051, "step": 643, "step_time": 12.77392442697601 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 323.34375, "completions/mean_terminated_length": 323.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08747203461825848, "epoch": 0.01288, "frac_reward_zero_std": 0.0, "grad_norm": 0.4897097051143646, "kl": 1.5998111376538873, "learning_rate": 4.018350277894324e-06, "loss": -0.0586, "num_tokens": 31639276.0, "reward": 1.5, "reward_std": 1.0130786895751953, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.0160009860992432, "sampling/importance_sampling_ratio/max": 1.287207841873169, "sampling/importance_sampling_ratio/mean": 1.020280122756958, "sampling/importance_sampling_ratio/min": 0.47899797558784485, "sampling/sampling_logp_difference/max": 0.7463955879211426, "sampling/sampling_logp_difference/mean": 0.016408011317253113, "step": 644, "step_time": 12.988505405024625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 251.03125, "completions/mean_terminated_length": 251.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08107086690142751, "epoch": 0.0129, "frac_reward_zero_std": 0.0, "grad_norm": 0.23353135585784912, "kl": 0.7749113459140062, "learning_rate": 4.007419655643977e-06, "loss": -0.0665, "num_tokens": 31685889.0, "reward": 1.40625, "reward_std": 0.7858734130859375, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.797551691532135, "sampling/importance_sampling_ratio/max": 1.219665288925171, "sampling/importance_sampling_ratio/mean": 1.0157225131988525, "sampling/importance_sampling_ratio/min": 0.6639337539672852, "sampling/sampling_logp_difference/max": 0.42068010568618774, "sampling/sampling_logp_difference/mean": 0.013097407296299934, "step": 645, "step_time": 13.535362243972486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 325.59375, "completions/mean_terminated_length": 325.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.058525705710053444, "epoch": 0.01292, "frac_reward_zero_std": 0.0, "grad_norm": 0.1032337099313736, "kl": 0.9445177456364036, "learning_rate": 3.996518614544363e-06, "loss": -0.0297, "num_tokens": 31737492.0, "reward": 1.3125, "reward_std": 0.9643757939338684, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 0.9651173949241638, "sampling/importance_sampling_ratio/max": 1.1646610498428345, "sampling/importance_sampling_ratio/mean": 1.0167174339294434, "sampling/importance_sampling_ratio/min": 0.6146109104156494, "sampling/sampling_logp_difference/max": 0.4899158477783203, "sampling/sampling_logp_difference/mean": 0.011827152222394943, "step": 646, "step_time": 14.305399990989827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 245.125, "completions/mean_terminated_length": 245.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06924691004678607, "epoch": 0.01294, "frac_reward_zero_std": 0.0, "grad_norm": 0.17406775057315826, "kl": 1.4638303276151419, "learning_rate": 3.985647298387584e-06, "loss": -0.0576, "num_tokens": 31784227.0, "reward": 1.34375, "reward_std": 0.7996478080749512, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 0.8654431700706482, "sampling/importance_sampling_ratio/max": 1.3678075075149536, "sampling/importance_sampling_ratio/mean": 1.0036066770553589, "sampling/importance_sampling_ratio/min": 0.304457426071167, "sampling/sampling_logp_difference/max": 1.2893877029418945, "sampling/sampling_logp_difference/mean": 0.029771260917186737, "step": 647, "step_time": 14.012234014997375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 278.96875, "completions/mean_terminated_length": 278.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07960863062180579, "epoch": 0.01296, "frac_reward_zero_std": 0.0, "grad_norm": 0.2342163324356079, "kl": 1.192301144823432, "learning_rate": 3.9748058505736474e-06, "loss": -0.0633, "num_tokens": 31832789.0, "reward": 1.5, "reward_std": 0.781498908996582, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.8032193183898926, "sampling/importance_sampling_ratio/max": 1.1303458213806152, "sampling/importance_sampling_ratio/mean": 0.992014467716217, "sampling/importance_sampling_ratio/min": 0.4320351183414459, "sampling/sampling_logp_difference/max": 0.8493766784667969, "sampling/sampling_logp_difference/mean": 0.01732516847550869, "step": 648, "step_time": 13.295661590018426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 282.1875, "completions/mean_terminated_length": 282.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05756959808059037, "epoch": 0.01298, "frac_reward_zero_std": 0.0, "grad_norm": 0.03684381768107414, "kl": 0.6454198304563761, "learning_rate": 3.9639944141085785e-06, "loss": -0.045, "num_tokens": 31881903.0, "reward": 1.53125, "reward_std": 0.7746413946151733, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.8025915622711182, "sampling/importance_sampling_ratio/max": 1.1078933477401733, "sampling/importance_sampling_ratio/mean": 1.0239953994750977, "sampling/importance_sampling_ratio/min": 1.0077711343765259, "sampling/sampling_logp_difference/max": 0.09564356505870819, "sampling/sampling_logp_difference/mean": 0.006845094729214907, "step": 649, "step_time": 14.951693820985383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 339.65625, "completions/mean_terminated_length": 339.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08066861471161246, "epoch": 0.013, "frac_reward_zero_std": 0.0, "grad_norm": 0.2195240557193756, "kl": 0.7065083868801594, "learning_rate": 3.953213131602535e-06, "loss": -0.0463, "num_tokens": 31933497.0, "reward": 1.3125, "reward_std": 1.2033610343933105, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.2296733856201172, "sampling/importance_sampling_ratio/max": 1.0961765050888062, "sampling/importance_sampling_ratio/mean": 1.0222302675247192, "sampling/importance_sampling_ratio/min": 0.946949303150177, "sampling/sampling_logp_difference/max": 0.055922120809555054, "sampling/sampling_logp_difference/mean": 0.007873794995248318, "step": 650, "step_time": 13.685158230990055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 288.21875, "completions/mean_terminated_length": 288.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07055927347391844, "epoch": 0.01302, "frac_reward_zero_std": 0.0, "grad_norm": 0.24750718474388123, "kl": 0.7626603972166777, "learning_rate": 3.942462145267919e-06, "loss": -0.0379, "num_tokens": 31983463.0, "reward": 1.4375, "reward_std": 0.8617627620697021, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9816871285438538, "sampling/importance_sampling_ratio/max": 1.1967229843139648, "sampling/importance_sampling_ratio/mean": 1.0207698345184326, "sampling/importance_sampling_ratio/min": 0.807661235332489, "sampling/sampling_logp_difference/max": 0.21445322036743164, "sampling/sampling_logp_difference/mean": 0.009401539340615273, "step": 651, "step_time": 13.61172570202325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 320.84375, "completions/mean_terminated_length": 320.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09772467403672636, "epoch": 0.01304, "frac_reward_zero_std": 0.0, "grad_norm": 0.46168768405914307, "kl": 0.6881179045885801, "learning_rate": 3.93174159691751e-06, "loss": -0.0504, "num_tokens": 32034126.0, "reward": 1.375, "reward_std": 1.0806139707565308, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.1288018226623535, "sampling/importance_sampling_ratio/max": 1.138808012008667, "sampling/importance_sampling_ratio/mean": 1.018115520477295, "sampling/importance_sampling_ratio/min": 0.8105384111404419, "sampling/sampling_logp_difference/max": 0.21850860118865967, "sampling/sampling_logp_difference/mean": 0.009143251925706863, "step": 652, "step_time": 14.537199621001491 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 200.28125, "completions/mean_terminated_length": 200.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08181899669580162, "epoch": 0.01306, "frac_reward_zero_std": 0.0, "grad_norm": 0.36220112442970276, "kl": 0.6370787601917982, "learning_rate": 3.921051627962586e-06, "loss": -0.0478, "num_tokens": 32079590.0, "reward": 1.1875, "reward_std": 0.9412152767181396, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 0.9651173949241638, "sampling/importance_sampling_ratio/max": 1.0707818269729614, "sampling/importance_sampling_ratio/mean": 0.9807361364364624, "sampling/importance_sampling_ratio/min": 0.41360968351364136, "sampling/sampling_logp_difference/max": 0.8952723741531372, "sampling/sampling_logp_difference/mean": 0.022137776017189026, "step": 653, "step_time": 12.324080875026993 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 319.6875, "completions/mean_terminated_length": 319.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07044155756011605, "epoch": 0.01308, "frac_reward_zero_std": 0.0, "grad_norm": 0.11927934736013412, "kl": 0.735666356049478, "learning_rate": 3.9103923794110645e-06, "loss": -0.0653, "num_tokens": 32130145.0, "reward": 1.6875, "reward_std": 0.9605993032455444, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 0.9651173949241638, "sampling/importance_sampling_ratio/max": 1.135928988456726, "sampling/importance_sampling_ratio/mean": 1.0138238668441772, "sampling/importance_sampling_ratio/min": 0.6149370670318604, "sampling/sampling_logp_difference/max": 0.4975466728210449, "sampling/sampling_logp_difference/mean": 0.011646000668406487, "step": 654, "step_time": 14.193398449991946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 293.125, "completions/mean_terminated_length": 293.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08074291911907494, "epoch": 0.0131, "frac_reward_zero_std": 0.0, "grad_norm": 0.41910067200660706, "kl": 1.0184490885585546, "learning_rate": 3.89976399186564e-06, "loss": -0.0613, "num_tokens": 32179578.0, "reward": 1.46875, "reward_std": 0.9829838275909424, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.0155048370361328, "sampling/importance_sampling_ratio/max": 1.4086413383483887, "sampling/importance_sampling_ratio/mean": 1.0142194032669067, "sampling/importance_sampling_ratio/min": 0.39296677708625793, "sampling/sampling_logp_difference/max": 0.9338631629943848, "sampling/sampling_logp_difference/mean": 0.017928505316376686, "step": 655, "step_time": 14.073734770994633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 874.0, "completions/max_terminated_length": 874.0, "completions/mean_length": 277.6875, "completions/mean_terminated_length": 277.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06823068368248641, "epoch": 0.01312, "frac_reward_zero_std": 0.0, "grad_norm": 0.7738965153694153, "kl": 2.536605393514037, "learning_rate": 3.8891666055219305e-06, "loss": -0.0352, "num_tokens": 32227693.0, "reward": 1.28125, "reward_std": 1.0162278413772583, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.1139692068099976, "sampling/importance_sampling_ratio/max": 1.4478679895401, "sampling/importance_sampling_ratio/mean": 0.9934346079826355, "sampling/importance_sampling_ratio/min": 0.4447634518146515, "sampling/sampling_logp_difference/max": 0.8427846431732178, "sampling/sampling_logp_difference/mean": 0.024943573400378227, "step": 656, "step_time": 12.490683534022537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1688.0, "completions/max_terminated_length": 1688.0, "completions/mean_length": 353.71875, "completions/mean_terminated_length": 353.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10277408198453486, "epoch": 0.01314, "frac_reward_zero_std": 0.0, "grad_norm": 0.3684566915035248, "kl": 0.6356909014284611, "learning_rate": 3.878600360166621e-06, "loss": -0.0464, "num_tokens": 32280029.0, "reward": 1.3125, "reward_std": 1.091225028038025, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.1198358535766602, "sampling/importance_sampling_ratio/max": 1.0885767936706543, "sampling/importance_sampling_ratio/mean": 0.9966416954994202, "sampling/importance_sampling_ratio/min": 0.43333008885383606, "sampling/sampling_logp_difference/max": 0.836284875869751, "sampling/sampling_logp_difference/mean": 0.016915515065193176, "step": 657, "step_time": 16.582440212994698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1598.0, "completions/max_terminated_length": 1598.0, "completions/mean_length": 334.90625, "completions/mean_terminated_length": 334.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.058942430187016726, "epoch": 0.01316, "frac_reward_zero_std": 0.0, "grad_norm": 0.07732608914375305, "kl": 3.1185914408415556, "learning_rate": 3.868065395175636e-06, "loss": -0.066, "num_tokens": 32329986.0, "reward": 1.5625, "reward_std": 0.8404130935668945, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 0.9136068224906921, "sampling/importance_sampling_ratio/max": 1.0937234163284302, "sampling/importance_sampling_ratio/mean": 0.9752697944641113, "sampling/importance_sampling_ratio/min": 0.3769713342189789, "sampling/sampling_logp_difference/max": 0.9754588603973389, "sampling/sampling_logp_difference/mean": 0.02579391375184059, "step": 658, "step_time": 16.10555756998656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 233.3125, "completions/mean_terminated_length": 233.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05440742685459554, "epoch": 0.01318, "frac_reward_zero_std": 0.0, "grad_norm": 0.19111940264701843, "kl": 2.1335212253034115, "learning_rate": 3.857561849512283e-06, "loss": -0.0474, "num_tokens": 32376200.0, "reward": 1.40625, "reward_std": 0.7573349475860596, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.756024181842804, "sampling/importance_sampling_ratio/max": 1.4305198192596436, "sampling/importance_sampling_ratio/mean": 0.9924488067626953, "sampling/importance_sampling_ratio/min": 0.37539446353912354, "sampling/sampling_logp_difference/max": 1.0897135734558105, "sampling/sampling_logp_difference/mean": 0.03077235072851181, "step": 659, "step_time": 13.756931098978384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 251.28125, "completions/mean_terminated_length": 251.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05197429843246937, "epoch": 0.0132, "frac_reward_zero_std": 0.0, "grad_norm": 0.16755297780036926, "kl": 1.981261070817709, "learning_rate": 3.847089861725429e-06, "loss": -0.0539, "num_tokens": 32422718.0, "reward": 1.40625, "reward_std": 0.6967816948890686, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.7120789289474487, "sampling/importance_sampling_ratio/max": 1.063712239265442, "sampling/importance_sampling_ratio/mean": 0.9692877531051636, "sampling/importance_sampling_ratio/min": 0.37801340222358704, "sampling/sampling_logp_difference/max": 0.9728758335113525, "sampling/sampling_logp_difference/mean": 0.029334887862205505, "step": 660, "step_time": 13.937417617955361 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1571.0, "completions/max_terminated_length": 1571.0, "completions/mean_length": 407.9375, "completions/mean_terminated_length": 407.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.054117143619805574, "epoch": 0.01322, "frac_reward_zero_std": 0.0, "grad_norm": 0.11665460467338562, "kl": 2.2963927276432514, "learning_rate": 3.836649569947673e-06, "loss": -0.0764, "num_tokens": 32476610.0, "reward": 1.6875, "reward_std": 1.2080458402633667, "rewards/rollout_reward_func/mean": 1.6875, "rewards/rollout_reward_func/std": 1.2556324005126953, "sampling/importance_sampling_ratio/max": 1.4663543701171875, "sampling/importance_sampling_ratio/mean": 1.0250494480133057, "sampling/importance_sampling_ratio/min": 0.610869288444519, "sampling/sampling_logp_difference/max": 0.493011474609375, "sampling/sampling_logp_difference/mean": 0.012344511225819588, "step": 661, "step_time": 16.215843494966975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 333.46875, "completions/mean_terminated_length": 333.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05564596992917359, "epoch": 0.01324, "frac_reward_zero_std": 0.0, "grad_norm": 0.088921919465065, "kl": 1.107970604673028, "learning_rate": 3.82624111189352e-06, "loss": -0.0587, "num_tokens": 32526695.0, "reward": 1.4375, "reward_std": 0.9003537893295288, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 0.9136068224906921, "sampling/importance_sampling_ratio/max": 1.0867384672164917, "sampling/importance_sampling_ratio/mean": 0.9835033416748047, "sampling/importance_sampling_ratio/min": 0.36904779076576233, "sampling/sampling_logp_difference/max": 0.9998235702514648, "sampling/sampling_logp_difference/mean": 0.021423030644655228, "step": 662, "step_time": 15.361639898983412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1499.0, "completions/max_terminated_length": 1499.0, "completions/mean_length": 338.375, "completions/mean_terminated_length": 338.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06068347953259945, "epoch": 0.01326, "frac_reward_zero_std": 0.0, "grad_norm": 0.18856889009475708, "kl": 1.6962498929351568, "learning_rate": 3.81586462485757e-06, "loss": -0.0777, "num_tokens": 32578220.0, "reward": 1.59375, "reward_std": 1.0116691589355469, "rewards/rollout_reward_func/mean": 1.59375, "rewards/rollout_reward_func/std": 0.9791166186332703, "sampling/importance_sampling_ratio/max": 1.1117066144943237, "sampling/importance_sampling_ratio/mean": 0.9847983121871948, "sampling/importance_sampling_ratio/min": 0.47648027539253235, "sampling/sampling_logp_difference/max": 0.7450242042541504, "sampling/sampling_logp_difference/mean": 0.020641744136810303, "step": 663, "step_time": 15.731244196998887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1715.0, "completions/max_terminated_length": 1715.0, "completions/mean_length": 385.0, "completions/mean_terminated_length": 385.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06075053662061691, "epoch": 0.01328, "frac_reward_zero_std": 0.0, "grad_norm": 0.8879587650299072, "kl": 2.1693931985646486, "learning_rate": 3.8055202457127015e-06, "loss": -0.0782, "num_tokens": 32631084.0, "reward": 1.71875, "reward_std": 1.0494742393493652, "rewards/rollout_reward_func/mean": 1.71875, "rewards/rollout_reward_func/std": 1.0234153270721436, "sampling/importance_sampling_ratio/max": 1.055189609527588, "sampling/importance_sampling_ratio/mean": 0.9877291917800903, "sampling/importance_sampling_ratio/min": 0.2916043996810913, "sampling/sampling_logp_difference/max": 1.2370667457580566, "sampling/sampling_logp_difference/mean": 0.017021633684635162, "step": 664, "step_time": 16.557334211989655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 301.28125, "completions/mean_terminated_length": 301.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05588814103975892, "epoch": 0.0133, "frac_reward_zero_std": 0.0, "grad_norm": 0.09210275113582611, "kl": 0.712736202403903, "learning_rate": 3.795208110908265e-06, "loss": -0.0818, "num_tokens": 32679896.0, "reward": 1.78125, "reward_std": 0.7281336784362793, "rewards/rollout_reward_func/mean": 1.78125, "rewards/rollout_reward_func/std": 0.7506717443466187, "sampling/importance_sampling_ratio/max": 1.1676207780838013, "sampling/importance_sampling_ratio/mean": 1.026950716972351, "sampling/importance_sampling_ratio/min": 1.0020771026611328, "sampling/sampling_logp_difference/max": 0.13334229588508606, "sampling/sampling_logp_difference/mean": 0.007462005130946636, "step": 665, "step_time": 15.764468176974333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 251.125, "completions/mean_terminated_length": 251.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07341422187164426, "epoch": 0.01332, "frac_reward_zero_std": 0.0, "grad_norm": 0.6037290096282959, "kl": 5.508242690935731, "learning_rate": 3.784928356468293e-06, "loss": -0.0605, "num_tokens": 32728101.0, "reward": 1.1875, "reward_std": 1.2698851823806763, "rewards/rollout_reward_func/mean": 1.1875, "rewards/rollout_reward_func/std": 1.2556324005126953, "sampling/importance_sampling_ratio/max": 1.0981088876724243, "sampling/importance_sampling_ratio/mean": 0.9660240411758423, "sampling/importance_sampling_ratio/min": 0.30520376563072205, "sampling/sampling_logp_difference/max": 1.1866397857666016, "sampling/sampling_logp_difference/mean": 0.03351057320833206, "step": 666, "step_time": 13.666324379038997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 218.6875, "completions/mean_terminated_length": 218.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06030866131186485, "epoch": 0.01334, "frac_reward_zero_std": 0.0, "grad_norm": 0.29886093735694885, "kl": 0.6669160267338157, "learning_rate": 3.7746811179896937e-06, "loss": -0.0526, "num_tokens": 32773917.0, "reward": 1.40625, "reward_std": 0.6289876699447632, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.7120789289474487, "sampling/importance_sampling_ratio/max": 1.2994592189788818, "sampling/importance_sampling_ratio/mean": 1.0258033275604248, "sampling/importance_sampling_ratio/min": 0.8317172527313232, "sampling/sampling_logp_difference/max": 0.2562144696712494, "sampling/sampling_logp_difference/mean": 0.011117802001535892, "step": 667, "step_time": 14.046824933990138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 397.8125, "completions/mean_terminated_length": 397.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05420020001474768, "epoch": 0.01336, "frac_reward_zero_std": 0.0, "grad_norm": 0.08799777179956436, "kl": 1.4298119489103556, "learning_rate": 3.7644665306404726e-06, "loss": -0.0829, "num_tokens": 32827421.0, "reward": 1.625, "reward_std": 1.1461822986602783, "rewards/rollout_reward_func/mean": 1.625, "rewards/rollout_reward_func/std": 1.1570261716842651, "sampling/importance_sampling_ratio/max": 1.157531976699829, "sampling/importance_sampling_ratio/mean": 1.0139806270599365, "sampling/importance_sampling_ratio/min": 0.4229814112186432, "sampling/sampling_logp_difference/max": 0.8603811264038086, "sampling/sampling_logp_difference/mean": 0.014401648193597794, "step": 668, "step_time": 16.169748685962986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 356.21875, "completions/mean_terminated_length": 356.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.059381644474342465, "epoch": 0.01338, "frac_reward_zero_std": 0.0, "grad_norm": 0.17732909321784973, "kl": 1.62298733741045, "learning_rate": 3.7542847291579375e-06, "loss": -0.0654, "num_tokens": 32877851.0, "reward": 1.75, "reward_std": 0.6817530393600464, "rewards/rollout_reward_func/mean": 1.75, "rewards/rollout_reward_func/std": 0.7620007395744324, "sampling/importance_sampling_ratio/max": 1.1432119607925415, "sampling/importance_sampling_ratio/mean": 0.9998195171356201, "sampling/importance_sampling_ratio/min": 0.4442078471183777, "sampling/sampling_logp_difference/max": 0.8511903285980225, "sampling/sampling_logp_difference/mean": 0.02070648781955242, "step": 669, "step_time": 15.803513194958214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 321.6875, "completions/mean_terminated_length": 321.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0746223833411932, "epoch": 0.0134, "frac_reward_zero_std": 0.0, "grad_norm": 0.6669606566429138, "kl": 3.408542811870575, "learning_rate": 3.744135847846937e-06, "loss": -0.0467, "num_tokens": 32927707.0, "reward": 1.4375, "reward_std": 0.9866507053375244, "rewards/rollout_reward_func/mean": 1.4375, "rewards/rollout_reward_func/std": 1.014014720916748, "sampling/importance_sampling_ratio/max": 1.2399128675460815, "sampling/importance_sampling_ratio/mean": 1.0380980968475342, "sampling/importance_sampling_ratio/min": 0.8901597857475281, "sampling/sampling_logp_difference/max": 0.204263836145401, "sampling/sampling_logp_difference/mean": 0.012548938393592834, "step": 670, "step_time": 14.155690989995492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1908.0, "completions/max_terminated_length": 1908.0, "completions/mean_length": 451.78125, "completions/mean_terminated_length": 451.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07061795028857887, "epoch": 0.01342, "frac_reward_zero_std": 0.0, "grad_norm": 0.358122855424881, "kl": 1.3231714386492968, "learning_rate": 3.7340200205780724e-06, "loss": -0.0857, "num_tokens": 32982731.0, "reward": 1.3125, "reward_std": 1.5432931184768677, "rewards/rollout_reward_func/mean": 1.3125, "rewards/rollout_reward_func/std": 1.5332340002059937, "sampling/importance_sampling_ratio/max": 1.3505940437316895, "sampling/importance_sampling_ratio/mean": 0.9569368362426758, "sampling/importance_sampling_ratio/min": 0.38830602169036865, "sampling/sampling_logp_difference/max": 1.1026579141616821, "sampling/sampling_logp_difference/mean": 0.03713159263134003, "step": 671, "step_time": 16.985881603963207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 240.46875, "completions/mean_terminated_length": 240.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05523931956849992, "epoch": 0.01344, "frac_reward_zero_std": 0.0, "grad_norm": 0.051619578152894974, "kl": 1.032597467303276, "learning_rate": 3.723937380785948e-06, "loss": -0.0546, "num_tokens": 33028637.0, "reward": 1.5, "reward_std": 0.5260357856750488, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.1118632555007935, "sampling/importance_sampling_ratio/mean": 0.9889015555381775, "sampling/importance_sampling_ratio/min": 0.38065671920776367, "sampling/sampling_logp_difference/max": 0.9773776531219482, "sampling/sampling_logp_difference/mean": 0.022909702733159065, "step": 672, "step_time": 14.912200646984274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 348.65625, "completions/mean_terminated_length": 348.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0760893221013248, "epoch": 0.01346, "frac_reward_zero_std": 0.0, "grad_norm": 0.1706496775150299, "kl": 4.001097034662962, "learning_rate": 3.7138880614673957e-06, "loss": -0.0809, "num_tokens": 33080182.0, "reward": 1.5, "reward_std": 1.080770492553711, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 1.0776318311691284, "sampling/importance_sampling_ratio/max": 1.3238277435302734, "sampling/importance_sampling_ratio/mean": 0.9852933287620544, "sampling/importance_sampling_ratio/min": 0.29585185647010803, "sampling/sampling_logp_difference/max": 1.2194812297821045, "sampling/sampling_logp_difference/mean": 0.03375532105565071, "step": 673, "step_time": 13.907851323965588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1643.0, "completions/max_terminated_length": 1643.0, "completions/mean_length": 315.9375, "completions/mean_terminated_length": 315.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07311079511418939, "epoch": 0.01348, "frac_reward_zero_std": 0.0, "grad_norm": 0.1932215541601181, "kl": 0.8390037454664707, "learning_rate": 3.7038721951797318e-06, "loss": -0.0607, "num_tokens": 33129948.0, "reward": 1.28125, "reward_std": 1.021676778793335, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.054464340209961, "sampling/importance_sampling_ratio/max": 1.2332608699798584, "sampling/importance_sampling_ratio/mean": 1.0265370607376099, "sampling/importance_sampling_ratio/min": 0.747738778591156, "sampling/sampling_logp_difference/max": 0.30303955078125, "sampling/sampling_logp_difference/mean": 0.011980025097727776, "step": 674, "step_time": 16.395741716973134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 244.28125, "completions/mean_terminated_length": 244.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.061781576252542436, "epoch": 0.0135, "frac_reward_zero_std": 0.0, "grad_norm": 0.22899369895458221, "kl": 1.6343337092548609, "learning_rate": 3.6938899140390016e-06, "loss": -0.0551, "num_tokens": 33175737.0, "reward": 1.28125, "reward_std": 0.822746753692627, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 0.8513509035110474, "sampling/importance_sampling_ratio/max": 1.0780376195907593, "sampling/importance_sampling_ratio/mean": 0.9910967946052551, "sampling/importance_sampling_ratio/min": 0.4286038875579834, "sampling/sampling_logp_difference/max": 0.8474347591400146, "sampling/sampling_logp_difference/mean": 0.021232545375823975, "step": 675, "step_time": 15.596168249001494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 286.4375, "completions/mean_terminated_length": 286.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0715662487782538, "epoch": 0.01352, "frac_reward_zero_std": 0.0, "grad_norm": 0.20106269419193268, "kl": 1.882515151053667, "learning_rate": 3.683941349718244e-06, "loss": -0.0685, "num_tokens": 33223030.0, "reward": 1.34375, "reward_std": 0.9752559065818787, "rewards/rollout_reward_func/mean": 1.34375, "rewards/rollout_reward_func/std": 1.0658745765686035, "sampling/importance_sampling_ratio/max": 1.185667634010315, "sampling/importance_sampling_ratio/mean": 0.9857357740402222, "sampling/importance_sampling_ratio/min": 0.42622068524360657, "sampling/sampling_logp_difference/max": 0.8528261184692383, "sampling/sampling_logp_difference/mean": 0.029495753347873688, "step": 676, "step_time": 13.972588977019768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 197.03125, "completions/mean_terminated_length": 197.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06610518088564277, "epoch": 0.01354, "frac_reward_zero_std": 0.0, "grad_norm": 0.10553254932165146, "kl": 0.6294401008635759, "learning_rate": 3.674026633445747e-06, "loss": -0.0496, "num_tokens": 33269048.0, "reward": 1.46875, "reward_std": 0.7324849963188171, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 0.761339008808136, "sampling/importance_sampling_ratio/max": 1.1397427320480347, "sampling/importance_sampling_ratio/mean": 1.011904001235962, "sampling/importance_sampling_ratio/min": 0.7506617903709412, "sampling/sampling_logp_difference/max": 0.28681015968322754, "sampling/sampling_logp_difference/mean": 0.009260179474949837, "step": 677, "step_time": 13.855582427015179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 356.4375, "completions/mean_terminated_length": 356.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07229595794342458, "epoch": 0.01356, "frac_reward_zero_std": 0.0, "grad_norm": 0.4826365113258362, "kl": 2.4904955122619867, "learning_rate": 3.6641458960033194e-06, "loss": -0.0814, "num_tokens": 33320786.0, "reward": 1.375, "reward_std": 0.9453872442245483, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.941858172416687, "sampling/importance_sampling_ratio/max": 1.2368361949920654, "sampling/importance_sampling_ratio/mean": 0.9910211563110352, "sampling/importance_sampling_ratio/min": 0.3104995787143707, "sampling/sampling_logp_difference/max": 1.178657054901123, "sampling/sampling_logp_difference/mean": 0.024544749408960342, "step": 678, "step_time": 14.143467489004252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 237.9375, "completions/mean_terminated_length": 237.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.052044145413674414, "epoch": 0.01358, "frac_reward_zero_std": 0.0, "grad_norm": 0.5357210636138916, "kl": 0.822309510782361, "learning_rate": 3.654299267724567e-06, "loss": -0.0461, "num_tokens": 33366837.0, "reward": 1.5, "reward_std": 0.6946247816085815, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.6720215082168579, "sampling/importance_sampling_ratio/max": 1.0613303184509277, "sampling/importance_sampling_ratio/mean": 0.9936050772666931, "sampling/importance_sampling_ratio/min": 0.528894305229187, "sampling/sampling_logp_difference/max": 0.6427799463272095, "sampling/sampling_logp_difference/mean": 0.016951624304056168, "step": 679, "step_time": 13.696736071971827 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 343.28125, "completions/mean_terminated_length": 343.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06189836375415325, "epoch": 0.0136, "frac_reward_zero_std": 0.0, "grad_norm": 0.45010629296302795, "kl": 2.3131552785634995, "learning_rate": 3.6444868784931752e-06, "loss": -0.0554, "num_tokens": 33418330.0, "reward": 1.5, "reward_std": 0.884680986404419, "rewards/rollout_reward_func/mean": 1.5, "rewards/rollout_reward_func/std": 0.9158109426498413, "sampling/importance_sampling_ratio/max": 1.1669450998306274, "sampling/importance_sampling_ratio/mean": 0.9972683191299438, "sampling/importance_sampling_ratio/min": 0.421919584274292, "sampling/sampling_logp_difference/max": 0.8705010414123535, "sampling/sampling_logp_difference/mean": 0.01940082386136055, "step": 680, "step_time": 16.408535568029038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1261.0, "completions/max_terminated_length": 1261.0, "completions/mean_length": 259.4375, "completions/mean_terminated_length": 259.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06706908298656344, "epoch": 0.01362, "frac_reward_zero_std": 0.0, "grad_norm": 0.2389785647392273, "kl": 1.9678827114403248, "learning_rate": 3.6347088577411906e-06, "loss": -0.0624, "num_tokens": 33465980.0, "reward": 1.21875, "reward_std": 1.0800068378448486, "rewards/rollout_reward_func/mean": 1.21875, "rewards/rollout_reward_func/std": 1.0993949174880981, "sampling/importance_sampling_ratio/max": 1.1893384456634521, "sampling/importance_sampling_ratio/mean": 0.9734315872192383, "sampling/importance_sampling_ratio/min": 0.2582632601261139, "sampling/sampling_logp_difference/max": 1.3614559173583984, "sampling/sampling_logp_difference/mean": 0.030047884210944176, "step": 681, "step_time": 14.859536168005434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 410.75, "completions/mean_terminated_length": 410.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08274864917621017, "epoch": 0.01364, "frac_reward_zero_std": 0.0, "grad_norm": 0.4429612457752228, "kl": 3.4725145921111107, "learning_rate": 3.6249653344473162e-06, "loss": -0.0756, "num_tokens": 33519611.0, "reward": 1.5625, "reward_std": 1.1684800386428833, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 1.1622419357299805, "sampling/importance_sampling_ratio/max": 1.3139814138412476, "sampling/importance_sampling_ratio/mean": 1.0097146034240723, "sampling/importance_sampling_ratio/min": 0.298782616853714, "sampling/sampling_logp_difference/max": 1.2221484184265137, "sampling/sampling_logp_difference/mean": 0.02441716194152832, "step": 682, "step_time": 15.254157615971053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 287.25, "completions/mean_terminated_length": 287.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06302850903011858, "epoch": 0.01366, "frac_reward_zero_std": 0.0, "grad_norm": 0.1722397655248642, "kl": 0.8841247390955687, "learning_rate": 3.61525643713521e-06, "loss": -0.034, "num_tokens": 33568962.0, "reward": 1.40625, "reward_std": 0.7925156354904175, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 0.9108441472053528, "sampling/importance_sampling_ratio/max": 1.076150894165039, "sampling/importance_sampling_ratio/mean": 1.003968358039856, "sampling/importance_sampling_ratio/min": 0.49374642968177795, "sampling/sampling_logp_difference/max": 0.7114965915679932, "sampling/sampling_logp_difference/mean": 0.012352990917861462, "step": 683, "step_time": 12.800843767035985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 257.46875, "completions/mean_terminated_length": 257.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07475509191863239, "epoch": 0.01368, "frac_reward_zero_std": 0.0, "grad_norm": 0.39890578389167786, "kl": 0.7174127511680126, "learning_rate": 3.6055822938717933e-06, "loss": -0.0662, "num_tokens": 33616372.0, "reward": 1.65625, "reward_std": 0.555522084236145, "rewards/rollout_reward_func/mean": 1.65625, "rewards/rollout_reward_func/std": 0.6015772223472595, "sampling/importance_sampling_ratio/max": 1.519679307937622, "sampling/importance_sampling_ratio/mean": 1.0274708271026611, "sampling/importance_sampling_ratio/min": 0.7128245234489441, "sampling/sampling_logp_difference/max": 0.33861780166625977, "sampling/sampling_logp_difference/mean": 0.012472162023186684, "step": 684, "step_time": 13.805238284985535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1310.0, "completions/max_terminated_length": 1310.0, "completions/mean_length": 407.09375, "completions/mean_terminated_length": 407.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06470836466178298, "epoch": 0.0137, "frac_reward_zero_std": 0.0, "grad_norm": 0.2752668559551239, "kl": 2.2128655165433884, "learning_rate": 3.595943032265554e-06, "loss": -0.0572, "num_tokens": 33669872.0, "reward": 1.375, "reward_std": 1.2343297004699707, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.2115039825439453, "sampling/importance_sampling_ratio/max": 1.129644751548767, "sampling/importance_sampling_ratio/mean": 0.9574573040008545, "sampling/importance_sampling_ratio/min": 0.3524482250213623, "sampling/sampling_logp_difference/max": 1.1099690198898315, "sampling/sampling_logp_difference/mean": 0.034887224435806274, "step": 685, "step_time": 14.733808773977216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1760.0, "completions/max_terminated_length": 1760.0, "completions/mean_length": 366.46875, "completions/mean_terminated_length": 366.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.058880759635940194, "epoch": 0.01372, "frac_reward_zero_std": 0.0, "grad_norm": 0.3958154022693634, "kl": 4.144379839301109, "learning_rate": 3.5863387794648695e-06, "loss": -0.058, "num_tokens": 33720947.0, "reward": 1.46875, "reward_std": 0.9501429200172424, "rewards/rollout_reward_func/mean": 1.46875, "rewards/rollout_reward_func/std": 1.0155048370361328, "sampling/importance_sampling_ratio/max": 1.1646445989608765, "sampling/importance_sampling_ratio/mean": 0.9474985599517822, "sampling/importance_sampling_ratio/min": 0.2252776324748993, "sampling/sampling_logp_difference/max": 1.1746368408203125, "sampling/sampling_logp_difference/mean": 0.0471048466861248, "step": 686, "step_time": 16.692866521028918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 392.46875, "completions/mean_terminated_length": 392.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06354508409276605, "epoch": 0.01374, "frac_reward_zero_std": 0.0, "grad_norm": 0.09413468837738037, "kl": 0.7325948951765895, "learning_rate": 3.5767696621563247e-06, "loss": -0.0732, "num_tokens": 33773582.0, "reward": 1.8125, "reward_std": 0.8794784545898438, "rewards/rollout_reward_func/mean": 1.8125, "rewards/rollout_reward_func/std": 0.8957785964012146, "sampling/importance_sampling_ratio/max": 1.2763417959213257, "sampling/importance_sampling_ratio/mean": 1.0419652462005615, "sampling/importance_sampling_ratio/min": 1.0068929195404053, "sampling/sampling_logp_difference/max": 0.24166935682296753, "sampling/sampling_logp_difference/mean": 0.010131238028407097, "step": 687, "step_time": 15.461645193019649 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 334.5625, "completions/mean_terminated_length": 334.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.060067824088037014, "epoch": 0.01376, "frac_reward_zero_std": 0.0, "grad_norm": 0.09923091530799866, "kl": 3.547912683337927, "learning_rate": 3.5672358065630486e-06, "loss": -0.0786, "num_tokens": 33825176.0, "reward": 1.5625, "reward_std": 1.1829134225845337, "rewards/rollout_reward_func/mean": 1.5625, "rewards/rollout_reward_func/std": 1.1896733045578003, "sampling/importance_sampling_ratio/max": 1.2230404615402222, "sampling/importance_sampling_ratio/mean": 0.9743572473526001, "sampling/importance_sampling_ratio/min": 0.37832367420196533, "sampling/sampling_logp_difference/max": 0.9734766483306885, "sampling/sampling_logp_difference/mean": 0.025436118245124817, "step": 688, "step_time": 15.901833622017875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1791.0, "completions/max_terminated_length": 1791.0, "completions/mean_length": 355.9375, "completions/mean_terminated_length": 355.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08102198713459074, "epoch": 0.01378, "frac_reward_zero_std": 0.0, "grad_norm": 0.36352258920669556, "kl": 1.2132229609414935, "learning_rate": 3.5577373384430397e-06, "loss": -0.0506, "num_tokens": 33877742.0, "reward": 1.375, "reward_std": 1.0296026468276978, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.008032202720642, "sampling/importance_sampling_ratio/max": 1.1990530490875244, "sampling/importance_sampling_ratio/mean": 0.9877309799194336, "sampling/importance_sampling_ratio/min": 0.32544660568237305, "sampling/sampling_logp_difference/max": 1.058868408203125, "sampling/sampling_logp_difference/mean": 0.024108141660690308, "step": 689, "step_time": 16.746552256983705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 284.15625, "completions/mean_terminated_length": 284.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06646771053783596, "epoch": 0.0138, "frac_reward_zero_std": 0.0, "grad_norm": 0.251034140586853, "kl": 1.510088324546814, "learning_rate": 3.5482743830875137e-06, "loss": -0.0643, "num_tokens": 33926862.0, "reward": 1.53125, "reward_std": 0.7310357689857483, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.761339008808136, "sampling/importance_sampling_ratio/max": 1.1824918985366821, "sampling/importance_sampling_ratio/mean": 0.9802519083023071, "sampling/importance_sampling_ratio/min": 0.2970295250415802, "sampling/sampling_logp_difference/max": 1.232856273651123, "sampling/sampling_logp_difference/mean": 0.029140401631593704, "step": 690, "step_time": 13.172906147039612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 271.65625, "completions/mean_terminated_length": 271.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05452257767319679, "epoch": 0.01382, "frac_reward_zero_std": 0.0, "grad_norm": 0.11116041988134384, "kl": 3.8733133785426617, "learning_rate": 3.538847065319248e-06, "loss": -0.0792, "num_tokens": 33974722.0, "reward": 1.375, "reward_std": 1.1728968620300293, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 1.1845782995224, "sampling/importance_sampling_ratio/max": 1.060695767402649, "sampling/importance_sampling_ratio/mean": 0.9709091186523438, "sampling/importance_sampling_ratio/min": 0.4240904450416565, "sampling/sampling_logp_difference/max": 0.857755184173584, "sampling/sampling_logp_difference/mean": 0.02749958634376526, "step": 691, "step_time": 14.11179084701871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 311.4375, "completions/mean_terminated_length": 311.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06927778944373131, "epoch": 0.01384, "frac_reward_zero_std": 0.0, "grad_norm": 0.06941089779138565, "kl": 1.103413861244917, "learning_rate": 3.5294555094909387e-06, "loss": -0.0642, "num_tokens": 34023715.0, "reward": 1.40625, "reward_std": 1.0838029384613037, "rewards/rollout_reward_func/mean": 1.40625, "rewards/rollout_reward_func/std": 1.0734140872955322, "sampling/importance_sampling_ratio/max": 1.2072718143463135, "sampling/importance_sampling_ratio/mean": 1.020906925201416, "sampling/importance_sampling_ratio/min": 0.7022214531898499, "sampling/sampling_logp_difference/max": 0.353485107421875, "sampling/sampling_logp_difference/mean": 0.011718081310391426, "step": 692, "step_time": 15.520545714971377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 347.375, "completions/mean_terminated_length": 347.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06663013668730855, "epoch": 0.01386, "frac_reward_zero_std": 0.0, "grad_norm": 0.2995471954345703, "kl": 2.6755752004683018, "learning_rate": 3.520099839483554e-06, "loss": -0.0438, "num_tokens": 34075990.0, "reward": 1.28125, "reward_std": 0.842013418674469, "rewards/rollout_reward_func/mean": 1.28125, "rewards/rollout_reward_func/std": 1.0234153270721436, "sampling/importance_sampling_ratio/max": 1.1793161630630493, "sampling/importance_sampling_ratio/mean": 0.9711602926254272, "sampling/importance_sampling_ratio/min": 0.35566842555999756, "sampling/sampling_logp_difference/max": 1.0659725666046143, "sampling/sampling_logp_difference/mean": 0.030320139601826668, "step": 693, "step_time": 15.311282160982955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 332.1875, "completions/mean_terminated_length": 332.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.060409385710954666, "epoch": 0.01388, "frac_reward_zero_std": 0.0, "grad_norm": 0.4058360755443573, "kl": 0.8353172792121768, "learning_rate": 3.5107801787047088e-06, "loss": -0.0631, "num_tokens": 34128618.0, "reward": 1.53125, "reward_std": 0.8630099296569824, "rewards/rollout_reward_func/mean": 1.53125, "rewards/rollout_reward_func/std": 0.915260374546051, "sampling/importance_sampling_ratio/max": 1.0849990844726562, "sampling/importance_sampling_ratio/mean": 0.9672768115997314, "sampling/importance_sampling_ratio/min": 0.3146301507949829, "sampling/sampling_logp_difference/max": 1.1590635776519775, "sampling/sampling_logp_difference/mean": 0.02726798504590988, "step": 694, "step_time": 15.917104422027478 } ], "logging_steps": 1.0, "max_steps": 900, "num_input_tokens_seen": 34128618, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }