{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.015, "eval_steps": 500, "global_step": 750, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 83.34375, "completions/mean_terminated_length": 83.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0397388190031052, "epoch": 2e-05, "frac_reward_zero_std": 0.25, "grad_norm": 0.7260493636131287, "kl": 0.0, "learning_rate": 0.0, "loss": 0.0019, "num_tokens": 42118.0, "reward": -0.2968750298023224, "reward_std": 0.09859944134950638, "rewards/rollout_reward_func/mean": -0.2968750298023224, "rewards/rollout_reward_func/std": 0.13071608543395996, "sampling/importance_sampling_ratio/max": 1.2712912559509277, "sampling/importance_sampling_ratio/mean": 1.004990816116333, "sampling/importance_sampling_ratio/min": 0.6276183128356934, "sampling/sampling_logp_difference/max": 0.39116203784942627, "sampling/sampling_logp_difference/mean": 0.05087655410170555, "step": 1, "step_time": 12.273976954997124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 61.40625, "completions/mean_terminated_length": 61.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1221930272877216, "epoch": 4e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.3843214511871338, "kl": 0.0, "learning_rate": 2.8571428571428575e-07, "loss": 0.0134, "num_tokens": 84436.0, "reward": -0.3187500238418579, "reward_std": 0.08591178804636002, "rewards/rollout_reward_func/mean": -0.3187500238418579, "rewards/rollout_reward_func/std": 0.08957786113023758, "sampling/importance_sampling_ratio/max": 1.2282915115356445, "sampling/importance_sampling_ratio/mean": 1.0003398656845093, "sampling/importance_sampling_ratio/min": 0.7719176411628723, "sampling/sampling_logp_difference/max": 0.20552897453308105, "sampling/sampling_logp_difference/mean": 0.03787406533956528, "step": 2, "step_time": 11.86914219600294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 420.0, "completions/max_terminated_length": 420.0, "completions/mean_length": 23.34375, "completions/mean_terminated_length": 23.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0694084540009499, "epoch": 6e-05, "frac_reward_zero_std": 0.5, "grad_norm": 1.2200404405593872, "kl": 0.0016136430203914642, "learning_rate": 5.714285714285715e-07, "loss": 0.0118, "num_tokens": 123518.0, "reward": -0.3125, "reward_std": 0.03946366906166077, "rewards/rollout_reward_func/mean": -0.3125, "rewards/rollout_reward_func/std": 0.05535807088017464, "sampling/importance_sampling_ratio/max": 1.3418179750442505, "sampling/importance_sampling_ratio/mean": 1.0008726119995117, "sampling/importance_sampling_ratio/min": 0.8257654905319214, "sampling/sampling_logp_difference/max": 0.29405879974365234, "sampling/sampling_logp_difference/mean": 0.03328881412744522, "step": 3, "step_time": 11.428667624993977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 113.25, "completions/mean_terminated_length": 113.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0204078741371632, "epoch": 8e-05, "frac_reward_zero_std": 0.25, "grad_norm": 0.8974786996841431, "kl": 0.0023681921884417534, "learning_rate": 8.571428571428572e-07, "loss": -0.0151, "num_tokens": 166576.0, "reward": -0.26875001192092896, "reward_std": 0.1148378849029541, "rewards/rollout_reward_func/mean": -0.26875001192092896, "rewards/rollout_reward_func/std": 0.13060036301612854, "sampling/importance_sampling_ratio/max": 1.3098548650741577, "sampling/importance_sampling_ratio/mean": 0.9761923551559448, "sampling/importance_sampling_ratio/min": 0.6181589961051941, "sampling/sampling_logp_difference/max": 0.4020094871520996, "sampling/sampling_logp_difference/mean": 0.04733721911907196, "step": 4, "step_time": 11.281017079994854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 113.96875, "completions/mean_terminated_length": 113.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1056617759168148, "epoch": 0.0001, "frac_reward_zero_std": 0.25, "grad_norm": 1.1588466167449951, "kl": 0.0021881135180592537, "learning_rate": 1.142857142857143e-06, "loss": -0.0005, "num_tokens": 209811.0, "reward": -0.2875000238418579, "reward_std": 0.11553343385457993, "rewards/rollout_reward_func/mean": -0.2875000238418579, "rewards/rollout_reward_func/std": 0.131369948387146, "sampling/importance_sampling_ratio/max": 1.3284457921981812, "sampling/importance_sampling_ratio/mean": 1.0197780132293701, "sampling/importance_sampling_ratio/min": 0.646064043045044, "sampling/sampling_logp_difference/max": 0.4368925094604492, "sampling/sampling_logp_difference/mean": 0.04656064510345459, "step": 5, "step_time": 13.126878773993667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 449.0, "completions/max_terminated_length": 449.0, "completions/mean_length": 30.84375, "completions/mean_terminated_length": 30.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1447249948978424, "epoch": 0.00012, "frac_reward_zero_std": 0.5, "grad_norm": 0.6994903683662415, "kl": 0.001296980928600533, "learning_rate": 1.4285714285714286e-06, "loss": -0.0031, "num_tokens": 250411.0, "reward": -0.26875001192092896, "reward_std": 0.07703812420368195, "rewards/rollout_reward_func/mean": -0.26875001192092896, "rewards/rollout_reward_func/std": 0.13304740190505981, "sampling/importance_sampling_ratio/max": 1.2058994770050049, "sampling/importance_sampling_ratio/mean": 1.0176870822906494, "sampling/importance_sampling_ratio/min": 0.8486367464065552, "sampling/sampling_logp_difference/max": 0.18718594312667847, "sampling/sampling_logp_difference/mean": 0.03341532126069069, "step": 6, "step_time": 10.64448972100763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 83.625, "completions/mean_terminated_length": 83.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0607584677636623, "epoch": 0.00014, "frac_reward_zero_std": 0.25, "grad_norm": 1.3435159921646118, "kl": 0.003473934717476368, "learning_rate": 1.7142857142857145e-06, "loss": 0.0207, "num_tokens": 292217.0, "reward": -0.3031250238418579, "reward_std": 0.0990089625120163, "rewards/rollout_reward_func/mean": -0.3031250238418579, "rewards/rollout_reward_func/std": 0.12822453677654266, "sampling/importance_sampling_ratio/max": 1.3013660907745361, "sampling/importance_sampling_ratio/mean": 1.0478312969207764, "sampling/importance_sampling_ratio/min": 0.7597411274909973, "sampling/sampling_logp_difference/max": 0.2633233070373535, "sampling/sampling_logp_difference/mean": 0.04644894599914551, "step": 7, "step_time": 11.122955887996795 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.03125, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 143.46875, "completions/mean_terminated_length": 147.5806427001953, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0313463173806667, "epoch": 0.00016, "frac_reward_zero_std": 0.0, "grad_norm": 1.09323251247406, "kl": 0.003279088530689478, "learning_rate": 2.0000000000000003e-06, "loss": 0.0201, "num_tokens": 336759.0, "reward": -0.3031250238418579, "reward_std": 0.1953982710838318, "rewards/rollout_reward_func/mean": -0.3031250238418579, "rewards/rollout_reward_func/std": 0.21324583888053894, "sampling/importance_sampling_ratio/max": 1.2366067171096802, "sampling/importance_sampling_ratio/mean": 0.9369519352912903, "sampling/importance_sampling_ratio/min": 0.5705882906913757, "sampling/sampling_logp_difference/max": 0.5535283088684082, "sampling/sampling_logp_difference/mean": 0.0574895516037941, "step": 8, "step_time": 12.327238315992872 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 112.6875, "completions/mean_terminated_length": 112.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9977701045572758, "epoch": 0.00018, "frac_reward_zero_std": 0.0, "grad_norm": 1.0577385425567627, "kl": 0.0027210311964154243, "learning_rate": 2.285714285714286e-06, "loss": 0.0114, "num_tokens": 378225.0, "reward": -0.24062500894069672, "reward_std": 0.22852176427841187, "rewards/rollout_reward_func/mean": -0.24062500894069672, "rewards/rollout_reward_func/std": 0.22412464022636414, "sampling/importance_sampling_ratio/max": 1.18338143825531, "sampling/importance_sampling_ratio/mean": 0.9805333018302917, "sampling/importance_sampling_ratio/min": 0.7500500082969666, "sampling/sampling_logp_difference/max": 0.22987604141235352, "sampling/sampling_logp_difference/mean": 0.038257744163274765, "step": 9, "step_time": 11.296767864996582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.0, "completions/max_terminated_length": 294.0, "completions/mean_length": 32.875, "completions/mean_terminated_length": 32.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0721546858549118, "epoch": 0.0002, "frac_reward_zero_std": 0.25, "grad_norm": 0.9819568395614624, "kl": 0.003052409738302231, "learning_rate": 2.571428571428571e-06, "loss": -0.0166, "num_tokens": 418035.0, "reward": -0.24375000596046448, "reward_std": 0.15015468001365662, "rewards/rollout_reward_func/mean": -0.24375000596046448, "rewards/rollout_reward_func/std": 0.17586193978786469, "sampling/importance_sampling_ratio/max": 1.271618127822876, "sampling/importance_sampling_ratio/mean": 0.995391309261322, "sampling/importance_sampling_ratio/min": 0.5087024569511414, "sampling/sampling_logp_difference/max": 0.653008222579956, "sampling/sampling_logp_difference/mean": 0.04060251638293266, "step": 10, "step_time": 10.5546528520008 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 109.875, "completions/mean_terminated_length": 109.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0462245643138885, "epoch": 0.00022, "frac_reward_zero_std": 0.25, "grad_norm": 0.9850079417228699, "kl": 0.003675127401947975, "learning_rate": 2.8571428571428573e-06, "loss": -0.0125, "num_tokens": 461313.0, "reward": -0.25312501192092896, "reward_std": 0.14475291967391968, "rewards/rollout_reward_func/mean": -0.25312501192092896, "rewards/rollout_reward_func/std": 0.17957724630832672, "sampling/importance_sampling_ratio/max": 1.2111577987670898, "sampling/importance_sampling_ratio/mean": 0.9596511721611023, "sampling/importance_sampling_ratio/min": 0.488716721534729, "sampling/sampling_logp_difference/max": 0.7027390003204346, "sampling/sampling_logp_difference/mean": 0.042339835315942764, "step": 11, "step_time": 12.815628993997962 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 69.84375, "completions/mean_terminated_length": 69.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0279440879821777, "epoch": 0.00024, "frac_reward_zero_std": 0.0, "grad_norm": 1.1289873123168945, "kl": 0.0030637606978416443, "learning_rate": 3.142857142857143e-06, "loss": 0.0073, "num_tokens": 501927.0, "reward": -0.2562500238418579, "reward_std": 0.14171326160430908, "rewards/rollout_reward_func/mean": -0.2562500238418579, "rewards/rollout_reward_func/std": 0.18128128349781036, "sampling/importance_sampling_ratio/max": 1.2403898239135742, "sampling/importance_sampling_ratio/mean": 1.0321519374847412, "sampling/importance_sampling_ratio/min": 0.7848139405250549, "sampling/sampling_logp_difference/max": 0.26319456100463867, "sampling/sampling_logp_difference/mean": 0.041116178035736084, "step": 12, "step_time": 10.835548502996971 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 62.125, "completions/mean_terminated_length": 62.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0623879544436932, "epoch": 0.00026, "frac_reward_zero_std": 0.25, "grad_norm": 0.9900627732276917, "kl": 0.003036552108824253, "learning_rate": 3.428571428571429e-06, "loss": 0.0054, "num_tokens": 543903.0, "reward": -0.2875000238418579, "reward_std": 0.09943857789039612, "rewards/rollout_reward_func/mean": -0.2875000238418579, "rewards/rollout_reward_func/std": 0.12889105081558228, "sampling/importance_sampling_ratio/max": 1.4891777038574219, "sampling/importance_sampling_ratio/mean": 0.9849542379379272, "sampling/importance_sampling_ratio/min": 0.5315830111503601, "sampling/sampling_logp_difference/max": 0.4459831714630127, "sampling/sampling_logp_difference/mean": 0.05372384935617447, "step": 13, "step_time": 12.247658517007949 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.03125, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 61.96875, "completions/mean_terminated_length": 63.45161056518555, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0129014030098915, "epoch": 0.00028, "frac_reward_zero_std": 0.0, "grad_norm": 1.4545326232910156, "kl": 0.003433040459640324, "learning_rate": 3.7142857142857146e-06, "loss": 0.003, "num_tokens": 583924.0, "reward": -0.18437501788139343, "reward_std": 0.20255166292190552, "rewards/rollout_reward_func/mean": -0.18437501788139343, "rewards/rollout_reward_func/std": 0.22159157693386078, "sampling/importance_sampling_ratio/max": 1.4509334564208984, "sampling/importance_sampling_ratio/mean": 0.9957286715507507, "sampling/importance_sampling_ratio/min": 0.5931320190429688, "sampling/sampling_logp_difference/max": 0.46323466300964355, "sampling/sampling_logp_difference/mean": 0.04744837060570717, "step": 14, "step_time": 11.63132996600143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 483.0, "completions/max_terminated_length": 483.0, "completions/mean_length": 52.0, "completions/mean_terminated_length": 52.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0597737990319729, "epoch": 0.0003, "frac_reward_zero_std": 0.0, "grad_norm": 1.70634126663208, "kl": 0.005508672446012497, "learning_rate": 4.000000000000001e-06, "loss": 0.0103, "num_tokens": 624166.0, "reward": -0.25, "reward_std": 0.20947116613388062, "rewards/rollout_reward_func/mean": -0.25, "rewards/rollout_reward_func/std": 0.22143228352069855, "sampling/importance_sampling_ratio/max": 1.3188961744308472, "sampling/importance_sampling_ratio/mean": 0.9935175180435181, "sampling/importance_sampling_ratio/min": 0.4995708167552948, "sampling/sampling_logp_difference/max": 0.6939938068389893, "sampling/sampling_logp_difference/mean": 0.06921803951263428, "step": 15, "step_time": 10.588919255995279 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 330.0, "completions/max_terminated_length": 330.0, "completions/mean_length": 37.09375, "completions/mean_terminated_length": 37.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0824791602790356, "epoch": 0.00032, "frac_reward_zero_std": 0.25, "grad_norm": 0.8279272317886353, "kl": 0.006280859466642141, "learning_rate": 4.2857142857142855e-06, "loss": -0.0035, "num_tokens": 665309.0, "reward": -0.28125, "reward_std": 0.08490689098834991, "rewards/rollout_reward_func/mean": -0.28125, "rewards/rollout_reward_func/std": 0.10906493663787842, "sampling/importance_sampling_ratio/max": 1.457956075668335, "sampling/importance_sampling_ratio/mean": 1.0392749309539795, "sampling/importance_sampling_ratio/min": 0.757925271987915, "sampling/sampling_logp_difference/max": 0.3768739700317383, "sampling/sampling_logp_difference/mean": 0.04727257788181305, "step": 16, "step_time": 10.937567215003583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 62.40625, "completions/mean_terminated_length": 62.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0184055864810944, "epoch": 0.00034, "frac_reward_zero_std": 0.0, "grad_norm": 1.5595347881317139, "kl": 0.011038398370146751, "learning_rate": 4.571428571428572e-06, "loss": -0.0179, "num_tokens": 706668.0, "reward": -0.23750001192092896, "reward_std": 0.16588535904884338, "rewards/rollout_reward_func/mean": -0.23750001192092896, "rewards/rollout_reward_func/std": 0.1699146032333374, "sampling/importance_sampling_ratio/max": 1.275768518447876, "sampling/importance_sampling_ratio/mean": 0.9901074171066284, "sampling/importance_sampling_ratio/min": 0.6280444860458374, "sampling/sampling_logp_difference/max": 0.4652111530303955, "sampling/sampling_logp_difference/mean": 0.06158074736595154, "step": 17, "step_time": 12.009412818002602 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 92.03125, "completions/mean_terminated_length": 92.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.012953046709299, "epoch": 0.00036, "frac_reward_zero_std": 0.25, "grad_norm": 1.2660462856292725, "kl": 0.015820687636733055, "learning_rate": 4.857142857142858e-06, "loss": -0.0124, "num_tokens": 749723.0, "reward": -0.3031250238418579, "reward_std": 0.10824207216501236, "rewards/rollout_reward_func/mean": -0.3031250238418579, "rewards/rollout_reward_func/std": 0.13071607053279877, "sampling/importance_sampling_ratio/max": 1.30086088180542, "sampling/importance_sampling_ratio/mean": 0.9927637577056885, "sampling/importance_sampling_ratio/min": 0.5447718501091003, "sampling/sampling_logp_difference/max": 0.3807334899902344, "sampling/sampling_logp_difference/mean": 0.06408992409706116, "step": 18, "step_time": 10.92201188099898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 74.34375, "completions/mean_terminated_length": 74.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9684820547699928, "epoch": 0.00038, "frac_reward_zero_std": 0.0, "grad_norm": 1.1994088888168335, "kl": 0.029833496548235416, "learning_rate": 5.142857142857142e-06, "loss": 0.0033, "num_tokens": 791716.0, "reward": -0.26875001192092896, "reward_std": 0.15372103452682495, "rewards/rollout_reward_func/mean": -0.26875001192092896, "rewards/rollout_reward_func/std": 0.16740427911281586, "sampling/importance_sampling_ratio/max": 1.4069099426269531, "sampling/importance_sampling_ratio/mean": 0.9959492087364197, "sampling/importance_sampling_ratio/min": 0.35965102910995483, "sampling/sampling_logp_difference/max": 0.8241863250732422, "sampling/sampling_logp_difference/mean": 0.08972693234682083, "step": 19, "step_time": 12.252770283997961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 41.625, "completions/mean_terminated_length": 41.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9645320028066635, "epoch": 0.0004, "frac_reward_zero_std": 0.5, "grad_norm": 1.0281755924224854, "kl": 0.041882773221004754, "learning_rate": 5.428571428571429e-06, "loss": -0.008, "num_tokens": 832287.0, "reward": -0.24687501788139343, "reward_std": 0.13421207666397095, "rewards/rollout_reward_func/mean": -0.24687501788139343, "rewards/rollout_reward_func/std": 0.19004985690116882, "sampling/importance_sampling_ratio/max": 1.3697320222854614, "sampling/importance_sampling_ratio/mean": 0.9618366956710815, "sampling/importance_sampling_ratio/min": 0.5362023711204529, "sampling/sampling_logp_difference/max": 0.8683369159698486, "sampling/sampling_logp_difference/mean": 0.08718827366828918, "step": 20, "step_time": 10.75229195099746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 35.0, "completions/mean_terminated_length": 35.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8598319329321384, "epoch": 0.00042, "frac_reward_zero_std": 0.25, "grad_norm": 0.8959540128707886, "kl": 0.08626679610460997, "learning_rate": 5.7142857142857145e-06, "loss": -0.0174, "num_tokens": 872583.0, "reward": -0.24062500894069672, "reward_std": 0.18020054697990417, "rewards/rollout_reward_func/mean": -0.24062500894069672, "rewards/rollout_reward_func/std": 0.20613083243370056, "sampling/importance_sampling_ratio/max": 1.9147882461547852, "sampling/importance_sampling_ratio/mean": 1.020963430404663, "sampling/importance_sampling_ratio/min": 0.6859787702560425, "sampling/sampling_logp_difference/max": 0.6496127843856812, "sampling/sampling_logp_difference/mean": 0.0984824001789093, "step": 21, "step_time": 10.622931698000684 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 47.25, "completions/mean_terminated_length": 47.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9116772077977657, "epoch": 0.00044, "frac_reward_zero_std": 0.0, "grad_norm": 1.9752256870269775, "kl": 0.12213384243659675, "learning_rate": 6e-06, "loss": 0.0734, "num_tokens": 913857.0, "reward": -0.234375, "reward_std": 0.19321706891059875, "rewards/rollout_reward_func/mean": -0.234375, "rewards/rollout_reward_func/std": 0.2103903889656067, "sampling/importance_sampling_ratio/max": 1.9885673522949219, "sampling/importance_sampling_ratio/mean": 0.9943646788597107, "sampling/importance_sampling_ratio/min": 0.3362596929073334, "sampling/sampling_logp_difference/max": 1.2416329383850098, "sampling/sampling_logp_difference/mean": 0.13329307734966278, "step": 22, "step_time": 12.359769816999687 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 204.0, "completions/max_terminated_length": 204.0, "completions/mean_length": 14.625, "completions/mean_terminated_length": 14.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8666190654039383, "epoch": 0.00046, "frac_reward_zero_std": 0.0, "grad_norm": 1.4545626640319824, "kl": 0.0859798975288868, "learning_rate": 6.285714285714286e-06, "loss": -0.0198, "num_tokens": 954051.0, "reward": -0.21562500298023224, "reward_std": 0.19265145063400269, "rewards/rollout_reward_func/mean": -0.21562500298023224, "rewards/rollout_reward_func/std": 0.19528207182884216, "sampling/importance_sampling_ratio/max": 1.5457075834274292, "sampling/importance_sampling_ratio/mean": 0.9845030307769775, "sampling/importance_sampling_ratio/min": 0.5672403573989868, "sampling/sampling_logp_difference/max": 0.5668177604675293, "sampling/sampling_logp_difference/mean": 0.09585564583539963, "step": 23, "step_time": 10.231479685000522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 474.0, "completions/max_terminated_length": 474.0, "completions/mean_length": 35.84375, "completions/mean_terminated_length": 35.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7341250628232956, "epoch": 0.00048, "frac_reward_zero_std": 0.0, "grad_norm": 1.1490495204925537, "kl": 0.16938328556716442, "learning_rate": 6.571428571428572e-06, "loss": -0.0403, "num_tokens": 994703.0, "reward": -0.21250000596046448, "reward_std": 0.1677180528640747, "rewards/rollout_reward_func/mean": -0.21250000596046448, "rewards/rollout_reward_func/std": 0.1660742312669754, "sampling/importance_sampling_ratio/max": 2.444854259490967, "sampling/importance_sampling_ratio/mean": 1.0264357328414917, "sampling/importance_sampling_ratio/min": 0.3020070493221283, "sampling/sampling_logp_difference/max": 0.89404296875, "sampling/sampling_logp_difference/mean": 0.17652437090873718, "step": 24, "step_time": 11.194836254999245 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 27.5625, "completions/mean_terminated_length": 27.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.732364397495985, "epoch": 0.0005, "frac_reward_zero_std": 0.0, "grad_norm": 1.2313753366470337, "kl": 0.38774173241108656, "learning_rate": 6.857142857142858e-06, "loss": -0.0439, "num_tokens": 1033820.0, "reward": -0.11562500894069672, "reward_std": 0.2870022654533386, "rewards/rollout_reward_func/mean": -0.11562500894069672, "rewards/rollout_reward_func/std": 0.2852382957935333, "sampling/importance_sampling_ratio/max": 2.9343440532684326, "sampling/importance_sampling_ratio/mean": 1.034637689590454, "sampling/importance_sampling_ratio/min": 0.2391902655363083, "sampling/sampling_logp_difference/max": 1.7675800323486328, "sampling/sampling_logp_difference/mean": 0.24248787760734558, "step": 25, "step_time": 11.97011864099477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 93.875, "completions/mean_terminated_length": 93.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7024979125708342, "epoch": 0.00052, "frac_reward_zero_std": 0.0, "grad_norm": 0.6958019137382507, "kl": 0.41196694504469633, "learning_rate": 7.1428571428571436e-06, "loss": -0.0115, "num_tokens": 1077150.0, "reward": -0.140625, "reward_std": 0.24972474575042725, "rewards/rollout_reward_func/mean": -0.140625, "rewards/rollout_reward_func/std": 0.288331538438797, "sampling/importance_sampling_ratio/max": 1.9983669519424438, "sampling/importance_sampling_ratio/mean": 1.0290610790252686, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5962053537368774, "sampling/sampling_logp_difference/mean": 0.24064385890960693, "step": 26, "step_time": 11.18451771600121 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 53.96875, "completions/mean_terminated_length": 53.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6019238065928221, "epoch": 0.00054, "frac_reward_zero_std": 0.0, "grad_norm": 0.5256407856941223, "kl": 0.7407107260078192, "learning_rate": 7.428571428571429e-06, "loss": -0.0307, "num_tokens": 1117409.0, "reward": -0.109375, "reward_std": 0.2733415961265564, "rewards/rollout_reward_func/mean": -0.109375, "rewards/rollout_reward_func/std": 0.2763347625732422, "sampling/importance_sampling_ratio/max": 2.017904281616211, "sampling/importance_sampling_ratio/mean": 1.0500385761260986, "sampling/importance_sampling_ratio/min": 0.0709243044257164, "sampling/sampling_logp_difference/max": 1.6867249011993408, "sampling/sampling_logp_difference/mean": 0.23219454288482666, "step": 27, "step_time": 11.016059601999586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 35.15625, "completions/mean_terminated_length": 35.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5626984536647797, "epoch": 0.00056, "frac_reward_zero_std": 0.0, "grad_norm": 0.39997315406799316, "kl": 0.9575172234326601, "learning_rate": 7.714285714285716e-06, "loss": -0.0263, "num_tokens": 1156576.0, "reward": -0.11249999701976776, "reward_std": 0.28933054208755493, "rewards/rollout_reward_func/mean": -0.11249999701976776, "rewards/rollout_reward_func/std": 0.29154759645462036, "sampling/importance_sampling_ratio/max": 1.9845322370529175, "sampling/importance_sampling_ratio/mean": 1.0441625118255615, "sampling/importance_sampling_ratio/min": 0.18121188879013062, "sampling/sampling_logp_difference/max": 1.708014726638794, "sampling/sampling_logp_difference/mean": 0.2511787712574005, "step": 28, "step_time": 12.620438947002185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 411.0, "completions/max_terminated_length": 411.0, "completions/mean_length": 41.3125, "completions/mean_terminated_length": 41.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5920814573764801, "epoch": 0.00058, "frac_reward_zero_std": 0.0, "grad_norm": 0.9717136025428772, "kl": 1.013459773734212, "learning_rate": 8.000000000000001e-06, "loss": -0.017, "num_tokens": 1197886.0, "reward": -0.08437500149011612, "reward_std": 0.2666243016719818, "rewards/rollout_reward_func/mean": -0.08437500149011612, "rewards/rollout_reward_func/std": 0.27837011218070984, "sampling/importance_sampling_ratio/max": 1.69435453414917, "sampling/importance_sampling_ratio/mean": 0.9213829040527344, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.05306339263916, "sampling/sampling_logp_difference/mean": 0.1925433874130249, "step": 29, "step_time": 10.572838477000914 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 17.3125, "completions/mean_terminated_length": 17.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5898061292245984, "epoch": 0.0006, "frac_reward_zero_std": 0.0, "grad_norm": 0.7541149258613586, "kl": 3.3988371435552835, "learning_rate": 8.285714285714287e-06, "loss": -0.0236, "num_tokens": 1238193.0, "reward": -0.18437500298023224, "reward_std": 0.207572802901268, "rewards/rollout_reward_func/mean": -0.18437500298023224, "rewards/rollout_reward_func/std": 0.2397705465555191, "sampling/importance_sampling_ratio/max": 1.9151802062988281, "sampling/importance_sampling_ratio/mean": 0.8026424050331116, "sampling/importance_sampling_ratio/min": 0.08420759439468384, "sampling/sampling_logp_difference/max": 2.274259567260742, "sampling/sampling_logp_difference/mean": 0.2954370081424713, "step": 30, "step_time": 10.68174958000236 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 49.625, "completions/mean_terminated_length": 49.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5868647922761738, "epoch": 0.00062, "frac_reward_zero_std": 0.0, "grad_norm": 0.49025872349739075, "kl": 1.4634936433285475, "learning_rate": 8.571428571428571e-06, "loss": -0.0291, "num_tokens": 1279323.0, "reward": -0.13437500596046448, "reward_std": 0.2742108106613159, "rewards/rollout_reward_func/mean": -0.13437500596046448, "rewards/rollout_reward_func/std": 0.2634870111942291, "sampling/importance_sampling_ratio/max": 1.720544457435608, "sampling/importance_sampling_ratio/mean": 0.8649041652679443, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8548316955566406, "sampling/sampling_logp_difference/mean": 0.2290564775466919, "step": 31, "step_time": 12.042660880002586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 30.78125, "completions/mean_terminated_length": 30.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.591502342838794, "epoch": 0.00064, "frac_reward_zero_std": 0.0, "grad_norm": 0.7420617341995239, "kl": 5.5722776632755995, "learning_rate": 8.857142857142858e-06, "loss": -0.0229, "num_tokens": 1320778.0, "reward": -0.14375001192092896, "reward_std": 0.24282093346118927, "rewards/rollout_reward_func/mean": -0.14375001192092896, "rewards/rollout_reward_func/std": 0.2381887286901474, "sampling/importance_sampling_ratio/max": 1.68848717212677, "sampling/importance_sampling_ratio/mean": 0.744141697883606, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0187416076660156, "sampling/sampling_logp_difference/mean": 0.25374218821525574, "step": 32, "step_time": 10.66116141600287 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 54.125, "completions/mean_terminated_length": 54.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.38044621353037655, "epoch": 0.00066, "frac_reward_zero_std": 0.0, "grad_norm": 0.448842316865921, "kl": 1.51786033436656, "learning_rate": 9.142857142857144e-06, "loss": -0.0259, "num_tokens": 1361339.0, "reward": -0.06875000149011612, "reward_std": 0.25947943329811096, "rewards/rollout_reward_func/mean": -0.06875000149011612, "rewards/rollout_reward_func/std": 0.2822004556655884, "sampling/importance_sampling_ratio/max": 1.3879947662353516, "sampling/importance_sampling_ratio/mean": 0.7947115898132324, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2967371940612793, "sampling/sampling_logp_difference/mean": 0.22021695971488953, "step": 33, "step_time": 11.07244608599467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 48.21875, "completions/mean_terminated_length": 48.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.40916316490620375, "epoch": 0.00068, "frac_reward_zero_std": 0.0, "grad_norm": 1.1431283950805664, "kl": 8.728530923835933, "learning_rate": 9.42857142857143e-06, "loss": -0.0139, "num_tokens": 1402626.0, "reward": -0.0312500037252903, "reward_std": 0.30494827032089233, "rewards/rollout_reward_func/mean": -0.0312500037252903, "rewards/rollout_reward_func/std": 0.29885536432266235, "sampling/importance_sampling_ratio/max": 1.104201078414917, "sampling/importance_sampling_ratio/mean": 0.7145457863807678, "sampling/importance_sampling_ratio/min": 0.04647346958518028, "sampling/sampling_logp_difference/max": 3.0688798427581787, "sampling/sampling_logp_difference/mean": 0.2610732316970825, "step": 34, "step_time": 11.510182576999796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 112.125, "completions/mean_terminated_length": 112.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29634726722724736, "epoch": 0.0007, "frac_reward_zero_std": 0.0, "grad_norm": 0.19187694787979126, "kl": 1.5572640430182219, "learning_rate": 9.714285714285715e-06, "loss": -0.0254, "num_tokens": 1446561.0, "reward": -0.08437500149011612, "reward_std": 0.24383562803268433, "rewards/rollout_reward_func/mean": -0.08437500149011612, "rewards/rollout_reward_func/std": 0.24640561640262604, "sampling/importance_sampling_ratio/max": 2.9406237602233887, "sampling/importance_sampling_ratio/mean": 0.7193533182144165, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8099942207336426, "sampling/sampling_logp_difference/mean": 0.25988394021987915, "step": 35, "step_time": 11.142466192999564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 105.25, "completions/mean_terminated_length": 105.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18131516338326037, "epoch": 0.00072, "frac_reward_zero_std": 0.0, "grad_norm": 0.23722071945667267, "kl": 1.1602741237729788, "learning_rate": 1e-05, "loss": 0.0291, "num_tokens": 1488915.0, "reward": -0.02187500149011612, "reward_std": 0.27359336614608765, "rewards/rollout_reward_func/mean": -0.02187500149011612, "rewards/rollout_reward_func/std": 0.2756041884422302, "sampling/importance_sampling_ratio/max": 2.678105592727661, "sampling/importance_sampling_ratio/mean": 1.098912000656128, "sampling/importance_sampling_ratio/min": 0.13700050115585327, "sampling/sampling_logp_difference/max": 1.9879024028778076, "sampling/sampling_logp_difference/mean": 0.19873152673244476, "step": 36, "step_time": 11.944209822995617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 146.0625, "completions/mean_terminated_length": 146.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20993901719339192, "epoch": 0.00074, "frac_reward_zero_std": 0.0, "grad_norm": 0.578970193862915, "kl": 2.258807346224785, "learning_rate": 9.999975267482496e-06, "loss": -0.0125, "num_tokens": 1533393.0, "reward": -0.02499999850988388, "reward_std": 0.25581634044647217, "rewards/rollout_reward_func/mean": -0.02499999850988388, "rewards/rollout_reward_func/std": 0.262739896774292, "sampling/importance_sampling_ratio/max": 2.394359827041626, "sampling/importance_sampling_ratio/mean": 1.23587965965271, "sampling/importance_sampling_ratio/min": 0.012298667803406715, "sampling/sampling_logp_difference/max": 2.789980411529541, "sampling/sampling_logp_difference/mean": 0.2613030970096588, "step": 37, "step_time": 11.154853187999834 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.03125, "clip_ratio/region_mean": 0.0625, "completions/clipped_ratio": 0.03125, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 116.625, "completions/mean_terminated_length": 119.87096405029297, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2877486627548933, "epoch": 0.00076, "frac_reward_zero_std": 0.0, "grad_norm": 0.44263288378715515, "kl": 3.1276672892272472, "learning_rate": 9.99990107025622e-06, "loss": 0.0108, "num_tokens": 1577340.0, "reward": -0.10625000298023224, "reward_std": 0.3224383592605591, "rewards/rollout_reward_func/mean": -0.10625000298023224, "rewards/rollout_reward_func/std": 0.31514206528663635, "sampling/importance_sampling_ratio/max": 2.0693089962005615, "sampling/importance_sampling_ratio/mean": 0.8069508075714111, "sampling/importance_sampling_ratio/min": 0.08802572637796402, "sampling/sampling_logp_difference/max": 2.430077075958252, "sampling/sampling_logp_difference/mean": 0.35225343704223633, "step": 38, "step_time": 11.100302551001732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 501.0, "completions/max_terminated_length": 501.0, "completions/mean_length": 81.28125, "completions/mean_terminated_length": 81.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19899273430928588, "epoch": 0.00078, "frac_reward_zero_std": 0.0, "grad_norm": 0.2870699465274811, "kl": 2.173931872472167, "learning_rate": 9.99977740929988e-06, "loss": -0.0119, "num_tokens": 1618797.0, "reward": -0.00624999962747097, "reward_std": 0.2953369617462158, "rewards/rollout_reward_func/mean": -0.00624999962747097, "rewards/rollout_reward_func/std": 0.2906472086906433, "sampling/importance_sampling_ratio/max": 1.8685941696166992, "sampling/importance_sampling_ratio/mean": 0.7884964942932129, "sampling/importance_sampling_ratio/min": 0.10528180748224258, "sampling/sampling_logp_difference/max": 2.25122332572937, "sampling/sampling_logp_difference/mean": 0.24677368998527527, "step": 39, "step_time": 12.535899386997698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 92.59375, "completions/mean_terminated_length": 92.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16962543758563697, "epoch": 0.0008, "frac_reward_zero_std": 0.0, "grad_norm": 0.2785174548625946, "kl": 4.552771881222725, "learning_rate": 9.999604286244655e-06, "loss": -0.007, "num_tokens": 1661274.0, "reward": -0.11874999105930328, "reward_std": 0.2676457166671753, "rewards/rollout_reward_func/mean": -0.11874999105930328, "rewards/rollout_reward_func/std": 0.25832900404930115, "sampling/importance_sampling_ratio/max": 1.7765941619873047, "sampling/importance_sampling_ratio/mean": 0.8383759260177612, "sampling/importance_sampling_ratio/min": 0.08936071395874023, "sampling/sampling_logp_difference/max": 2.4148213863372803, "sampling/sampling_logp_difference/mean": 0.29100096225738525, "step": 40, "step_time": 11.163051270999858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 173.65625, "completions/mean_terminated_length": 173.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15532990638166666, "epoch": 0.00082, "frac_reward_zero_std": 0.0, "grad_norm": 0.29483646154403687, "kl": 3.5289941169321537, "learning_rate": 9.999381703374151e-06, "loss": 0.0228, "num_tokens": 1706080.0, "reward": -0.05937500298023224, "reward_std": 0.2836533188819885, "rewards/rollout_reward_func/mean": -0.05937500298023224, "rewards/rollout_reward_func/std": 0.29277247190475464, "sampling/importance_sampling_ratio/max": 1.8405553102493286, "sampling/importance_sampling_ratio/mean": 1.156214952468872, "sampling/importance_sampling_ratio/min": 0.10801295191049576, "sampling/sampling_logp_difference/max": 2.2252731323242188, "sampling/sampling_logp_difference/mean": 0.18507401645183563, "step": 41, "step_time": 11.33675845900143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 74.0625, "completions/mean_terminated_length": 74.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11048455070704222, "epoch": 0.00084, "frac_reward_zero_std": 0.0, "grad_norm": 0.18113428354263306, "kl": 1.7759376261383295, "learning_rate": 9.999109663624387e-06, "loss": 0.0205, "num_tokens": 1744869.0, "reward": 0.08750000596046448, "reward_std": 0.28894108533859253, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.28026482462882996, "sampling/importance_sampling_ratio/max": 1.620446801185608, "sampling/importance_sampling_ratio/mean": 0.9351440668106079, "sampling/importance_sampling_ratio/min": 0.10050303488969803, "sampling/sampling_logp_difference/max": 2.297565460205078, "sampling/sampling_logp_difference/mean": 0.16802553832530975, "step": 42, "step_time": 12.940062022998973 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 158.75, "completions/mean_terminated_length": 158.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16554167680442333, "epoch": 0.00086, "frac_reward_zero_std": 0.0, "grad_norm": 0.23327907919883728, "kl": 2.48025418817997, "learning_rate": 9.99878817058375e-06, "loss": -0.0399, "num_tokens": 1789949.0, "reward": -0.0625, "reward_std": 0.23899509012699127, "rewards/rollout_reward_func/mean": -0.0625, "rewards/rollout_reward_func/std": 0.2511264979839325, "sampling/importance_sampling_ratio/max": 1.6884547472000122, "sampling/importance_sampling_ratio/mean": 0.9956019520759583, "sampling/importance_sampling_ratio/min": 0.10622864216566086, "sampling/sampling_logp_difference/max": 2.2420148849487305, "sampling/sampling_logp_difference/mean": 0.18983688950538635, "step": 43, "step_time": 11.24787668800127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 159.71875, "completions/mean_terminated_length": 159.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22343985363841057, "epoch": 0.00088, "frac_reward_zero_std": 0.0, "grad_norm": 0.14092789590358734, "kl": 2.0855566300451756, "learning_rate": 9.998417228492952e-06, "loss": 0.004, "num_tokens": 1835607.0, "reward": -0.09999999403953552, "reward_std": 0.28680914640426636, "rewards/rollout_reward_func/mean": -0.09999999403953552, "rewards/rollout_reward_func/std": 0.27474328875541687, "sampling/importance_sampling_ratio/max": 1.7114570140838623, "sampling/importance_sampling_ratio/mean": 0.9638450145721436, "sampling/importance_sampling_ratio/min": 0.11154000461101532, "sampling/sampling_logp_difference/max": 2.1932966709136963, "sampling/sampling_logp_difference/mean": 0.23201864957809448, "step": 44, "step_time": 11.311454336995666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 146.09375, "completions/mean_terminated_length": 146.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23337038466706872, "epoch": 0.0009, "frac_reward_zero_std": 0.0, "grad_norm": 0.3899145722389221, "kl": 4.845795013010502, "learning_rate": 9.99799684224497e-06, "loss": 0.0247, "num_tokens": 1879771.0, "reward": -0.0031249970197677612, "reward_std": 0.25569599866867065, "rewards/rollout_reward_func/mean": -0.0031249970197677612, "rewards/rollout_reward_func/std": 0.2705840468406677, "sampling/importance_sampling_ratio/max": 1.7009395360946655, "sampling/importance_sampling_ratio/mean": 0.95084148645401, "sampling/importance_sampling_ratio/min": 0.10427333414554596, "sampling/sampling_logp_difference/max": 2.260716438293457, "sampling/sampling_logp_difference/mean": 0.25133082270622253, "step": 45, "step_time": 12.474208057998112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 114.09375, "completions/mean_terminated_length": 114.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2298557860776782, "epoch": 0.00092, "frac_reward_zero_std": 0.0, "grad_norm": 0.4556872844696045, "kl": 4.5213728453963995, "learning_rate": 9.997527017384983e-06, "loss": 0.0007, "num_tokens": 1922584.0, "reward": -0.07499999552965164, "reward_std": 0.27271273732185364, "rewards/rollout_reward_func/mean": -0.07499999552965164, "rewards/rollout_reward_func/std": 0.27474328875541687, "sampling/importance_sampling_ratio/max": 1.6783581972122192, "sampling/importance_sampling_ratio/mean": 0.7982770204544067, "sampling/importance_sampling_ratio/min": 0.08830086886882782, "sampling/sampling_logp_difference/max": 2.027076005935669, "sampling/sampling_logp_difference/mean": 0.2809655964374542, "step": 46, "step_time": 10.93607420299304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 124.34375, "completions/mean_terminated_length": 124.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2073972539510578, "epoch": 0.00094, "frac_reward_zero_std": 0.0, "grad_norm": 0.17832288146018982, "kl": 1.0834360159933567, "learning_rate": 9.997007760110298e-06, "loss": 0.0234, "num_tokens": 1964721.0, "reward": 0.012500006705522537, "reward_std": 0.2522428631782532, "rewards/rollout_reward_func/mean": 0.012500006705522537, "rewards/rollout_reward_func/std": 0.27090054750442505, "sampling/importance_sampling_ratio/max": 1.771154522895813, "sampling/importance_sampling_ratio/mean": 1.1139253377914429, "sampling/importance_sampling_ratio/min": 0.1874149888753891, "sampling/sampling_logp_difference/max": 1.6743755340576172, "sampling/sampling_logp_difference/mean": 0.1026790589094162, "step": 47, "step_time": 11.130758332001278 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 147.78125, "completions/mean_terminated_length": 147.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24872009828686714, "epoch": 0.00096, "frac_reward_zero_std": 0.0, "grad_norm": 0.14451554417610168, "kl": 1.0314295943826437, "learning_rate": 9.996439077270269e-06, "loss": 0.001, "num_tokens": 2008685.0, "reward": -0.03437500074505806, "reward_std": 0.23380109667778015, "rewards/rollout_reward_func/mean": -0.03437500074505806, "rewards/rollout_reward_func/std": 0.23086006939411163, "sampling/importance_sampling_ratio/max": 1.7651574611663818, "sampling/importance_sampling_ratio/mean": 1.1108081340789795, "sampling/importance_sampling_ratio/min": 0.22357527911663055, "sampling/sampling_logp_difference/max": 1.4978723526000977, "sampling/sampling_logp_difference/mean": 0.12039029598236084, "step": 48, "step_time": 12.291172437004207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 137.6875, "completions/mean_terminated_length": 137.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.30497108632698655, "epoch": 0.00098, "frac_reward_zero_std": 0.0, "grad_norm": 0.3902129530906677, "kl": 1.3539033886045218, "learning_rate": 9.995820976366208e-06, "loss": 0.0176, "num_tokens": 2053662.0, "reward": -0.11562500149011612, "reward_std": 0.2872692942619324, "rewards/rollout_reward_func/mean": -0.11562500149011612, "rewards/rollout_reward_func/std": 0.2863669693470001, "sampling/importance_sampling_ratio/max": 1.727237582206726, "sampling/importance_sampling_ratio/mean": 0.9422577619552612, "sampling/importance_sampling_ratio/min": 0.19504192471504211, "sampling/sampling_logp_difference/max": 1.634671688079834, "sampling/sampling_logp_difference/mean": 0.16423432528972626, "step": 49, "step_time": 11.050756222004566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 165.78125, "completions/mean_terminated_length": 165.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3961345008574426, "epoch": 0.001, "frac_reward_zero_std": 0.0, "grad_norm": 0.35966381430625916, "kl": 1.387764971703291, "learning_rate": 9.995153465551283e-06, "loss": 0.0089, "num_tokens": 2098699.0, "reward": -0.07499999552965164, "reward_std": 0.2959960997104645, "rewards/rollout_reward_func/mean": -0.07499999552965164, "rewards/rollout_reward_func/std": 0.29512161016464233, "sampling/importance_sampling_ratio/max": 1.8711365461349487, "sampling/importance_sampling_ratio/mean": 1.0243897438049316, "sampling/importance_sampling_ratio/min": 0.23714116215705872, "sampling/sampling_logp_difference/max": 1.4389008283615112, "sampling/sampling_logp_difference/mean": 0.18079081177711487, "step": 50, "step_time": 12.071326983996187 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 528.0, "completions/max_terminated_length": 528.0, "completions/mean_length": 108.59375, "completions/mean_terminated_length": 108.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3933054287917912, "epoch": 0.00102, "frac_reward_zero_std": 0.0, "grad_norm": 0.2603336274623871, "kl": 1.937150089070201, "learning_rate": 9.994436553630411e-06, "loss": -0.0064, "num_tokens": 2141175.0, "reward": 0.003125001210719347, "reward_std": 0.2718885540962219, "rewards/rollout_reward_func/mean": 0.003125001210719347, "rewards/rollout_reward_func/std": 0.260872483253479, "sampling/importance_sampling_ratio/max": 1.8100311756134033, "sampling/importance_sampling_ratio/mean": 0.9598448276519775, "sampling/importance_sampling_ratio/min": 0.2854175865650177, "sampling/sampling_logp_difference/max": 1.2537529468536377, "sampling/sampling_logp_difference/mean": 0.14605772495269775, "step": 51, "step_time": 11.224896592000732 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 109.875, "completions/mean_terminated_length": 109.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4408538439311087, "epoch": 0.00104, "frac_reward_zero_std": 0.0, "grad_norm": 0.26130643486976624, "kl": 1.0319377183914185, "learning_rate": 9.993670250060152e-06, "loss": 0.0136, "num_tokens": 2183782.0, "reward": -0.01875000074505806, "reward_std": 0.28954020142555237, "rewards/rollout_reward_func/mean": -0.01875000074505806, "rewards/rollout_reward_func/std": 0.2822004556655884, "sampling/importance_sampling_ratio/max": 1.839620590209961, "sampling/importance_sampling_ratio/mean": 1.0097739696502686, "sampling/importance_sampling_ratio/min": 0.30520883202552795, "sampling/sampling_logp_difference/max": 1.186706304550171, "sampling/sampling_logp_difference/mean": 0.14148648083209991, "step": 52, "step_time": 11.18042464899736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 125.1875, "completions/mean_terminated_length": 125.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5513601321727037, "epoch": 0.00106, "frac_reward_zero_std": 0.0, "grad_norm": 0.4864451587200165, "kl": 1.3276541624218225, "learning_rate": 9.99285456494856e-06, "loss": 0.0141, "num_tokens": 2228172.0, "reward": -0.16249999403953552, "reward_std": 0.2478259801864624, "rewards/rollout_reward_func/mean": -0.16249999403953552, "rewards/rollout_reward_func/std": 0.2574690878391266, "sampling/importance_sampling_ratio/max": 1.9777826070785522, "sampling/importance_sampling_ratio/mean": 0.9027001857757568, "sampling/importance_sampling_ratio/min": 0.14299145340919495, "sampling/sampling_logp_difference/max": 1.1306499242782593, "sampling/sampling_logp_difference/mean": 0.18693894147872925, "step": 53, "step_time": 12.086175061995164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 74.78125, "completions/mean_terminated_length": 76.67741394042969, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.419326551258564, "epoch": 0.00108, "frac_reward_zero_std": 0.0, "grad_norm": 0.4347839653491974, "kl": 3.010229142382741, "learning_rate": 9.991989509055075e-06, "loss": 0.0223, "num_tokens": 2267664.0, "reward": 0.02500000223517418, "reward_std": 0.2931596636772156, "rewards/rollout_reward_func/mean": 0.02500000223517418, "rewards/rollout_reward_func/std": 0.2839809060096741, "sampling/importance_sampling_ratio/max": 1.7290469408035278, "sampling/importance_sampling_ratio/mean": 0.9977864027023315, "sampling/importance_sampling_ratio/min": 0.26406097412109375, "sampling/sampling_logp_difference/max": 1.3314881324768066, "sampling/sampling_logp_difference/mean": 0.0983460545539856, "step": 54, "step_time": 12.054193198997382 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 199.0625, "completions/mean_terminated_length": 199.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5275807566940784, "epoch": 0.0011, "frac_reward_zero_std": 0.0, "grad_norm": 0.3840061128139496, "kl": 1.3732723910361528, "learning_rate": 9.991075093790372e-06, "loss": 0.0204, "num_tokens": 2314085.0, "reward": -0.06875000149011612, "reward_std": 0.28039801120758057, "rewards/rollout_reward_func/mean": -0.06875000149011612, "rewards/rollout_reward_func/std": 0.27642592787742615, "sampling/importance_sampling_ratio/max": 1.9573321342468262, "sampling/importance_sampling_ratio/mean": 1.1044135093688965, "sampling/importance_sampling_ratio/min": 0.2842760980129242, "sampling/sampling_logp_difference/max": 1.102674961090088, "sampling/sampling_logp_difference/mean": 0.11034712195396423, "step": 55, "step_time": 11.333799679998265 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 119.9375, "completions/mean_terminated_length": 119.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5743398573249578, "epoch": 0.00112, "frac_reward_zero_std": 0.0, "grad_norm": 0.5087872743606567, "kl": 1.4628004813566804, "learning_rate": 9.990111331216204e-06, "loss": 0.0017, "num_tokens": 2357595.0, "reward": -0.02812500111758709, "reward_std": 0.27700191736221313, "rewards/rollout_reward_func/mean": -0.02812500111758709, "rewards/rollout_reward_func/std": 0.2842470109462738, "sampling/importance_sampling_ratio/max": 1.913257360458374, "sampling/importance_sampling_ratio/mean": 1.05607271194458, "sampling/importance_sampling_ratio/min": 0.1461808979511261, "sampling/sampling_logp_difference/max": 1.603846788406372, "sampling/sampling_logp_difference/mean": 0.1286735087633133, "step": 56, "step_time": 11.854098038002121 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 135.4375, "completions/mean_terminated_length": 135.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7281919755041599, "epoch": 0.00114, "frac_reward_zero_std": 0.0, "grad_norm": 0.7088514566421509, "kl": 0.8849395364522934, "learning_rate": 9.989098234045254e-06, "loss": -0.0444, "num_tokens": 2402066.0, "reward": -0.0156250037252903, "reward_std": 0.25878578424453735, "rewards/rollout_reward_func/mean": -0.0156250037252903, "rewards/rollout_reward_func/std": 0.27604275941848755, "sampling/importance_sampling_ratio/max": 1.6577712297439575, "sampling/importance_sampling_ratio/mean": 0.9105332493782043, "sampling/importance_sampling_ratio/min": 0.3801342248916626, "sampling/sampling_logp_difference/max": 0.9672622680664062, "sampling/sampling_logp_difference/mean": 0.1324087381362915, "step": 57, "step_time": 11.037133848001758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 156.65625, "completions/mean_terminated_length": 156.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6395923467352986, "epoch": 0.00116, "frac_reward_zero_std": 0.0, "grad_norm": 1.195318341255188, "kl": 3.836987042799592, "learning_rate": 9.988035815640953e-06, "loss": -0.0028, "num_tokens": 2446922.0, "reward": -0.08125000447034836, "reward_std": 0.30148592591285706, "rewards/rollout_reward_func/mean": -0.08125000447034836, "rewards/rollout_reward_func/std": 0.29009175300598145, "sampling/importance_sampling_ratio/max": 1.5143545866012573, "sampling/importance_sampling_ratio/mean": 0.9916036128997803, "sampling/importance_sampling_ratio/min": 0.2861623466014862, "sampling/sampling_logp_difference/max": 1.6131682395935059, "sampling/sampling_logp_difference/mean": 0.10863298177719116, "step": 58, "step_time": 11.226664856998468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 228.375, "completions/mean_terminated_length": 228.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.644195169210434, "epoch": 0.00118, "frac_reward_zero_std": 0.0, "grad_norm": 0.9292266964912415, "kl": 0.3817840525880456, "learning_rate": 9.986924090017322e-06, "loss": 0.0188, "num_tokens": 2495754.0, "reward": -0.078125, "reward_std": 0.29619455337524414, "rewards/rollout_reward_func/mean": -0.078125, "rewards/rollout_reward_func/std": 0.2904216945171356, "sampling/importance_sampling_ratio/max": 1.6704291105270386, "sampling/importance_sampling_ratio/mean": 1.1021101474761963, "sampling/importance_sampling_ratio/min": 0.7147196531295776, "sampling/sampling_logp_difference/max": 0.5786266326904297, "sampling/sampling_logp_difference/mean": 0.05197087675333023, "step": 59, "step_time": 12.925712045996988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 136.0625, "completions/mean_terminated_length": 136.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6203140085563064, "epoch": 0.0012, "frac_reward_zero_std": 0.0, "grad_norm": 1.10093355178833, "kl": 1.1470853416249156, "learning_rate": 9.985763071838774e-06, "loss": -0.0307, "num_tokens": 2539131.0, "reward": -0.0031249988824129105, "reward_std": 0.28817176818847656, "rewards/rollout_reward_func/mean": -0.0031249988824129105, "rewards/rollout_reward_func/std": 0.29125356674194336, "sampling/importance_sampling_ratio/max": 2.2410364151000977, "sampling/importance_sampling_ratio/mean": 1.0475972890853882, "sampling/importance_sampling_ratio/min": 0.2532808780670166, "sampling/sampling_logp_difference/max": 1.2886860370635986, "sampling/sampling_logp_difference/mean": 0.10613550990819931, "step": 60, "step_time": 11.279086669002936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 154.96875, "completions/mean_terminated_length": 154.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7541638985276222, "epoch": 0.00122, "frac_reward_zero_std": 0.0, "grad_norm": 0.4667148292064667, "kl": 1.069685067050159, "learning_rate": 9.98455277641992e-06, "loss": -0.0064, "num_tokens": 2583754.0, "reward": -0.06875000894069672, "reward_std": 0.30232733488082886, "rewards/rollout_reward_func/mean": -0.06875000894069672, "rewards/rollout_reward_func/std": 0.2966887354850769, "sampling/importance_sampling_ratio/max": 2.0150234699249268, "sampling/importance_sampling_ratio/mean": 0.9742628335952759, "sampling/importance_sampling_ratio/min": 0.2607097625732422, "sampling/sampling_logp_difference/max": 1.481550693511963, "sampling/sampling_logp_difference/mean": 0.10388601571321487, "step": 61, "step_time": 11.492867544993715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 127.625, "completions/mean_terminated_length": 127.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.634353382512927, "epoch": 0.00124, "frac_reward_zero_std": 0.0, "grad_norm": 0.40648922324180603, "kl": 2.1665764497593045, "learning_rate": 9.983293219725379e-06, "loss": -0.0134, "num_tokens": 2627112.0, "reward": -0.04374999552965164, "reward_std": 0.30804431438446045, "rewards/rollout_reward_func/mean": -0.04374999552965164, "rewards/rollout_reward_func/std": 0.3015417456626892, "sampling/importance_sampling_ratio/max": 1.6427249908447266, "sampling/importance_sampling_ratio/mean": 0.8530735373497009, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4362597465515137, "sampling/sampling_logp_difference/mean": 0.14343991875648499, "step": 62, "step_time": 11.440008787005354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 192.625, "completions/mean_terminated_length": 192.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.708164919167757, "epoch": 0.00126, "frac_reward_zero_std": 0.0, "grad_norm": 1.5144699811935425, "kl": 0.8892286708578467, "learning_rate": 9.981984418369552e-06, "loss": -0.0488, "num_tokens": 2673647.0, "reward": -0.08437500894069672, "reward_std": 0.3194201588630676, "rewards/rollout_reward_func/mean": -0.08437500894069672, "rewards/rollout_reward_func/std": 0.3111989200115204, "sampling/importance_sampling_ratio/max": 2.3106741905212402, "sampling/importance_sampling_ratio/mean": 0.9929068684577942, "sampling/importance_sampling_ratio/min": 0.3451598882675171, "sampling/sampling_logp_difference/max": 1.0636367797851562, "sampling/sampling_logp_difference/mean": 0.08631584048271179, "step": 63, "step_time": 11.237050638997971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 175.3125, "completions/mean_terminated_length": 175.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6482668500393629, "epoch": 0.00128, "frac_reward_zero_std": 0.0, "grad_norm": 0.4774710536003113, "kl": 1.530721453949809, "learning_rate": 9.980626389616414e-06, "loss": -0.0094, "num_tokens": 2718479.0, "reward": -0.00937499850988388, "reward_std": 0.30746251344680786, "rewards/rollout_reward_func/mean": -0.00937499850988388, "rewards/rollout_reward_func/std": 0.31659412384033203, "sampling/importance_sampling_ratio/max": 1.796738624572754, "sampling/importance_sampling_ratio/mean": 0.9007509350776672, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.881629467010498, "sampling/sampling_logp_difference/mean": 0.1548471748828888, "step": 64, "step_time": 12.386156486998516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 209.96875, "completions/mean_terminated_length": 209.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.618705628439784, "epoch": 0.0013, "frac_reward_zero_std": 0.0, "grad_norm": 0.5894741415977478, "kl": 0.666377043351531, "learning_rate": 9.979219151379285e-06, "loss": -0.0184, "num_tokens": 2766086.0, "reward": -0.04374999925494194, "reward_std": 0.2869294583797455, "rewards/rollout_reward_func/mean": -0.04374999925494194, "rewards/rollout_reward_func/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.8739320039749146, "sampling/importance_sampling_ratio/mean": 0.9675271511077881, "sampling/importance_sampling_ratio/min": 0.20434865355491638, "sampling/sampling_logp_difference/max": 1.1845147609710693, "sampling/sampling_logp_difference/mean": 0.09806595742702484, "step": 65, "step_time": 11.926742346999163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 213.21875, "completions/mean_terminated_length": 213.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5620344541966915, "epoch": 0.00132, "frac_reward_zero_std": 0.0, "grad_norm": 0.6312305927276611, "kl": 0.762151756323874, "learning_rate": 9.977762722220585e-06, "loss": 0.0008, "num_tokens": 2812964.0, "reward": 0.0031250007450580597, "reward_std": 0.25280633568763733, "rewards/rollout_reward_func/mean": 0.0031250007450580597, "rewards/rollout_reward_func/std": 0.25963297486305237, "sampling/importance_sampling_ratio/max": 2.133820056915283, "sampling/importance_sampling_ratio/mean": 1.0070066452026367, "sampling/importance_sampling_ratio/min": 0.2440081536769867, "sampling/sampling_logp_difference/max": 1.0896363258361816, "sampling/sampling_logp_difference/mean": 0.1211574450135231, "step": 66, "step_time": 11.291782136999245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 169.53125, "completions/mean_terminated_length": 169.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.46685375832021236, "epoch": 0.00134, "frac_reward_zero_std": 0.0, "grad_norm": 1.0418833494186401, "kl": 2.352157788351178, "learning_rate": 9.976257121351602e-06, "loss": -0.0368, "num_tokens": 2856750.0, "reward": -0.012500000186264515, "reward_std": 0.3025353252887726, "rewards/rollout_reward_func/mean": -0.012500000186264515, "rewards/rollout_reward_func/std": 0.2959402799606323, "sampling/importance_sampling_ratio/max": 2.5504872798919678, "sampling/importance_sampling_ratio/mean": 0.9419388771057129, "sampling/importance_sampling_ratio/min": 0.2725259065628052, "sampling/sampling_logp_difference/max": 1.2833704948425293, "sampling/sampling_logp_difference/mean": 0.18767249584197998, "step": 67, "step_time": 11.377942251001514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.01171875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01171875, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 180.78125, "completions/mean_terminated_length": 180.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.436415022239089, "epoch": 0.00136, "frac_reward_zero_std": 0.0, "grad_norm": 1.207202672958374, "kl": 4.3735333532094955, "learning_rate": 9.974702368632228e-06, "loss": 0.0274, "num_tokens": 2901179.0, "reward": 4.6566128730773926e-09, "reward_std": 0.2716658115386963, "rewards/rollout_reward_func/mean": 4.6566128730773926e-09, "rewards/rollout_reward_func/std": 0.3121207356452942, "sampling/importance_sampling_ratio/max": 2.8554563522338867, "sampling/importance_sampling_ratio/mean": 0.8897923231124878, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8979930877685547, "sampling/sampling_logp_difference/mean": 0.19784726202487946, "step": 68, "step_time": 11.946266881996053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 656.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 235.84375, "completions/mean_terminated_length": 235.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5173613112419844, "epoch": 0.00138, "frac_reward_zero_std": 0.0, "grad_norm": 0.8966071605682373, "kl": 2.9685468580573797, "learning_rate": 9.973098484570702e-06, "loss": 0.0057, "num_tokens": 2948718.0, "reward": -0.06875000149011612, "reward_std": 0.3181586265563965, "rewards/rollout_reward_func/mean": -0.06875000149011612, "rewards/rollout_reward_func/std": 0.30314216017723083, "sampling/importance_sampling_ratio/max": 2.296177864074707, "sampling/importance_sampling_ratio/mean": 0.8117730617523193, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2387843132019043, "sampling/sampling_logp_difference/mean": 0.2252735197544098, "step": 69, "step_time": 11.377752644999418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 665.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 262.5, "completions/mean_terminated_length": 262.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5308686569333076, "epoch": 0.0014, "frac_reward_zero_std": 0.0, "grad_norm": 0.778303861618042, "kl": 2.2760814502835274, "learning_rate": 9.97144549032334e-06, "loss": 0.0046, "num_tokens": 2997371.0, "reward": -0.009374994784593582, "reward_std": 0.2856472432613373, "rewards/rollout_reward_func/mean": -0.009374994784593582, "rewards/rollout_reward_func/std": 0.29551416635513306, "sampling/importance_sampling_ratio/max": 2.271819591522217, "sampling/importance_sampling_ratio/mean": 1.1214137077331543, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2301709651947021, "sampling/sampling_logp_difference/mean": 0.15493647754192352, "step": 70, "step_time": 11.839647161999892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 532.0, "completions/max_terminated_length": 532.0, "completions/mean_length": 155.5625, "completions/mean_terminated_length": 155.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.41631849482655525, "epoch": 0.00142, "frac_reward_zero_std": 0.0, "grad_norm": 0.8966524004936218, "kl": 1.5040771141648293, "learning_rate": 9.969743407694254e-06, "loss": 0.0091, "num_tokens": 3041215.0, "reward": 0.10625000298023224, "reward_std": 0.21471109986305237, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.2169249951839447, "sampling/importance_sampling_ratio/max": 1.7940783500671387, "sampling/importance_sampling_ratio/mean": 0.9938610792160034, "sampling/importance_sampling_ratio/min": 0.3492223918437958, "sampling/sampling_logp_difference/max": 1.0574288368225098, "sampling/sampling_logp_difference/mean": 0.1216852217912674, "step": 71, "step_time": 11.553316193001592 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 665.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 267.5625, "completions/mean_terminated_length": 267.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4987291041761637, "epoch": 0.00144, "frac_reward_zero_std": 0.0, "grad_norm": 0.7415627837181091, "kl": 1.1952915536239743, "learning_rate": 9.967992259135063e-06, "loss": 0.0087, "num_tokens": 3090071.0, "reward": 0.03437500074505806, "reward_std": 0.2964325249195099, "rewards/rollout_reward_func/mean": 0.03437500074505806, "rewards/rollout_reward_func/std": 0.3022596538066864, "sampling/importance_sampling_ratio/max": 1.6471632719039917, "sampling/importance_sampling_ratio/mean": 0.8802411556243896, "sampling/importance_sampling_ratio/min": 0.24935072660446167, "sampling/sampling_logp_difference/max": 1.388807773590088, "sampling/sampling_logp_difference/mean": 0.16446097195148468, "step": 72, "step_time": 11.446644291005214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 242.375, "completions/mean_terminated_length": 242.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4668980687856674, "epoch": 0.00146, "frac_reward_zero_std": 0.0, "grad_norm": 1.1040256023406982, "kl": 1.70635586977005, "learning_rate": 9.9661920677446e-06, "loss": 0.0145, "num_tokens": 3137094.0, "reward": 0.05937500298023224, "reward_std": 0.27639612555503845, "rewards/rollout_reward_func/mean": 0.05937500298023224, "rewards/rollout_reward_func/std": 0.27923786640167236, "sampling/importance_sampling_ratio/max": 1.8849077224731445, "sampling/importance_sampling_ratio/mean": 0.9049147367477417, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3857386112213135, "sampling/sampling_logp_difference/mean": 0.1348995864391327, "step": 73, "step_time": 12.234138552996228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 299.5, "completions/mean_terminated_length": 299.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5109915547072887, "epoch": 0.00148, "frac_reward_zero_std": 0.0, "grad_norm": 0.7389334440231323, "kl": 1.2791589684784412, "learning_rate": 9.964342857268609e-06, "loss": -0.0176, "num_tokens": 3187025.0, "reward": -0.05000000447034836, "reward_std": 0.3144170045852661, "rewards/rollout_reward_func/mean": -0.05000000447034836, "rewards/rollout_reward_func/std": 0.3192733824253082, "sampling/importance_sampling_ratio/max": 2.0568928718566895, "sampling/importance_sampling_ratio/mean": 0.9717973470687866, "sampling/importance_sampling_ratio/min": 0.17520615458488464, "sampling/sampling_logp_difference/max": 1.3764185905456543, "sampling/sampling_logp_difference/mean": 0.16508722305297852, "step": 74, "step_time": 11.555549031007104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 253.375, "completions/mean_terminated_length": 253.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4376304280012846, "epoch": 0.0015, "frac_reward_zero_std": 0.0, "grad_norm": 1.080005168914795, "kl": 0.7355061583220959, "learning_rate": 9.962444652099422e-06, "loss": 0.0362, "num_tokens": 3235734.0, "reward": 0.05937499552965164, "reward_std": 0.2729761600494385, "rewards/rollout_reward_func/mean": 0.05937499552965164, "rewards/rollout_reward_func/std": 0.2722183167934418, "sampling/importance_sampling_ratio/max": 1.4768556356430054, "sampling/importance_sampling_ratio/mean": 0.9129464626312256, "sampling/importance_sampling_ratio/min": 0.24038104712963104, "sampling/sampling_logp_difference/max": 1.5735561847686768, "sampling/sampling_logp_difference/mean": 0.12041331827640533, "step": 75, "step_time": 11.882746192999548 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.03125, "completions/max_length": 686.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 240.84375, "completions/mean_terminated_length": 226.48387145996094, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4580497033894062, "epoch": 0.00152, "frac_reward_zero_std": 0.0, "grad_norm": 0.9605908393859863, "kl": 0.7944184886291623, "learning_rate": 9.960497477275653e-06, "loss": 0.0318, "num_tokens": 3283694.0, "reward": 0.04062500223517418, "reward_std": 0.2651963233947754, "rewards/rollout_reward_func/mean": 0.04062500223517418, "rewards/rollout_reward_func/std": 0.26379287242889404, "sampling/importance_sampling_ratio/max": 1.7033323049545288, "sampling/importance_sampling_ratio/mean": 0.9714037775993347, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.104940414428711, "sampling/sampling_logp_difference/mean": 0.08820510655641556, "step": 76, "step_time": 12.523160294003901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02566964295692742, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02566964295692742, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 263.6875, "completions/mean_terminated_length": 263.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4300597347319126, "epoch": 0.00154, "frac_reward_zero_std": 0.0, "grad_norm": 1.6805750131607056, "kl": 1.3630296420305967, "learning_rate": 9.95850135848185e-06, "loss": -0.0421, "num_tokens": 3331912.0, "reward": 0.009375004097819328, "reward_std": 0.30763936042785645, "rewards/rollout_reward_func/mean": 0.009375004097819328, "rewards/rollout_reward_func/std": 0.3196362555027008, "sampling/importance_sampling_ratio/max": 2.7178947925567627, "sampling/importance_sampling_ratio/mean": 1.0489997863769531, "sampling/importance_sampling_ratio/min": 0.26937347650527954, "sampling/sampling_logp_difference/max": 1.4588689804077148, "sampling/sampling_logp_difference/mean": 0.10825955867767334, "step": 77, "step_time": 11.788244641005804 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.01302083395421505, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01302083395421505, "completions/clipped_ratio": 0.0, "completions/max_length": 656.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 269.5625, "completions/mean_terminated_length": 269.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.42625460866838694, "epoch": 0.00156, "frac_reward_zero_std": 0.0, "grad_norm": 1.4374905824661255, "kl": 0.7787520494312048, "learning_rate": 9.956456322048174e-06, "loss": 0.011, "num_tokens": 3380825.0, "reward": 0.02187500149011612, "reward_std": 0.29394739866256714, "rewards/rollout_reward_func/mean": 0.02187500149011612, "rewards/rollout_reward_func/std": 0.2915303111076355, "sampling/importance_sampling_ratio/max": 2.1008663177490234, "sampling/importance_sampling_ratio/mean": 1.0042246580123901, "sampling/importance_sampling_ratio/min": 0.4788663983345032, "sampling/sampling_logp_difference/max": 0.9390687942504883, "sampling/sampling_logp_difference/mean": 0.09266875684261322, "step": 78, "step_time": 11.962713395994797 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 273.8125, "completions/mean_terminated_length": 273.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4502659887075424, "epoch": 0.00158, "frac_reward_zero_std": 0.0, "grad_norm": 0.9490029811859131, "kl": 1.5540950745344162, "learning_rate": 9.954362394950035e-06, "loss": 0.0191, "num_tokens": 3430227.0, "reward": -3.725290298461914e-09, "reward_std": 0.24442045390605927, "rewards/rollout_reward_func/mean": -3.725290298461914e-09, "rewards/rollout_reward_func/std": 0.26518407464027405, "sampling/importance_sampling_ratio/max": 2.0061187744140625, "sampling/importance_sampling_ratio/mean": 1.0645253658294678, "sampling/importance_sampling_ratio/min": 0.2577177882194519, "sampling/sampling_logp_difference/max": 1.6016087532043457, "sampling/sampling_logp_difference/mean": 0.10993731021881104, "step": 79, "step_time": 11.83600498099986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 170.71875, "completions/mean_terminated_length": 170.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37832839600741863, "epoch": 0.0016, "frac_reward_zero_std": 0.0, "grad_norm": 0.9014107584953308, "kl": 1.1778284218162298, "learning_rate": 9.952219604807746e-06, "loss": -0.0012, "num_tokens": 3474605.0, "reward": -0.02500000037252903, "reward_std": 0.3435736894607544, "rewards/rollout_reward_func/mean": -0.02500000037252903, "rewards/rollout_reward_func/std": 0.333118200302124, "sampling/importance_sampling_ratio/max": 1.6518880128860474, "sampling/importance_sampling_ratio/mean": 0.994209885597229, "sampling/importance_sampling_ratio/min": 0.13644254207611084, "sampling/sampling_logp_difference/max": 1.75632905960083, "sampling/sampling_logp_difference/mean": 0.09294585138559341, "step": 80, "step_time": 11.046795827991446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 183.5625, "completions/mean_terminated_length": 183.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36683487379923463, "epoch": 0.00162, "frac_reward_zero_std": 0.0, "grad_norm": 1.0819720029830933, "kl": 0.7554028183221817, "learning_rate": 9.950027979886159e-06, "loss": 0.0048, "num_tokens": 3519421.0, "reward": 0.046875, "reward_std": 0.226852148771286, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.25142738223075867, "sampling/importance_sampling_ratio/max": 1.9561083316802979, "sampling/importance_sampling_ratio/mean": 1.0627553462982178, "sampling/importance_sampling_ratio/min": 0.5150871872901917, "sampling/sampling_logp_difference/max": 0.6825180053710938, "sampling/sampling_logp_difference/mean": 0.06404304504394531, "step": 81, "step_time": 11.591076093998709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 190.65625, "completions/mean_terminated_length": 190.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.325567118357867, "epoch": 0.00164, "frac_reward_zero_std": 0.0, "grad_norm": 0.7309789657592773, "kl": 0.8548763953149319, "learning_rate": 9.947787549094288e-06, "loss": 0.0508, "num_tokens": 3563388.0, "reward": 0.15312498807907104, "reward_std": 0.21303477883338928, "rewards/rollout_reward_func/mean": 0.15312498807907104, "rewards/rollout_reward_func/std": 0.22857818007469177, "sampling/importance_sampling_ratio/max": 2.482966899871826, "sampling/importance_sampling_ratio/mean": 1.066469669342041, "sampling/importance_sampling_ratio/min": 0.34105008840560913, "sampling/sampling_logp_difference/max": 1.066892147064209, "sampling/sampling_logp_difference/mean": 0.07394260168075562, "step": 82, "step_time": 11.662379756005976 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 138.03125, "completions/mean_terminated_length": 138.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.39051534375175834, "epoch": 0.00166, "frac_reward_zero_std": 0.0, "grad_norm": 3.98054575920105, "kl": 20.298863353207707, "learning_rate": 9.945498341984929e-06, "loss": 0.0141, "num_tokens": 3606377.0, "reward": 0.009375001303851604, "reward_std": 0.28136491775512695, "rewards/rollout_reward_func/mean": 0.009375001303851604, "rewards/rollout_reward_func/std": 0.2751649022102356, "sampling/importance_sampling_ratio/max": 2.244656562805176, "sampling/importance_sampling_ratio/mean": 0.9521924257278442, "sampling/importance_sampling_ratio/min": 4.692416766260976e-08, "sampling/sampling_logp_difference/max": 16.929128646850586, "sampling/sampling_logp_difference/mean": 0.31568166613578796, "step": 83, "step_time": 11.113466541006346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 647.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 234.09375, "completions/mean_terminated_length": 234.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.46396899688988924, "epoch": 0.00168, "frac_reward_zero_std": 0.0, "grad_norm": 1.0432907342910767, "kl": 1.7000005040317774, "learning_rate": 9.943160388754274e-06, "loss": -0.0132, "num_tokens": 3654234.0, "reward": 0.078125, "reward_std": 0.26887503266334534, "rewards/rollout_reward_func/mean": 0.078125, "rewards/rollout_reward_func/std": 0.2870701551437378, "sampling/importance_sampling_ratio/max": 1.4994876384735107, "sampling/importance_sampling_ratio/mean": 0.9418887495994568, "sampling/importance_sampling_ratio/min": 0.277060866355896, "sampling/sampling_logp_difference/max": 1.380357265472412, "sampling/sampling_logp_difference/mean": 0.0985015481710434, "step": 84, "step_time": 12.40714328800459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 237.09375, "completions/mean_terminated_length": 237.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.47508566081523895, "epoch": 0.0017, "frac_reward_zero_std": 0.0, "grad_norm": 0.893117368221283, "kl": 0.6055086245760322, "learning_rate": 9.940773720241503e-06, "loss": -0.006, "num_tokens": 3702291.0, "reward": 0.02187500149011612, "reward_std": 0.295116126537323, "rewards/rollout_reward_func/mean": 0.02187500149011612, "rewards/rollout_reward_func/std": 0.2980954349040985, "sampling/importance_sampling_ratio/max": 1.383468747138977, "sampling/importance_sampling_ratio/mean": 0.9309595823287964, "sampling/importance_sampling_ratio/min": 0.42784833908081055, "sampling/sampling_logp_difference/max": 0.7891690731048584, "sampling/sampling_logp_difference/mean": 0.0722569078207016, "step": 85, "step_time": 11.736402934013313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 205.96875, "completions/mean_terminated_length": 205.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4149830234237015, "epoch": 0.00172, "frac_reward_zero_std": 0.0, "grad_norm": 18.804866790771484, "kl": 75.69156743399799, "learning_rate": 9.938338367928391e-06, "loss": 0.0905, "num_tokens": 3748090.0, "reward": -3.725290298461914e-09, "reward_std": 0.3395557999610901, "rewards/rollout_reward_func/mean": -3.725290298461914e-09, "rewards/rollout_reward_func/std": 0.33792537450790405, "sampling/importance_sampling_ratio/max": 1.300073266029358, "sampling/importance_sampling_ratio/mean": 0.8741573691368103, "sampling/importance_sampling_ratio/min": 0.293846994638443, "sampling/sampling_logp_difference/max": 1.2316937446594238, "sampling/sampling_logp_difference/mean": 0.0982658863067627, "step": 86, "step_time": 11.345277100997919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 251.15625, "completions/mean_terminated_length": 251.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.44753647316247225, "epoch": 0.00174, "frac_reward_zero_std": 0.0, "grad_norm": 0.8272233009338379, "kl": 0.8794522546231747, "learning_rate": 9.935854363938876e-06, "loss": -0.0015, "num_tokens": 3796046.0, "reward": 0.07500000298023224, "reward_std": 0.253395140171051, "rewards/rollout_reward_func/mean": 0.07500000298023224, "rewards/rollout_reward_func/std": 0.27708157896995544, "sampling/importance_sampling_ratio/max": 1.5400238037109375, "sampling/importance_sampling_ratio/mean": 0.9839857816696167, "sampling/importance_sampling_ratio/min": 0.4767392575740814, "sampling/sampling_logp_difference/max": 0.8229259252548218, "sampling/sampling_logp_difference/mean": 0.08524755388498306, "step": 87, "step_time": 11.788524373005203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 262.125, "completions/mean_terminated_length": 262.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.49094272404909134, "epoch": 0.00176, "frac_reward_zero_std": 0.0, "grad_norm": 1.4186978340148926, "kl": 1.3427610881626606, "learning_rate": 9.933321741038655e-06, "loss": 0.0075, "num_tokens": 3845092.0, "reward": 0.0031249988824129105, "reward_std": 0.2901339530944824, "rewards/rollout_reward_func/mean": 0.0031249988824129105, "rewards/rollout_reward_func/std": 0.2811088562011719, "sampling/importance_sampling_ratio/max": 2.916379451751709, "sampling/importance_sampling_ratio/mean": 1.1575154066085815, "sampling/importance_sampling_ratio/min": 0.2619177997112274, "sampling/sampling_logp_difference/max": 1.199333667755127, "sampling/sampling_logp_difference/mean": 0.10128861665725708, "step": 88, "step_time": 12.065905861996725 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 228.875, "completions/mean_terminated_length": 228.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.39658508729189634, "epoch": 0.00178, "frac_reward_zero_std": 0.0, "grad_norm": 0.5423808693885803, "kl": 1.8361330358311534, "learning_rate": 9.930740532634733e-06, "loss": 0.0076, "num_tokens": 3891727.0, "reward": -0.024999994784593582, "reward_std": 0.3156512677669525, "rewards/rollout_reward_func/mean": -0.024999994784593582, "rewards/rollout_reward_func/std": 0.32328954339027405, "sampling/importance_sampling_ratio/max": 1.4205175638198853, "sampling/importance_sampling_ratio/mean": 0.9180161952972412, "sampling/importance_sampling_ratio/min": 0.21461042761802673, "sampling/sampling_logp_difference/max": 1.5388216972351074, "sampling/sampling_logp_difference/mean": 0.09925561398267746, "step": 89, "step_time": 12.171361298991542 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 253.53125, "completions/mean_terminated_length": 253.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.39000480668619275, "epoch": 0.0018, "frac_reward_zero_std": 0.0, "grad_norm": 1.0643731355667114, "kl": 6.130362963303924, "learning_rate": 9.928110772774995e-06, "loss": -0.0304, "num_tokens": 3938887.0, "reward": 0.01874999888241291, "reward_std": 0.33162644505500793, "rewards/rollout_reward_func/mean": 0.01874999888241291, "rewards/rollout_reward_func/std": 0.333541601896286, "sampling/importance_sampling_ratio/max": 1.6728304624557495, "sampling/importance_sampling_ratio/mean": 0.8214892745018005, "sampling/importance_sampling_ratio/min": 0.14574505388736725, "sampling/sampling_logp_difference/max": 1.595613956451416, "sampling/sampling_logp_difference/mean": 0.15293510258197784, "step": 90, "step_time": 14.436771657008649 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 914.0, "completions/max_terminated_length": 914.0, "completions/mean_length": 206.8125, "completions/mean_terminated_length": 206.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.39246736466884613, "epoch": 0.00182, "frac_reward_zero_std": 0.0, "grad_norm": 1.0796570777893066, "kl": 4.6461689826101065, "learning_rate": 9.92543249614775e-06, "loss": 0.048, "num_tokens": 3984979.0, "reward": 0.046875, "reward_std": 0.27574604749679565, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.28168201446533203, "sampling/importance_sampling_ratio/max": 2.6780264377593994, "sampling/importance_sampling_ratio/mean": 0.9492089748382568, "sampling/importance_sampling_ratio/min": 0.16390137374401093, "sampling/sampling_logp_difference/max": 1.7413606643676758, "sampling/sampling_logp_difference/mean": 0.11615259945392609, "step": 91, "step_time": 12.436968942998647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 229.1875, "completions/mean_terminated_length": 229.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4497671276330948, "epoch": 0.00184, "frac_reward_zero_std": 0.0, "grad_norm": 0.6639618873596191, "kl": 2.9282248821109533, "learning_rate": 9.922705738081279e-06, "loss": -0.0134, "num_tokens": 4032823.0, "reward": 0.06875000149011612, "reward_std": 0.25987619161605835, "rewards/rollout_reward_func/mean": 0.06875000149011612, "rewards/rollout_reward_func/std": 0.26449891924858093, "sampling/importance_sampling_ratio/max": 2.0149550437927246, "sampling/importance_sampling_ratio/mean": 0.9327768683433533, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3461580276489258, "sampling/sampling_logp_difference/mean": 0.08719690144062042, "step": 92, "step_time": 11.290040398005658 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1290.0, "completions/max_terminated_length": 1290.0, "completions/mean_length": 249.375, "completions/mean_terminated_length": 249.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3659423221834004, "epoch": 0.00186, "frac_reward_zero_std": 0.0, "grad_norm": 1.2618210315704346, "kl": 2.363375425338745, "learning_rate": 9.919930534543365e-06, "loss": 0.0373, "num_tokens": 4079805.0, "reward": 0.08125000447034836, "reward_std": 0.28313344717025757, "rewards/rollout_reward_func/mean": 0.08125000447034836, "rewards/rollout_reward_func/std": 0.2934087812900543, "sampling/importance_sampling_ratio/max": 1.9516531229019165, "sampling/importance_sampling_ratio/mean": 0.9938030242919922, "sampling/importance_sampling_ratio/min": 0.1819620281457901, "sampling/sampling_logp_difference/max": 1.7041077613830566, "sampling/sampling_logp_difference/mean": 0.09374533593654633, "step": 93, "step_time": 15.296910974007915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 912.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 219.0625, "completions/mean_terminated_length": 219.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.41759001463651657, "epoch": 0.00188, "frac_reward_zero_std": 0.0, "grad_norm": 0.6337612867355347, "kl": 0.7507315594702959, "learning_rate": 9.917106922140814e-06, "loss": 0.0032, "num_tokens": 4125816.0, "reward": 0.11875000596046448, "reward_std": 0.2739913761615753, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.27171796560287476, "sampling/importance_sampling_ratio/max": 1.3245856761932373, "sampling/importance_sampling_ratio/mean": 0.9892963171005249, "sampling/importance_sampling_ratio/min": 0.387395977973938, "sampling/sampling_logp_difference/max": 0.8333804607391357, "sampling/sampling_logp_difference/mean": 0.0632328987121582, "step": 94, "step_time": 12.484614294990024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 292.3125, "completions/mean_terminated_length": 292.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3856537682004273, "epoch": 0.0019, "frac_reward_zero_std": 0.0, "grad_norm": 1.3504183292388916, "kl": 0.5354123990982771, "learning_rate": 9.914234938118988e-06, "loss": 0.0523, "num_tokens": 4174392.0, "reward": 0.10312499850988388, "reward_std": 0.26045024394989014, "rewards/rollout_reward_func/mean": 0.10312499850988388, "rewards/rollout_reward_func/std": 0.26698359847068787, "sampling/importance_sampling_ratio/max": 2.841588020324707, "sampling/importance_sampling_ratio/mean": 1.083536148071289, "sampling/importance_sampling_ratio/min": 0.7307860255241394, "sampling/sampling_logp_difference/max": 0.8147077560424805, "sampling/sampling_logp_difference/mean": 0.049498941749334335, "step": 95, "step_time": 13.405290610997326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 243.78125, "completions/mean_terminated_length": 243.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.35124557046219707, "epoch": 0.00192, "frac_reward_zero_std": 0.0, "grad_norm": 1.6480892896652222, "kl": 4.543591367080808, "learning_rate": 9.911314620361294e-06, "loss": 0.0306, "num_tokens": 4221401.0, "reward": -0.01562499813735485, "reward_std": 0.31441545486450195, "rewards/rollout_reward_func/mean": -0.01562499813735485, "rewards/rollout_reward_func/std": 0.32637736201286316, "sampling/importance_sampling_ratio/max": 2.6073691844940186, "sampling/importance_sampling_ratio/mean": 1.0100125074386597, "sampling/importance_sampling_ratio/min": 0.21372155845165253, "sampling/sampling_logp_difference/max": 1.5431303977966309, "sampling/sampling_logp_difference/mean": 0.12072505801916122, "step": 96, "step_time": 14.188338181997096 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 194.90625, "completions/mean_terminated_length": 194.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.34608333138749003, "epoch": 0.00194, "frac_reward_zero_std": 0.0, "grad_norm": 1.0583473443984985, "kl": 1.6900225020945072, "learning_rate": 9.908346007388698e-06, "loss": 0.0287, "num_tokens": 4267070.0, "reward": 0.04374999552965164, "reward_std": 0.29125094413757324, "rewards/rollout_reward_func/mean": 0.04374999552965164, "rewards/rollout_reward_func/std": 0.2861733138561249, "sampling/importance_sampling_ratio/max": 1.632218360900879, "sampling/importance_sampling_ratio/mean": 0.9461942315101624, "sampling/importance_sampling_ratio/min": 0.24909579753875732, "sampling/sampling_logp_difference/max": 1.186600685119629, "sampling/sampling_logp_difference/mean": 0.08526910096406937, "step": 97, "step_time": 14.063727663000464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 208.8125, "completions/mean_terminated_length": 208.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3446831237524748, "epoch": 0.00196, "frac_reward_zero_std": 0.0, "grad_norm": 0.5804661512374878, "kl": 1.0470349751412868, "learning_rate": 9.905329138359213e-06, "loss": 0.0088, "num_tokens": 4313865.0, "reward": 0.06562500447034836, "reward_std": 0.2312580645084381, "rewards/rollout_reward_func/mean": 0.06562500447034836, "rewards/rollout_reward_func/std": 0.22376452386379242, "sampling/importance_sampling_ratio/max": 1.3587920665740967, "sampling/importance_sampling_ratio/mean": 0.9519028663635254, "sampling/importance_sampling_ratio/min": 0.29864606261253357, "sampling/sampling_logp_difference/max": 1.4805827140808105, "sampling/sampling_logp_difference/mean": 0.07908046245574951, "step": 98, "step_time": 14.054440996998892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 225.125, "completions/mean_terminated_length": 225.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32458328595384955, "epoch": 0.00198, "frac_reward_zero_std": 0.0, "grad_norm": 0.42324307560920715, "kl": 2.521756824105978, "learning_rate": 9.902264053067381e-06, "loss": 0.0042, "num_tokens": 4359963.0, "reward": 0.07500000298023224, "reward_std": 0.3007655739784241, "rewards/rollout_reward_func/mean": 0.07500000298023224, "rewards/rollout_reward_func/std": 0.3079589307308197, "sampling/importance_sampling_ratio/max": 1.3982504606246948, "sampling/importance_sampling_ratio/mean": 0.9508129358291626, "sampling/importance_sampling_ratio/min": 0.34227243065834045, "sampling/sampling_logp_difference/max": 1.0720171928405762, "sampling/sampling_logp_difference/mean": 0.09748084098100662, "step": 99, "step_time": 14.13059324999631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 229.3125, "completions/mean_terminated_length": 229.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3004692429676652, "epoch": 0.002, "frac_reward_zero_std": 0.0, "grad_norm": 0.35834774374961853, "kl": 1.7545361053198576, "learning_rate": 9.899150791943747e-06, "loss": -0.0128, "num_tokens": 4406726.0, "reward": 0.12187500298023224, "reward_std": 0.21875181794166565, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.21810677647590637, "sampling/importance_sampling_ratio/max": 1.2363519668579102, "sampling/importance_sampling_ratio/mean": 0.9671210050582886, "sampling/importance_sampling_ratio/min": 0.3177724778652191, "sampling/sampling_logp_difference/max": 1.1468405723571777, "sampling/sampling_logp_difference/mean": 0.07596529275178909, "step": 100, "step_time": 13.29064988900791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1292.0, "completions/max_terminated_length": 1292.0, "completions/mean_length": 217.875, "completions/mean_terminated_length": 217.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3439148962497711, "epoch": 0.00202, "frac_reward_zero_std": 0.0, "grad_norm": 0.697306215763092, "kl": 1.831751735880971, "learning_rate": 9.895989396054334e-06, "loss": -0.0028, "num_tokens": 4453863.0, "reward": -0.02499999850988388, "reward_std": 0.31913071870803833, "rewards/rollout_reward_func/mean": -0.02499999850988388, "rewards/rollout_reward_func/std": 0.31826141476631165, "sampling/importance_sampling_ratio/max": 1.5414952039718628, "sampling/importance_sampling_ratio/mean": 0.9769704341888428, "sampling/importance_sampling_ratio/min": 0.3929733633995056, "sampling/sampling_logp_difference/max": 0.7221977710723877, "sampling/sampling_logp_difference/mean": 0.07063555717468262, "step": 101, "step_time": 14.756879051001306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 195.15625, "completions/mean_terminated_length": 195.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3266562819480896, "epoch": 0.00204, "frac_reward_zero_std": 0.0, "grad_norm": 0.3833155035972595, "kl": 3.0373174250125885, "learning_rate": 9.892779907100084e-06, "loss": -0.0135, "num_tokens": 4500614.0, "reward": 0.08750000596046448, "reward_std": 0.24063827097415924, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.27090054750442505, "sampling/importance_sampling_ratio/max": 1.1816695928573608, "sampling/importance_sampling_ratio/mean": 0.9767264127731323, "sampling/importance_sampling_ratio/min": 0.26534131169319153, "sampling/sampling_logp_difference/max": 1.32666015625, "sampling/sampling_logp_difference/mean": 0.07508103549480438, "step": 102, "step_time": 12.266822609984956 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 204.78125, "completions/mean_terminated_length": 204.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2648054212331772, "epoch": 0.00206, "frac_reward_zero_std": 0.0, "grad_norm": 0.5154334306716919, "kl": 1.0411842428147793, "learning_rate": 9.889522367416332e-06, "loss": 0.015, "num_tokens": 4545435.0, "reward": 0.16875000298023224, "reward_std": 0.16996155679225922, "rewards/rollout_reward_func/mean": 0.16875000298023224, "rewards/rollout_reward_func/std": 0.1925005316734314, "sampling/importance_sampling_ratio/max": 1.18610417842865, "sampling/importance_sampling_ratio/mean": 0.9515349268913269, "sampling/importance_sampling_ratio/min": 0.39361637830734253, "sampling/sampling_logp_difference/max": 0.8632616996765137, "sampling/sampling_logp_difference/mean": 0.061651796102523804, "step": 103, "step_time": 12.58951356200123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 723.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 197.8125, "completions/mean_terminated_length": 197.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2936482820659876, "epoch": 0.00208, "frac_reward_zero_std": 0.0, "grad_norm": 1.98922860622406, "kl": 5.008606903254986, "learning_rate": 9.886216819972226e-06, "loss": 0.0148, "num_tokens": 4591239.0, "reward": 0.06562500447034836, "reward_std": 0.2689783573150635, "rewards/rollout_reward_func/mean": 0.06562500447034836, "rewards/rollout_reward_func/std": 0.27894893288612366, "sampling/importance_sampling_ratio/max": 1.1327497959136963, "sampling/importance_sampling_ratio/mean": 0.9099741578102112, "sampling/importance_sampling_ratio/min": 0.1385810524225235, "sampling/sampling_logp_difference/max": 2.109035015106201, "sampling/sampling_logp_difference/mean": 0.10624553263187408, "step": 104, "step_time": 12.752196621000621 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 202.46875, "completions/mean_terminated_length": 202.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26434253668412566, "epoch": 0.0021, "frac_reward_zero_std": 0.0, "grad_norm": 0.5037868618965149, "kl": 3.4269874058663845, "learning_rate": 9.882863308370175e-06, "loss": 0.0155, "num_tokens": 4637831.0, "reward": 0.02500000037252903, "reward_std": 0.29782116413116455, "rewards/rollout_reward_func/mean": 0.02500000037252903, "rewards/rollout_reward_func/std": 0.29621267318725586, "sampling/importance_sampling_ratio/max": 1.1961731910705566, "sampling/importance_sampling_ratio/mean": 0.9444249868392944, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1301612854003906, "sampling/sampling_logp_difference/mean": 0.08325405418872833, "step": 105, "step_time": 11.881546850989253 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 208.09375, "completions/mean_terminated_length": 208.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2993669081479311, "epoch": 0.00212, "frac_reward_zero_std": 0.0, "grad_norm": 0.20398235321044922, "kl": 1.6439198926091194, "learning_rate": 9.879461876845255e-06, "loss": -0.009, "num_tokens": 4684235.0, "reward": 0.13124999403953552, "reward_std": 0.2647007703781128, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.26692697405815125, "sampling/importance_sampling_ratio/max": 1.3727110624313354, "sampling/importance_sampling_ratio/mean": 1.0012590885162354, "sampling/importance_sampling_ratio/min": 0.353504478931427, "sampling/sampling_logp_difference/max": 1.1134586334228516, "sampling/sampling_logp_difference/mean": 0.05528904125094414, "step": 106, "step_time": 11.363588891996187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1112.0, "completions/max_terminated_length": 1112.0, "completions/mean_length": 289.46875, "completions/mean_terminated_length": 289.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2992045213468373, "epoch": 0.00214, "frac_reward_zero_std": 0.0, "grad_norm": 0.8955283164978027, "kl": 0.8790501765906811, "learning_rate": 9.876012570264658e-06, "loss": 0.0177, "num_tokens": 4733048.0, "reward": 0.07500000298023224, "reward_std": 0.32512110471725464, "rewards/rollout_reward_func/mean": 0.07500000298023224, "rewards/rollout_reward_func/std": 0.33696943521499634, "sampling/importance_sampling_ratio/max": 1.683269739151001, "sampling/importance_sampling_ratio/mean": 0.9839057922363281, "sampling/importance_sampling_ratio/min": 0.3084339201450348, "sampling/sampling_logp_difference/max": 1.1759097576141357, "sampling/sampling_logp_difference/mean": 0.08656682074069977, "step": 107, "step_time": 14.093497940997622 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0031250000465661287, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008333333535119891, "completions/clipped_ratio": 0.0, "completions/max_length": 770.0, "completions/max_terminated_length": 770.0, "completions/mean_length": 249.25, "completions/mean_terminated_length": 249.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25788985285907984, "epoch": 0.00216, "frac_reward_zero_std": 0.0, "grad_norm": 0.296503484249115, "kl": 1.1829804126173258, "learning_rate": 9.872515434127064e-06, "loss": 0.0111, "num_tokens": 4780747.0, "reward": 0.05937500298023224, "reward_std": 0.2058488428592682, "rewards/rollout_reward_func/mean": 0.05937500298023224, "rewards/rollout_reward_func/std": 0.22698496282100677, "sampling/importance_sampling_ratio/max": 1.6195276975631714, "sampling/importance_sampling_ratio/mean": 1.000553011894226, "sampling/importance_sampling_ratio/min": 0.4303126633167267, "sampling/sampling_logp_difference/max": 0.8430638313293457, "sampling/sampling_logp_difference/mean": 0.0575980469584465, "step": 108, "step_time": 12.490547448993311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 232.78125, "completions/mean_terminated_length": 232.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24424895271658897, "epoch": 0.00218, "frac_reward_zero_std": 0.0, "grad_norm": 0.5754547715187073, "kl": 1.555072333663702, "learning_rate": 9.86897051456206e-06, "loss": -0.0214, "num_tokens": 4828179.0, "reward": 0.14375001192092896, "reward_std": 0.27734389901161194, "rewards/rollout_reward_func/mean": 0.14375001192092896, "rewards/rollout_reward_func/std": 0.274669885635376, "sampling/importance_sampling_ratio/max": 1.5721135139465332, "sampling/importance_sampling_ratio/mean": 1.0164718627929688, "sampling/importance_sampling_ratio/min": 0.24816107749938965, "sampling/sampling_logp_difference/max": 1.0410022735595703, "sampling/sampling_logp_difference/mean": 0.05624841898679733, "step": 109, "step_time": 13.569237012994563 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 346.25, "completions/mean_terminated_length": 346.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25154087832197547, "epoch": 0.0022, "frac_reward_zero_std": 0.0, "grad_norm": 1.0647169351577759, "kl": 4.238150646910071, "learning_rate": 9.865377858329539e-06, "loss": -0.0031, "num_tokens": 4880122.0, "reward": 0.04375000670552254, "reward_std": 0.32386475801467896, "rewards/rollout_reward_func/mean": 0.04375000670552254, "rewards/rollout_reward_func/std": 0.3202191889286041, "sampling/importance_sampling_ratio/max": 1.1978249549865723, "sampling/importance_sampling_ratio/mean": 0.8832293748855591, "sampling/importance_sampling_ratio/min": 0.18632784485816956, "sampling/sampling_logp_difference/max": 1.7310974597930908, "sampling/sampling_logp_difference/mean": 0.09921973943710327, "step": 110, "step_time": 14.581374379999033 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 201.75, "completions/mean_terminated_length": 201.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20054214680567384, "epoch": 0.00222, "frac_reward_zero_std": 0.0, "grad_norm": 0.413242906332016, "kl": 3.113514844328165, "learning_rate": 9.861737512819058e-06, "loss": 0.0135, "num_tokens": 4925290.0, "reward": 0.05937500298023224, "reward_std": 0.2924154996871948, "rewards/rollout_reward_func/mean": 0.05937500298023224, "rewards/rollout_reward_func/std": 0.2803907096385956, "sampling/importance_sampling_ratio/max": 1.1220498085021973, "sampling/importance_sampling_ratio/mean": 0.9535717964172363, "sampling/importance_sampling_ratio/min": 0.26292628049850464, "sampling/sampling_logp_difference/max": 1.353344440460205, "sampling/sampling_logp_difference/mean": 0.06640806049108505, "step": 111, "step_time": 13.669375223995303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 856.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 128.21875, "completions/mean_terminated_length": 128.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2124088080599904, "epoch": 0.00224, "frac_reward_zero_std": 0.0, "grad_norm": 0.33900097012519836, "kl": 1.31798595469445, "learning_rate": 9.858049526049241e-06, "loss": -0.0046, "num_tokens": 4967438.0, "reward": 0.17812500894069672, "reward_std": 0.203436017036438, "rewards/rollout_reward_func/mean": 0.17812500894069672, "rewards/rollout_reward_func/std": 0.20593512058258057, "sampling/importance_sampling_ratio/max": 1.1272529363632202, "sampling/importance_sampling_ratio/mean": 0.9946641325950623, "sampling/importance_sampling_ratio/min": 0.30649933218955994, "sampling/sampling_logp_difference/max": 1.2476530075073242, "sampling/sampling_logp_difference/mean": 0.049943048506975174, "step": 112, "step_time": 12.487326954997116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 217.125, "completions/mean_terminated_length": 217.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2916075475513935, "epoch": 0.00226, "frac_reward_zero_std": 0.0, "grad_norm": 0.763655424118042, "kl": 0.6759616192430258, "learning_rate": 9.85431394666712e-06, "loss": -0.0064, "num_tokens": 5015089.0, "reward": 0.04375000298023224, "reward_std": 0.24051439762115479, "rewards/rollout_reward_func/mean": 0.04375000298023224, "rewards/rollout_reward_func/std": 0.2381887286901474, "sampling/importance_sampling_ratio/max": 1.1813639402389526, "sampling/importance_sampling_ratio/mean": 0.9538316130638123, "sampling/importance_sampling_ratio/min": 0.50259929895401, "sampling/sampling_logp_difference/max": 0.6860241889953613, "sampling/sampling_logp_difference/mean": 0.06292776763439178, "step": 113, "step_time": 11.717436049002572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1279.0, "completions/max_terminated_length": 1279.0, "completions/mean_length": 237.53125, "completions/mean_terminated_length": 237.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28384392615407705, "epoch": 0.00228, "frac_reward_zero_std": 0.0, "grad_norm": 0.501704752445221, "kl": 3.369587168097496, "learning_rate": 9.850530823947519e-06, "loss": 0.0008, "num_tokens": 5061678.0, "reward": 0.09687499701976776, "reward_std": 0.23002414405345917, "rewards/rollout_reward_func/mean": 0.09687499701976776, "rewards/rollout_reward_func/std": 0.22645141184329987, "sampling/importance_sampling_ratio/max": 1.7737129926681519, "sampling/importance_sampling_ratio/mean": 0.9480115175247192, "sampling/importance_sampling_ratio/min": 0.2751051187515259, "sampling/sampling_logp_difference/max": 1.1708531379699707, "sampling/sampling_logp_difference/mean": 0.09438836574554443, "step": 114, "step_time": 14.324374274990987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 201.15625, "completions/mean_terminated_length": 201.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.33769882936030626, "epoch": 0.0023, "frac_reward_zero_std": 0.0, "grad_norm": 0.37992438673973083, "kl": 1.0239699091762304, "learning_rate": 9.846700207792381e-06, "loss": -0.0071, "num_tokens": 5107313.0, "reward": 0.0031250035390257835, "reward_std": 0.2740105390548706, "rewards/rollout_reward_func/mean": 0.0031250035390257835, "rewards/rollout_reward_func/std": 0.28902995586395264, "sampling/importance_sampling_ratio/max": 1.0807331800460815, "sampling/importance_sampling_ratio/mean": 0.8693166971206665, "sampling/importance_sampling_ratio/min": 0.19341106712818146, "sampling/sampling_logp_difference/max": 1.6403775215148926, "sampling/sampling_logp_difference/mean": 0.0990855023264885, "step": 115, "step_time": 13.206800740998005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 217.0, "completions/mean_terminated_length": 217.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28839701879769564, "epoch": 0.00232, "frac_reward_zero_std": 0.0, "grad_norm": 0.48408156633377075, "kl": 1.029269577935338, "learning_rate": 9.842822148730122e-06, "loss": -0.0068, "num_tokens": 5153347.0, "reward": 0.12187501043081284, "reward_std": 0.30733466148376465, "rewards/rollout_reward_func/mean": 0.12187501043081284, "rewards/rollout_reward_func/std": 0.3097703158855438, "sampling/importance_sampling_ratio/max": 1.6210176944732666, "sampling/importance_sampling_ratio/mean": 0.9265075922012329, "sampling/importance_sampling_ratio/min": 0.3697737753391266, "sampling/sampling_logp_difference/max": 1.0372023582458496, "sampling/sampling_logp_difference/mean": 0.09927916526794434, "step": 116, "step_time": 14.025971915001719 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1315.0, "completions/max_terminated_length": 1315.0, "completions/mean_length": 280.75, "completions/mean_terminated_length": 280.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32195940241217613, "epoch": 0.00234, "frac_reward_zero_std": 0.0, "grad_norm": 0.8184932470321655, "kl": 1.2891521453857422, "learning_rate": 9.838896697914964e-06, "loss": 0.0204, "num_tokens": 5200486.0, "reward": 0.09062499552965164, "reward_std": 0.34046411514282227, "rewards/rollout_reward_func/mean": 0.09062499552965164, "rewards/rollout_reward_func/std": 0.35319679975509644, "sampling/importance_sampling_ratio/max": 1.6981090307235718, "sampling/importance_sampling_ratio/mean": 0.9332734942436218, "sampling/importance_sampling_ratio/min": 0.37692394852638245, "sampling/sampling_logp_difference/max": 1.2296710014343262, "sampling/sampling_logp_difference/mean": 0.09127403795719147, "step": 117, "step_time": 14.503186389993061 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 302.34375, "completions/mean_terminated_length": 302.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3317412110045552, "epoch": 0.00236, "frac_reward_zero_std": 0.0, "grad_norm": 0.39741164445877075, "kl": 1.0479370197281241, "learning_rate": 9.834923907126256e-06, "loss": -0.0168, "num_tokens": 5250320.0, "reward": 0.0312500074505806, "reward_std": 0.3397306799888611, "rewards/rollout_reward_func/mean": 0.0312500074505806, "rewards/rollout_reward_func/std": 0.32768741250038147, "sampling/importance_sampling_ratio/max": 1.2252277135849, "sampling/importance_sampling_ratio/mean": 0.8734408617019653, "sampling/importance_sampling_ratio/min": 4.690927198680583e-06, "sampling/sampling_logp_difference/max": 12.470120429992676, "sampling/sampling_logp_difference/mean": 0.17994505167007446, "step": 118, "step_time": 14.845855015002599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 235.15625, "completions/mean_terminated_length": 235.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24140833504498005, "epoch": 0.00238, "frac_reward_zero_std": 0.0, "grad_norm": 0.49425530433654785, "kl": 2.2515236604958773, "learning_rate": 9.830903828767796e-06, "loss": -0.0063, "num_tokens": 5295900.0, "reward": 0.06875000149011612, "reward_std": 0.31470200419425964, "rewards/rollout_reward_func/mean": 0.06875000149011612, "rewards/rollout_reward_func/std": 0.31667375564575195, "sampling/importance_sampling_ratio/max": 1.6615864038467407, "sampling/importance_sampling_ratio/mean": 0.9332354068756104, "sampling/importance_sampling_ratio/min": 0.2780778110027313, "sampling/sampling_logp_difference/max": 1.1978769302368164, "sampling/sampling_logp_difference/mean": 0.08609779179096222, "step": 119, "step_time": 13.877338754999073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 275.03125, "completions/mean_terminated_length": 275.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.33425380755215883, "epoch": 0.0024, "frac_reward_zero_std": 0.0, "grad_norm": 0.643244743347168, "kl": 0.6735842060297728, "learning_rate": 9.826836515867131e-06, "loss": 0.0097, "num_tokens": 5344814.0, "reward": 0.08437499403953552, "reward_std": 0.3090837597846985, "rewards/rollout_reward_func/mean": 0.08437499403953552, "rewards/rollout_reward_func/std": 0.3111989200115204, "sampling/importance_sampling_ratio/max": 1.8053289651870728, "sampling/importance_sampling_ratio/mean": 0.9024233222007751, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1728119850158691, "sampling/sampling_logp_difference/mean": 0.09721174091100693, "step": 120, "step_time": 13.368254041004548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 279.875, "completions/mean_terminated_length": 279.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.300092660356313, "epoch": 0.00242, "frac_reward_zero_std": 0.0, "grad_norm": 0.6000701785087585, "kl": 0.9331304505467415, "learning_rate": 9.822722022074871e-06, "loss": -0.0033, "num_tokens": 5393380.0, "reward": 0.04062499850988388, "reward_std": 0.289772093296051, "rewards/rollout_reward_func/mean": 0.04062499850988388, "rewards/rollout_reward_func/std": 0.32711777091026306, "sampling/importance_sampling_ratio/max": 1.0981312990188599, "sampling/importance_sampling_ratio/mean": 0.8180450797080994, "sampling/importance_sampling_ratio/min": 0.2523910403251648, "sampling/sampling_logp_difference/max": 0.7518954277038574, "sampling/sampling_logp_difference/mean": 0.1128598153591156, "step": 121, "step_time": 14.076159760999872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1218.0, "completions/max_terminated_length": 1218.0, "completions/mean_length": 362.71875, "completions/mean_terminated_length": 362.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32585998717695475, "epoch": 0.00244, "frac_reward_zero_std": 0.0, "grad_norm": 1.282537817955017, "kl": 0.6267124284058809, "learning_rate": 9.818560401663972e-06, "loss": 0.0526, "num_tokens": 5445051.0, "reward": 0.07500000298023224, "reward_std": 0.33250710368156433, "rewards/rollout_reward_func/mean": 0.07500000298023224, "rewards/rollout_reward_func/std": 0.3698299527168274, "sampling/importance_sampling_ratio/max": 2.982555389404297, "sampling/importance_sampling_ratio/mean": 0.8648760318756104, "sampling/importance_sampling_ratio/min": 0.09502250701189041, "sampling/sampling_logp_difference/max": 1.0772743225097656, "sampling/sampling_logp_difference/mean": 0.1359127163887024, "step": 122, "step_time": 15.13037581199751 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 270.53125, "completions/mean_terminated_length": 270.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27747094724327326, "epoch": 0.00246, "frac_reward_zero_std": 0.0, "grad_norm": 0.5541598796844482, "kl": 1.9753112588077784, "learning_rate": 9.814351709529018e-06, "loss": -0.0147, "num_tokens": 5492681.0, "reward": 0.03125, "reward_std": 0.27813541889190674, "rewards/rollout_reward_func/mean": 0.03125, "rewards/rollout_reward_func/std": 0.27759045362472534, "sampling/importance_sampling_ratio/max": 2.915048599243164, "sampling/importance_sampling_ratio/mean": 0.91321861743927, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3309125900268555, "sampling/sampling_logp_difference/mean": 0.15597669780254364, "step": 123, "step_time": 13.784536361006758 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 264.53125, "completions/mean_terminated_length": 264.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25665646977722645, "epoch": 0.00248, "frac_reward_zero_std": 0.0, "grad_norm": 0.6446283459663391, "kl": 0.9336430784314871, "learning_rate": 9.8100960011855e-06, "loss": -0.0047, "num_tokens": 5540269.0, "reward": 0.078125, "reward_std": 0.2977718710899353, "rewards/rollout_reward_func/mean": 0.078125, "rewards/rollout_reward_func/std": 0.2926347255706787, "sampling/importance_sampling_ratio/max": 1.893554925918579, "sampling/importance_sampling_ratio/mean": 0.8628972768783569, "sampling/importance_sampling_ratio/min": 0.13846144080162048, "sampling/sampling_logp_difference/max": 1.025696039199829, "sampling/sampling_logp_difference/mean": 0.14345775544643402, "step": 124, "step_time": 14.20437228699302 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1169.0, "completions/max_terminated_length": 1169.0, "completions/mean_length": 298.15625, "completions/mean_terminated_length": 298.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2724336590617895, "epoch": 0.0025, "frac_reward_zero_std": 0.0, "grad_norm": 1.0481928586959839, "kl": 1.0610086657106876, "learning_rate": 9.805793332769095e-06, "loss": 0.0001, "num_tokens": 5589334.0, "reward": 0.11875000596046448, "reward_std": 0.2595295011997223, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.27759045362472534, "sampling/importance_sampling_ratio/max": 2.6591691970825195, "sampling/importance_sampling_ratio/mean": 0.8148394823074341, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3737690448760986, "sampling/sampling_logp_difference/mean": 0.1840461939573288, "step": 125, "step_time": 14.414741896998748 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02734375, "completions/clipped_ratio": 0.0, "completions/max_length": 966.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 203.53125, "completions/mean_terminated_length": 203.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21761660347692668, "epoch": 0.00252, "frac_reward_zero_std": 0.0, "grad_norm": 1.147390365600586, "kl": 1.3154200240969658, "learning_rate": 9.801443761034903e-06, "loss": 0.0363, "num_tokens": 5635150.0, "reward": 0.10000000894069672, "reward_std": 0.29025429487228394, "rewards/rollout_reward_func/mean": 0.10000000894069672, "rewards/rollout_reward_func/std": 0.27708157896995544, "sampling/importance_sampling_ratio/max": 2.4440934658050537, "sampling/importance_sampling_ratio/mean": 1.0259366035461426, "sampling/importance_sampling_ratio/min": 0.22314757108688354, "sampling/sampling_logp_difference/max": 1.6528605222702026, "sampling/sampling_logp_difference/mean": 0.16445386409759521, "step": 126, "step_time": 13.941512244000478 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 1220.0, "completions/max_terminated_length": 1220.0, "completions/mean_length": 232.5, "completions/mean_terminated_length": 232.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2053882130421698, "epoch": 0.00254, "frac_reward_zero_std": 0.0, "grad_norm": 1.0303444862365723, "kl": 2.861621156334877, "learning_rate": 9.797047343356718e-06, "loss": 0.0322, "num_tokens": 5682018.0, "reward": 0.16875001788139343, "reward_std": 0.29378464818000793, "rewards/rollout_reward_func/mean": 0.16875001788139343, "rewards/rollout_reward_func/std": 0.2966887056827545, "sampling/importance_sampling_ratio/max": 1.8486286401748657, "sampling/importance_sampling_ratio/mean": 0.7638448476791382, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9839532375335693, "sampling/sampling_logp_difference/mean": 0.21160779893398285, "step": 127, "step_time": 15.248874683995382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 966.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 310.5, "completions/mean_terminated_length": 310.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2287447126582265, "epoch": 0.00256, "frac_reward_zero_std": 0.0, "grad_norm": 3.505937337875366, "kl": 1.7110635414719582, "learning_rate": 9.792604137726259e-06, "loss": -0.0101, "num_tokens": 5732492.0, "reward": 0.07500000298023224, "reward_std": 0.2635658383369446, "rewards/rollout_reward_func/mean": 0.07500000298023224, "rewards/rollout_reward_func/std": 0.27591490745544434, "sampling/importance_sampling_ratio/max": 2.5899813175201416, "sampling/importance_sampling_ratio/mean": 1.042312502861023, "sampling/importance_sampling_ratio/min": 0.23784540593624115, "sampling/sampling_logp_difference/max": 1.485189437866211, "sampling/sampling_logp_difference/mean": 0.16179947555065155, "step": 128, "step_time": 13.376315664005233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.03125, "completions/max_length": 893.0, "completions/max_terminated_length": 893.0, "completions/mean_length": 239.5625, "completions/mean_terminated_length": 246.77418518066406, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21395488735288382, "epoch": 0.00258, "frac_reward_zero_std": 0.0, "grad_norm": 0.5616178512573242, "kl": 1.4944073650985956, "learning_rate": 9.788114202752415e-06, "loss": -0.0051, "num_tokens": 5779060.0, "reward": 0.140625, "reward_std": 0.215613454580307, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.2153157889842987, "sampling/importance_sampling_ratio/max": 2.9189391136169434, "sampling/importance_sampling_ratio/mean": 0.9283194541931152, "sampling/importance_sampling_ratio/min": 0.19211764633655548, "sampling/sampling_logp_difference/max": 1.6887027025222778, "sampling/sampling_logp_difference/mean": 0.1487518846988678, "step": 129, "step_time": 12.813592635997338 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 230.84375, "completions/mean_terminated_length": 230.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20714781805872917, "epoch": 0.0026, "frac_reward_zero_std": 0.0, "grad_norm": 0.7991582751274109, "kl": 1.309498604387045, "learning_rate": 9.78357759766046e-06, "loss": -0.0332, "num_tokens": 5824921.0, "reward": 0.15937499701976776, "reward_std": 0.24971508979797363, "rewards/rollout_reward_func/mean": 0.15937499701976776, "rewards/rollout_reward_func/std": 0.2486860603094101, "sampling/importance_sampling_ratio/max": 2.3858134746551514, "sampling/importance_sampling_ratio/mean": 0.8600364923477173, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5253984928131104, "sampling/sampling_logp_difference/mean": 0.15735627710819244, "step": 130, "step_time": 15.503862707995722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 402.15625, "completions/mean_terminated_length": 402.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26479471288621426, "epoch": 0.00262, "frac_reward_zero_std": 0.0, "grad_norm": 1.847597599029541, "kl": 3.0194654725492, "learning_rate": 9.778994382291283e-06, "loss": 0.0282, "num_tokens": 5880139.0, "reward": 0.15000000596046448, "reward_std": 0.29012420773506165, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.286243736743927, "sampling/importance_sampling_ratio/max": 2.5753355026245117, "sampling/importance_sampling_ratio/mean": 0.920081377029419, "sampling/importance_sampling_ratio/min": 0.16100791096687317, "sampling/sampling_logp_difference/max": 2.2110836505889893, "sampling/sampling_logp_difference/mean": 0.17135745286941528, "step": 131, "step_time": 14.850544807006372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 894.0, "completions/max_terminated_length": 894.0, "completions/mean_length": 196.375, "completions/mean_terminated_length": 196.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17077618930488825, "epoch": 0.00264, "frac_reward_zero_std": 0.0, "grad_norm": 1.3148186206817627, "kl": 2.825585976243019, "learning_rate": 9.774364617100596e-06, "loss": 0.059, "num_tokens": 5925541.0, "reward": 0.13749998807907104, "reward_std": 0.26283448934555054, "rewards/rollout_reward_func/mean": 0.13749998807907104, "rewards/rollout_reward_func/std": 0.2779533565044403, "sampling/importance_sampling_ratio/max": 2.921295642852783, "sampling/importance_sampling_ratio/mean": 1.0246782302856445, "sampling/importance_sampling_ratio/min": 0.2993585467338562, "sampling/sampling_logp_difference/max": 1.2220849990844727, "sampling/sampling_logp_difference/mean": 0.1610684096813202, "step": 132, "step_time": 13.414998054002353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 838.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 171.09375, "completions/mean_terminated_length": 171.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16770659270696342, "epoch": 0.00266, "frac_reward_zero_std": 0.0, "grad_norm": 0.49109652638435364, "kl": 0.9260859275236726, "learning_rate": 9.769688363158127e-06, "loss": 0.0302, "num_tokens": 5970391.0, "reward": 0.17499999701976776, "reward_std": 0.24881362915039062, "rewards/rollout_reward_func/mean": 0.17499999701976776, "rewards/rollout_reward_func/std": 0.25778210163116455, "sampling/importance_sampling_ratio/max": 2.2854275703430176, "sampling/importance_sampling_ratio/mean": 0.9197807908058167, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2237012386322021, "sampling/sampling_logp_difference/mean": 0.1126866564154625, "step": 133, "step_time": 12.521774499004096 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 303.78125, "completions/mean_terminated_length": 303.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1743362054694444, "epoch": 0.00268, "frac_reward_zero_std": 0.0, "grad_norm": 0.8590671420097351, "kl": 0.6799564827233553, "learning_rate": 9.76496568214683e-06, "loss": 0.0178, "num_tokens": 6019184.0, "reward": 0.15625, "reward_std": 0.23074083030223846, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.23131810128688812, "sampling/importance_sampling_ratio/max": 1.4538109302520752, "sampling/importance_sampling_ratio/mean": 0.8999476432800293, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0063252449035645, "sampling/sampling_logp_difference/mean": 0.08788031339645386, "step": 134, "step_time": 13.694064075003553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 342.75, "completions/mean_terminated_length": 342.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16918938048183918, "epoch": 0.0027, "frac_reward_zero_std": 0.0, "grad_norm": 0.6431775093078613, "kl": 0.8783140610903502, "learning_rate": 9.760196636362058e-06, "loss": 0.0143, "num_tokens": 6070375.0, "reward": 0.19062499701976776, "reward_std": 0.23558858036994934, "rewards/rollout_reward_func/mean": 0.19062499701976776, "rewards/rollout_reward_func/std": 0.24144640564918518, "sampling/importance_sampling_ratio/max": 2.1637704372406006, "sampling/importance_sampling_ratio/mean": 0.9591006636619568, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.208862543106079, "sampling/sampling_logp_difference/mean": 0.06532647460699081, "step": 135, "step_time": 14.674816004993772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 274.09375, "completions/mean_terminated_length": 274.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15099770505912602, "epoch": 0.00272, "frac_reward_zero_std": 0.0, "grad_norm": 1.1833897829055786, "kl": 1.3897612262517214, "learning_rate": 9.755381288710751e-06, "loss": 0.0207, "num_tokens": 6117859.0, "reward": 0.12812501192092896, "reward_std": 0.2600041627883911, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.24787402153015137, "sampling/importance_sampling_ratio/max": 1.33008873462677, "sampling/importance_sampling_ratio/mean": 0.9736813306808472, "sampling/importance_sampling_ratio/min": 0.20651058852672577, "sampling/sampling_logp_difference/max": 1.4822115898132324, "sampling/sampling_logp_difference/mean": 0.0686761736869812, "step": 136, "step_time": 14.719777527006954 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1324.0, "completions/max_terminated_length": 1324.0, "completions/mean_length": 275.46875, "completions/mean_terminated_length": 275.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1463230603840202, "epoch": 0.00274, "frac_reward_zero_std": 0.0, "grad_norm": 0.509434700012207, "kl": 0.5253063701093197, "learning_rate": 9.750519702710598e-06, "loss": 0.0007, "num_tokens": 6165810.0, "reward": 0.14687499403953552, "reward_std": 0.1864965558052063, "rewards/rollout_reward_func/mean": 0.14687499403953552, "rewards/rollout_reward_func/std": 0.2031753957271576, "sampling/importance_sampling_ratio/max": 1.3324300050735474, "sampling/importance_sampling_ratio/mean": 0.983070969581604, "sampling/importance_sampling_ratio/min": 0.6590755581855774, "sampling/sampling_logp_difference/max": 0.4168107509613037, "sampling/sampling_logp_difference/mean": 0.04049529880285263, "step": 137, "step_time": 14.497101386001304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1297.0, "completions/max_terminated_length": 1297.0, "completions/mean_length": 300.0, "completions/mean_terminated_length": 300.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21237155562266707, "epoch": 0.00276, "frac_reward_zero_std": 0.0, "grad_norm": 0.654344916343689, "kl": 0.7562607862055302, "learning_rate": 9.745611942489202e-06, "loss": 0.0045, "num_tokens": 6214736.0, "reward": 0.08125000447034836, "reward_std": 0.30866295099258423, "rewards/rollout_reward_func/mean": 0.08125000447034836, "rewards/rollout_reward_func/std": 0.30314216017723083, "sampling/importance_sampling_ratio/max": 1.1320244073867798, "sampling/importance_sampling_ratio/mean": 0.863916277885437, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.272151231765747, "sampling/sampling_logp_difference/mean": 0.10016614943742752, "step": 138, "step_time": 15.475928243009548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 235.4375, "completions/mean_terminated_length": 235.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20066457497887313, "epoch": 0.00278, "frac_reward_zero_std": 0.0, "grad_norm": 0.3736792206764221, "kl": 0.7053939998149872, "learning_rate": 9.740658072783244e-06, "loss": -0.0093, "num_tokens": 6261829.0, "reward": -0.03749999776482582, "reward_std": 0.3121172785758972, "rewards/rollout_reward_func/mean": -0.03749999776482582, "rewards/rollout_reward_func/std": 0.32897689938545227, "sampling/importance_sampling_ratio/max": 1.312301754951477, "sampling/importance_sampling_ratio/mean": 0.8470527529716492, "sampling/importance_sampling_ratio/min": 0.13406482338905334, "sampling/sampling_logp_difference/max": 2.197702407836914, "sampling/sampling_logp_difference/mean": 0.10178340971469879, "step": 139, "step_time": 13.855161410003348 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 294.5, "completions/mean_terminated_length": 294.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24091670708730817, "epoch": 0.0028, "frac_reward_zero_std": 0.0, "grad_norm": 0.5921764373779297, "kl": 1.3767810259014368, "learning_rate": 9.73565815893761e-06, "loss": 0.0043, "num_tokens": 6310895.0, "reward": 0.03750000149011612, "reward_std": 0.2871403694152832, "rewards/rollout_reward_func/mean": 0.03750000149011612, "rewards/rollout_reward_func/std": 0.3055930435657501, "sampling/importance_sampling_ratio/max": 1.4182549715042114, "sampling/importance_sampling_ratio/mean": 0.8202349543571472, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.249713182449341, "sampling/sampling_logp_difference/mean": 0.10817522555589676, "step": 140, "step_time": 14.326569304997975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 289.21875, "completions/mean_terminated_length": 289.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20242785476148129, "epoch": 0.00282, "frac_reward_zero_std": 0.0, "grad_norm": 0.644902229309082, "kl": 1.338408226147294, "learning_rate": 9.730612266904548e-06, "loss": -0.0262, "num_tokens": 6360171.0, "reward": 0.06875000149011612, "reward_std": 0.29279688000679016, "rewards/rollout_reward_func/mean": 0.06875000149011612, "rewards/rollout_reward_func/std": 0.31769075989723206, "sampling/importance_sampling_ratio/max": 2.896160125732422, "sampling/importance_sampling_ratio/mean": 0.9740906357765198, "sampling/importance_sampling_ratio/min": 0.1554381251335144, "sampling/sampling_logp_difference/max": 1.861316204071045, "sampling/sampling_logp_difference/mean": 0.10840821266174316, "step": 141, "step_time": 13.52672512600111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1265.0, "completions/max_terminated_length": 1265.0, "completions/mean_length": 355.09375, "completions/mean_terminated_length": 355.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23999885516241193, "epoch": 0.00284, "frac_reward_zero_std": 0.0, "grad_norm": 0.724422812461853, "kl": 1.0554293654859066, "learning_rate": 9.72552046324278e-06, "loss": -0.001, "num_tokens": 6410260.0, "reward": 0.125, "reward_std": 0.30080604553222656, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.31004682183265686, "sampling/importance_sampling_ratio/max": 2.1713385581970215, "sampling/importance_sampling_ratio/mean": 0.7982389330863953, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7769567966461182, "sampling/sampling_logp_difference/mean": 0.17259740829467773, "step": 142, "step_time": 14.956212549004704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 287.6875, "completions/mean_terminated_length": 287.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2070290264673531, "epoch": 0.00286, "frac_reward_zero_std": 0.0, "grad_norm": 0.852437436580658, "kl": 0.8689701110124588, "learning_rate": 9.72038281511664e-06, "loss": -0.0233, "num_tokens": 6459337.0, "reward": 0.09375, "reward_std": 0.2876502275466919, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.3397698402404785, "sampling/importance_sampling_ratio/max": 1.6078977584838867, "sampling/importance_sampling_ratio/mean": 0.7922504544258118, "sampling/importance_sampling_ratio/min": 0.2113371044397354, "sampling/sampling_logp_difference/max": 1.642838954925537, "sampling/sampling_logp_difference/mean": 0.15154562890529633, "step": 143, "step_time": 14.497355081002752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 302.0, "completions/mean_terminated_length": 302.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2197235538624227, "epoch": 0.00288, "frac_reward_zero_std": 0.0, "grad_norm": 0.5652056336402893, "kl": 2.2033361895009875, "learning_rate": 9.715199390295181e-06, "loss": -0.0088, "num_tokens": 6508527.0, "reward": -0.003124999813735485, "reward_std": 0.270757794380188, "rewards/rollout_reward_func/mean": -0.003124999813735485, "rewards/rollout_reward_func/std": 0.28225404024124146, "sampling/importance_sampling_ratio/max": 2.916644811630249, "sampling/importance_sampling_ratio/mean": 0.8159888982772827, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.026862621307373, "sampling/sampling_logp_difference/mean": 0.19092261791229248, "step": 144, "step_time": 13.948486578003212 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.020833333488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.028645833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 277.6875, "completions/mean_terminated_length": 277.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23712927685119212, "epoch": 0.0029, "frac_reward_zero_std": 0.0, "grad_norm": 0.9720768928527832, "kl": 1.7415275797247887, "learning_rate": 9.70997025715128e-06, "loss": -0.0031, "num_tokens": 6557822.0, "reward": 0.046875, "reward_std": 0.35037124156951904, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.3491781949996948, "sampling/importance_sampling_ratio/max": 1.88459312915802, "sampling/importance_sampling_ratio/mean": 0.7528318166732788, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7870888710021973, "sampling/sampling_logp_difference/mean": 0.19711831212043762, "step": 145, "step_time": 14.157950498996797 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 308.4375, "completions/mean_terminated_length": 308.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22140651615336537, "epoch": 0.00292, "frac_reward_zero_std": 0.0, "grad_norm": 2.196427345275879, "kl": 3.6350285410881042, "learning_rate": 9.704695484660736e-06, "loss": -0.0129, "num_tokens": 6607947.0, "reward": 0.13750000298023224, "reward_std": 0.26322561502456665, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.28026482462882996, "sampling/importance_sampling_ratio/max": 1.0686298608779907, "sampling/importance_sampling_ratio/mean": 0.6130638122558594, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1169629096984863, "sampling/sampling_logp_difference/mean": 0.21412548422813416, "step": 146, "step_time": 14.102497750005568 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 289.90625, "completions/mean_terminated_length": 289.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.210488160722889, "epoch": 0.00294, "frac_reward_zero_std": 0.0, "grad_norm": 0.6185306310653687, "kl": 1.2037382163107395, "learning_rate": 9.699375142401365e-06, "loss": -0.0093, "num_tokens": 6655261.0, "reward": 0.0625, "reward_std": 0.2844155728816986, "rewards/rollout_reward_func/mean": 0.0625, "rewards/rollout_reward_func/std": 0.2859618663787842, "sampling/importance_sampling_ratio/max": 2.4925453662872314, "sampling/importance_sampling_ratio/mean": 0.8286665678024292, "sampling/importance_sampling_ratio/min": 0.13525161147117615, "sampling/sampling_logp_difference/max": 1.8772261142730713, "sampling/sampling_logp_difference/mean": 0.2358148992061615, "step": 147, "step_time": 14.911731973013957 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 309.125, "completions/mean_terminated_length": 309.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19678416289389133, "epoch": 0.00296, "frac_reward_zero_std": 0.0, "grad_norm": 1.7599304914474487, "kl": 1.2553344387561083, "learning_rate": 9.694009300552081e-06, "loss": 0.0082, "num_tokens": 6704946.0, "reward": 0.1875, "reward_std": 0.2543007731437683, "rewards/rollout_reward_func/mean": 0.1875, "rewards/rollout_reward_func/std": 0.2648797631263733, "sampling/importance_sampling_ratio/max": 1.9119799137115479, "sampling/importance_sampling_ratio/mean": 0.8234103918075562, "sampling/importance_sampling_ratio/min": 0.19480614364147186, "sampling/sampling_logp_difference/max": 1.4273314476013184, "sampling/sampling_logp_difference/mean": 0.15645036101341248, "step": 148, "step_time": 13.975181482997868 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.014322916977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.022135416977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 269.09375, "completions/mean_terminated_length": 269.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2025472135283053, "epoch": 0.00298, "frac_reward_zero_std": 0.0, "grad_norm": 1.6659015417099, "kl": 1.5771758370101452, "learning_rate": 9.68859802989196e-06, "loss": 0.023, "num_tokens": 6752963.0, "reward": 0.15937501192092896, "reward_std": 0.2668197453022003, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.28268226981163025, "sampling/importance_sampling_ratio/max": 2.645512580871582, "sampling/importance_sampling_ratio/mean": 0.8301820755004883, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0940420627593994, "sampling/sampling_logp_difference/mean": 0.21342727541923523, "step": 149, "step_time": 14.15561429900481 }, { "clip_ratio/high_max": 0.026041666977107525, "clip_ratio/high_mean": 0.013020833488553762, "clip_ratio/low_mean": 0.020833333488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.033854166977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1306.0, "completions/max_terminated_length": 1306.0, "completions/mean_length": 300.53125, "completions/mean_terminated_length": 300.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1726773357950151, "epoch": 0.003, "frac_reward_zero_std": 0.0, "grad_norm": 0.68708735704422, "kl": 1.84360196813941, "learning_rate": 9.683141401799326e-06, "loss": 0.0087, "num_tokens": 6803077.0, "reward": 0.13124999403953552, "reward_std": 0.20611491799354553, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.21165630221366882, "sampling/importance_sampling_ratio/max": 2.911655902862549, "sampling/importance_sampling_ratio/mean": 0.7750325202941895, "sampling/importance_sampling_ratio/min": 0.11080058664083481, "sampling/sampling_logp_difference/max": 2.228201150894165, "sampling/sampling_logp_difference/mean": 0.26136699318885803, "step": 150, "step_time": 14.949423336009204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 217.53125, "completions/mean_terminated_length": 217.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15382543415762484, "epoch": 0.00302, "frac_reward_zero_std": 0.0, "grad_norm": 0.3436124920845032, "kl": 1.9322837442159653, "learning_rate": 9.677639488250787e-06, "loss": 0.0074, "num_tokens": 6848303.0, "reward": 0.0625000074505806, "reward_std": 0.2804763913154602, "rewards/rollout_reward_func/mean": 0.0625000074505806, "rewards/rollout_reward_func/std": 0.2859618663787842, "sampling/importance_sampling_ratio/max": 1.391569972038269, "sampling/importance_sampling_ratio/mean": 0.7319640517234802, "sampling/importance_sampling_ratio/min": 0.09121346473693848, "sampling/sampling_logp_difference/max": 2.1626040935516357, "sampling/sampling_logp_difference/mean": 0.21542418003082275, "step": 151, "step_time": 13.410987963005027 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625000465661287, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 278.8125, "completions/mean_terminated_length": 278.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1824072195449844, "epoch": 0.00304, "frac_reward_zero_std": 0.0, "grad_norm": 1.0726398229599, "kl": 1.3967003133147955, "learning_rate": 9.672092361820306e-06, "loss": -0.0449, "num_tokens": 6896343.0, "reward": 0.12812499701976776, "reward_std": 0.25518667697906494, "rewards/rollout_reward_func/mean": 0.12812499701976776, "rewards/rollout_reward_func/std": 0.2465692013502121, "sampling/importance_sampling_ratio/max": 2.427722454071045, "sampling/importance_sampling_ratio/mean": 0.7812963724136353, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9475913047790527, "sampling/sampling_logp_difference/mean": 0.2332400679588318, "step": 152, "step_time": 14.365059988998837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 838.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 247.65625, "completions/mean_terminated_length": 247.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16078451741486788, "epoch": 0.00306, "frac_reward_zero_std": 0.0, "grad_norm": 0.3795492947101593, "kl": 1.4351594429463148, "learning_rate": 9.666500095678226e-06, "loss": -0.0041, "num_tokens": 6944603.0, "reward": 0.1875, "reward_std": 0.21980419754981995, "rewards/rollout_reward_func/mean": 0.1875, "rewards/rollout_reward_func/std": 0.22252219915390015, "sampling/importance_sampling_ratio/max": 2.7323594093322754, "sampling/importance_sampling_ratio/mean": 0.9324585199356079, "sampling/importance_sampling_ratio/min": 0.14144718647003174, "sampling/sampling_logp_difference/max": 1.5449787378311157, "sampling/sampling_logp_difference/mean": 0.17155314981937408, "step": 153, "step_time": 12.449936737008102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 320.125, "completions/mean_terminated_length": 320.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26893932232633233, "epoch": 0.00308, "frac_reward_zero_std": 0.0, "grad_norm": 326.888671875, "kl": 130.52419202774763, "learning_rate": 9.660862763590322e-06, "loss": 0.5657, "num_tokens": 6994250.0, "reward": 0.11875000596046448, "reward_std": 0.2988477945327759, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.3073691725730896, "sampling/importance_sampling_ratio/max": 2.072385787963867, "sampling/importance_sampling_ratio/mean": 0.7162079215049744, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.745759963989258, "sampling/sampling_logp_difference/mean": 0.22617614269256592, "step": 154, "step_time": 14.279265805998875 }, { "clip_ratio/high_max": 0.0062500000931322575, "clip_ratio/high_mean": 0.0031250000465661287, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01875000004656613, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 289.09375, "completions/mean_terminated_length": 289.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23145774425938725, "epoch": 0.0031, "frac_reward_zero_std": 0.0, "grad_norm": 0.42780861258506775, "kl": 2.939034454524517, "learning_rate": 9.655180439916814e-06, "loss": -0.0505, "num_tokens": 7043093.0, "reward": 0.13124999403953552, "reward_std": 0.22432835400104523, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.2693331241607666, "sampling/importance_sampling_ratio/max": 2.4796576499938965, "sampling/importance_sampling_ratio/mean": 0.7207521200180054, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8255071640014648, "sampling/sampling_logp_difference/mean": 0.18942058086395264, "step": 155, "step_time": 14.098653884000669 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 315.5625, "completions/mean_terminated_length": 315.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17382641322910786, "epoch": 0.00312, "frac_reward_zero_std": 0.0, "grad_norm": 0.6347624659538269, "kl": 1.5241753719747066, "learning_rate": 9.64945319961139e-06, "loss": 0.019, "num_tokens": 7093940.0, "reward": 0.13750000298023224, "reward_std": 0.2638257145881653, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.27327170968055725, "sampling/importance_sampling_ratio/max": 2.759106397628784, "sampling/importance_sampling_ratio/mean": 0.9290750026702881, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8566176891326904, "sampling/sampling_logp_difference/mean": 0.1849888265132904, "step": 156, "step_time": 13.647398387995054 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1040.0, "completions/max_terminated_length": 1040.0, "completions/mean_length": 224.3125, "completions/mean_terminated_length": 224.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11396455811336637, "epoch": 0.00314, "frac_reward_zero_std": 0.0, "grad_norm": 0.5438618063926697, "kl": 0.8419274110347033, "learning_rate": 9.643681118220224e-06, "loss": -0.0176, "num_tokens": 7139199.0, "reward": 0.18437500298023224, "reward_std": 0.28891855478286743, "rewards/rollout_reward_func/mean": 0.18437500298023224, "rewards/rollout_reward_func/std": 0.2863669991493225, "sampling/importance_sampling_ratio/max": 1.196180820465088, "sampling/importance_sampling_ratio/mean": 0.8839966058731079, "sampling/importance_sampling_ratio/min": 0.15871132910251617, "sampling/sampling_logp_difference/max": 1.519705057144165, "sampling/sampling_logp_difference/mean": 0.1235043853521347, "step": 157, "step_time": 13.369121443985932 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 210.71875, "completions/mean_terminated_length": 210.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10377447318751365, "epoch": 0.00316, "frac_reward_zero_std": 0.0, "grad_norm": 0.6939066648483276, "kl": 0.6991895660758018, "learning_rate": 9.637864271880972e-06, "loss": -0.0054, "num_tokens": 7183304.0, "reward": 0.1875, "reward_std": 0.2274293601512909, "rewards/rollout_reward_func/mean": 0.1875, "rewards/rollout_reward_func/std": 0.23928657174110413, "sampling/importance_sampling_ratio/max": 1.0723936557769775, "sampling/importance_sampling_ratio/mean": 0.8774169683456421, "sampling/importance_sampling_ratio/min": 0.2283279150724411, "sampling/sampling_logp_difference/max": 1.2478454113006592, "sampling/sampling_logp_difference/mean": 0.08585622906684875, "step": 158, "step_time": 12.358612280993839 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 261.78125, "completions/mean_terminated_length": 261.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17009137803688645, "epoch": 0.00318, "frac_reward_zero_std": 0.0, "grad_norm": 1.36485755443573, "kl": 0.8708535451442003, "learning_rate": 9.632002737321768e-06, "loss": 0.0466, "num_tokens": 7230781.0, "reward": 0.10625000298023224, "reward_std": 0.3302614092826843, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.31718266010284424, "sampling/importance_sampling_ratio/max": 1.8765630722045898, "sampling/importance_sampling_ratio/mean": 0.8999338150024414, "sampling/importance_sampling_ratio/min": 0.055254995822906494, "sampling/sampling_logp_difference/max": 1.6177525520324707, "sampling/sampling_logp_difference/mean": 0.13534794747829437, "step": 159, "step_time": 14.173388347007858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 273.46875, "completions/mean_terminated_length": 273.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14514031680300832, "epoch": 0.0032, "frac_reward_zero_std": 0.0, "grad_norm": 1.5327736139297485, "kl": 1.1388291269540787, "learning_rate": 9.626096591860215e-06, "loss": 0.127, "num_tokens": 7278506.0, "reward": 0.09687501192092896, "reward_std": 0.2772330641746521, "rewards/rollout_reward_func/mean": 0.09687501192092896, "rewards/rollout_reward_func/std": 0.3177383542060852, "sampling/importance_sampling_ratio/max": 2.6540112495422363, "sampling/importance_sampling_ratio/mean": 1.045914888381958, "sampling/importance_sampling_ratio/min": 0.21409763395786285, "sampling/sampling_logp_difference/max": 1.4779505729675293, "sampling/sampling_logp_difference/mean": 0.1257515251636505, "step": 160, "step_time": 14.416122346003249 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 216.34375, "completions/mean_terminated_length": 216.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12850235449150205, "epoch": 0.00322, "frac_reward_zero_std": 0.0, "grad_norm": 0.6101768612861633, "kl": 1.1634297855198383, "learning_rate": 9.620145913402366e-06, "loss": 0.002, "num_tokens": 7324728.0, "reward": 0.15312500298023224, "reward_std": 0.22592151165008545, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.24624192714691162, "sampling/importance_sampling_ratio/max": 2.484584331512451, "sampling/importance_sampling_ratio/mean": 0.839599609375, "sampling/importance_sampling_ratio/min": 0.1387995034456253, "sampling/sampling_logp_difference/max": 2.223395347595215, "sampling/sampling_logp_difference/mean": 0.18347331881523132, "step": 161, "step_time": 13.113501701001951 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 184.5, "completions/mean_terminated_length": 184.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11261867173016071, "epoch": 0.00324, "frac_reward_zero_std": 0.0, "grad_norm": 0.7654638886451721, "kl": 2.4393042344599962, "learning_rate": 9.614150780441687e-06, "loss": -0.018, "num_tokens": 7369607.0, "reward": 0.15312501788139343, "reward_std": 0.26691389083862305, "rewards/rollout_reward_func/mean": 0.15312501788139343, "rewards/rollout_reward_func/std": 0.2577625513076782, "sampling/importance_sampling_ratio/max": 2.4842371940612793, "sampling/importance_sampling_ratio/mean": 0.9260571002960205, "sampling/importance_sampling_ratio/min": 0.03340744972229004, "sampling/sampling_logp_difference/max": 2.1452789306640625, "sampling/sampling_logp_difference/mean": 0.19428101181983948, "step": 162, "step_time": 11.506142408001324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 271.5, "completions/mean_terminated_length": 271.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12228143424727023, "epoch": 0.00326, "frac_reward_zero_std": 0.0, "grad_norm": 0.6646633148193359, "kl": 2.8803985752165318, "learning_rate": 9.608111272058044e-06, "loss": -0.0252, "num_tokens": 7417590.0, "reward": 0.13750000298023224, "reward_std": 0.2560737729072571, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.27562248706817627, "sampling/importance_sampling_ratio/max": 1.1669354438781738, "sampling/importance_sampling_ratio/mean": 0.5836501121520996, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5084948539733887, "sampling/sampling_logp_difference/mean": 0.30290961265563965, "step": 163, "step_time": 14.000113649999548 }, { "clip_ratio/high_max": 0.026041666977107525, "clip_ratio/high_mean": 0.013020833488553762, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.026041666977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 313.40625, "completions/mean_terminated_length": 313.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12487253604922444, "epoch": 0.00328, "frac_reward_zero_std": 0.0, "grad_norm": 1.873429536819458, "kl": 2.1374831330031157, "learning_rate": 9.602027467916626e-06, "loss": 0.0468, "num_tokens": 7467648.0, "reward": 0.15000000596046448, "reward_std": 0.21180370450019836, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.2094539850950241, "sampling/importance_sampling_ratio/max": 2.7809698581695557, "sampling/importance_sampling_ratio/mean": 0.8847783803939819, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4665863513946533, "sampling/sampling_logp_difference/mean": 0.24712061882019043, "step": 164, "step_time": 12.401035503997264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 335.53125, "completions/mean_terminated_length": 335.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16405932244379073, "epoch": 0.0033, "frac_reward_zero_std": 0.0, "grad_norm": 2.80324125289917, "kl": 1.1774210259318352, "learning_rate": 9.595899448266928e-06, "loss": 0.0742, "num_tokens": 7517074.0, "reward": 0.16875000298023224, "reward_std": 0.29094383120536804, "rewards/rollout_reward_func/mean": 0.16875000298023224, "rewards/rollout_reward_func/std": 0.29885533452033997, "sampling/importance_sampling_ratio/max": 2.965005874633789, "sampling/importance_sampling_ratio/mean": 0.9595487713813782, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7190322875976562, "sampling/sampling_logp_difference/mean": 0.12872225046157837, "step": 165, "step_time": 14.486102404996927 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014322916977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1157.0, "completions/max_terminated_length": 1157.0, "completions/mean_length": 288.0, "completions/mean_terminated_length": 288.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11544691130984575, "epoch": 0.00332, "frac_reward_zero_std": 0.0, "grad_norm": 1.64107084274292, "kl": 3.8624702766537666, "learning_rate": 9.589727293941669e-06, "loss": -0.0311, "num_tokens": 7565476.0, "reward": 0.11250001192092896, "reward_std": 0.23548483848571777, "rewards/rollout_reward_func/mean": 0.11250001192092896, "rewards/rollout_reward_func/std": 0.2432972490787506, "sampling/importance_sampling_ratio/max": 2.6161201000213623, "sampling/importance_sampling_ratio/mean": 0.8819030523300171, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1917285919189453, "sampling/sampling_logp_difference/mean": 0.2075578272342682, "step": 166, "step_time": 14.18132161300673 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.015424679731950164, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.031049679731950164, "completions/clipped_ratio": 0.0, "completions/max_length": 1270.0, "completions/max_terminated_length": 1270.0, "completions/mean_length": 306.40625, "completions/mean_terminated_length": 306.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17656514793634415, "epoch": 0.00334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3148010969161987, "kl": 8.651863103732467, "learning_rate": 9.583511086355738e-06, "loss": -0.0371, "num_tokens": 7616246.0, "reward": 0.10625000298023224, "reward_std": 0.31399303674697876, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.31411677598953247, "sampling/importance_sampling_ratio/max": 2.2226598262786865, "sampling/importance_sampling_ratio/mean": 0.7440073490142822, "sampling/importance_sampling_ratio/min": 0.07040540128946304, "sampling/sampling_logp_difference/max": 1.871259093284607, "sampling/sampling_logp_difference/mean": 0.19927307963371277, "step": 167, "step_time": 15.067691738000576 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1279.0, "completions/max_terminated_length": 1279.0, "completions/mean_length": 369.875, "completions/mean_terminated_length": 369.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13464022241532803, "epoch": 0.00336, "frac_reward_zero_std": 0.0, "grad_norm": 0.8498160243034363, "kl": 1.5777826346457005, "learning_rate": 9.577250907505117e-06, "loss": 0.0481, "num_tokens": 7668116.0, "reward": 0.08750000596046448, "reward_std": 0.2406546026468277, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.24196773767471313, "sampling/importance_sampling_ratio/max": 2.3029370307922363, "sampling/importance_sampling_ratio/mean": 0.9457173347473145, "sampling/importance_sampling_ratio/min": 0.10222196578979492, "sampling/sampling_logp_difference/max": 1.6556226015090942, "sampling/sampling_logp_difference/mean": 0.16433356702327728, "step": 168, "step_time": 14.742136514003505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 242.34375, "completions/mean_terminated_length": 242.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12318800284992903, "epoch": 0.00338, "frac_reward_zero_std": 0.0, "grad_norm": 0.5126773715019226, "kl": 1.0167458597570658, "learning_rate": 9.570946839965795e-06, "loss": 0.0122, "num_tokens": 7714363.0, "reward": 0.1875, "reward_std": 0.2635037302970886, "rewards/rollout_reward_func/mean": 0.1875, "rewards/rollout_reward_func/std": 0.25368255376815796, "sampling/importance_sampling_ratio/max": 2.741812229156494, "sampling/importance_sampling_ratio/mean": 0.8840235471725464, "sampling/importance_sampling_ratio/min": 0.1587783843278885, "sampling/sampling_logp_difference/max": 1.8644360303878784, "sampling/sampling_logp_difference/mean": 0.1474263221025467, "step": 169, "step_time": 14.451571635006985 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 250.9375, "completions/mean_terminated_length": 250.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1483839936554432, "epoch": 0.0034, "frac_reward_zero_std": 0.0, "grad_norm": 2.5171029567718506, "kl": 16.871450755745173, "learning_rate": 9.564598966892683e-06, "loss": 0.0202, "num_tokens": 7762721.0, "reward": 0.04375000298023224, "reward_std": 0.2488172948360443, "rewards/rollout_reward_func/mean": 0.04375000298023224, "rewards/rollout_reward_func/std": 0.2906472086906433, "sampling/importance_sampling_ratio/max": 2.5922279357910156, "sampling/importance_sampling_ratio/mean": 0.8067804574966431, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5790648460388184, "sampling/sampling_logp_difference/mean": 0.1859809309244156, "step": 170, "step_time": 15.357304110006226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010937500046566129, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010937500046566129, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 331.78125, "completions/mean_terminated_length": 331.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14884609077125788, "epoch": 0.00342, "frac_reward_zero_std": 0.0, "grad_norm": 0.9730277061462402, "kl": 1.0313560888171196, "learning_rate": 9.558207372018519e-06, "loss": 0.1134, "num_tokens": 7813875.0, "reward": 0.05625000596046448, "reward_std": 0.33052217960357666, "rewards/rollout_reward_func/mean": 0.05625000596046448, "rewards/rollout_reward_func/std": 0.32122498750686646, "sampling/importance_sampling_ratio/max": 2.794945478439331, "sampling/importance_sampling_ratio/mean": 0.9486004114151001, "sampling/importance_sampling_ratio/min": 0.20351487398147583, "sampling/sampling_logp_difference/max": 1.1088135242462158, "sampling/sampling_logp_difference/mean": 0.13175225257873535, "step": 171, "step_time": 14.044304487008048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 310.5, "completions/mean_terminated_length": 310.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1541601475328207, "epoch": 0.00344, "frac_reward_zero_std": 0.0, "grad_norm": 0.9217245578765869, "kl": 0.7917409725487232, "learning_rate": 9.551772139652762e-06, "loss": 0.0174, "num_tokens": 7862956.0, "reward": 0.16250000894069672, "reward_std": 0.238946795463562, "rewards/rollout_reward_func/mean": 0.16250000894069672, "rewards/rollout_reward_func/std": 0.25368255376815796, "sampling/importance_sampling_ratio/max": 1.3028043508529663, "sampling/importance_sampling_ratio/mean": 0.8176382184028625, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6808195114135742, "sampling/sampling_logp_difference/mean": 0.10893665254116058, "step": 172, "step_time": 14.673083646997839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 328.8125, "completions/mean_terminated_length": 328.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.216947378590703, "epoch": 0.00346, "frac_reward_zero_std": 0.0, "grad_norm": 1.5225436687469482, "kl": 0.8954977570101619, "learning_rate": 9.545293354680477e-06, "loss": 0.0725, "num_tokens": 7913857.0, "reward": 0.046875, "reward_std": 0.2875579297542572, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.3026595711708069, "sampling/importance_sampling_ratio/max": 2.6406795978546143, "sampling/importance_sampling_ratio/mean": 0.8760957717895508, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5515458583831787, "sampling/sampling_logp_difference/mean": 0.1215914934873581, "step": 173, "step_time": 14.772172845005116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1787.0, "completions/max_terminated_length": 1787.0, "completions/mean_length": 191.53125, "completions/mean_terminated_length": 191.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15690614469349384, "epoch": 0.00348, "frac_reward_zero_std": 0.0, "grad_norm": 0.500063955783844, "kl": 3.2765282839536667, "learning_rate": 9.538771102561212e-06, "loss": 0.0145, "num_tokens": 7958220.0, "reward": 0.15937499701976776, "reward_std": 0.25333893299102783, "rewards/rollout_reward_func/mean": 0.15937499701976776, "rewards/rollout_reward_func/std": 0.2960594594478607, "sampling/importance_sampling_ratio/max": 1.8603215217590332, "sampling/importance_sampling_ratio/mean": 0.9075891375541687, "sampling/importance_sampling_ratio/min": 0.293429434299469, "sampling/sampling_logp_difference/max": 0.9723851680755615, "sampling/sampling_logp_difference/mean": 0.09444539248943329, "step": 174, "step_time": 17.211958315998345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 341.0625, "completions/mean_terminated_length": 341.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19634076883085072, "epoch": 0.0035, "frac_reward_zero_std": 0.0, "grad_norm": 0.8093025088310242, "kl": 1.084115281701088, "learning_rate": 9.53220546932789e-06, "loss": -0.0087, "num_tokens": 8007929.0, "reward": 0.078125, "reward_std": 0.33227401971817017, "rewards/rollout_reward_func/mean": 0.078125, "rewards/rollout_reward_func/std": 0.331890344619751, "sampling/importance_sampling_ratio/max": 2.0289461612701416, "sampling/importance_sampling_ratio/mean": 0.8274521827697754, "sampling/importance_sampling_ratio/min": 0.20284728705883026, "sampling/sampling_logp_difference/max": 1.3421025276184082, "sampling/sampling_logp_difference/mean": 0.12264728546142578, "step": 175, "step_time": 13.913908665985218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 256.40625, "completions/mean_terminated_length": 256.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18619238305836916, "epoch": 0.00352, "frac_reward_zero_std": 0.0, "grad_norm": 0.3161636292934418, "kl": 1.67344082146883, "learning_rate": 9.525596541585646e-06, "loss": 0.0252, "num_tokens": 8055810.0, "reward": 0.06875000149011612, "reward_std": 0.22688721120357513, "rewards/rollout_reward_func/mean": 0.06875000149011612, "rewards/rollout_reward_func/std": 0.2705281674861908, "sampling/importance_sampling_ratio/max": 2.337167263031006, "sampling/importance_sampling_ratio/mean": 0.9322199821472168, "sampling/importance_sampling_ratio/min": 0.24554532766342163, "sampling/sampling_logp_difference/max": 0.975318431854248, "sampling/sampling_logp_difference/mean": 0.12694187462329865, "step": 176, "step_time": 11.67123813299986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 286.59375, "completions/mean_terminated_length": 286.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2054314180277288, "epoch": 0.00354, "frac_reward_zero_std": 0.0, "grad_norm": 0.46454936265945435, "kl": 1.4305258886888623, "learning_rate": 9.518944406510707e-06, "loss": 0.0025, "num_tokens": 8104716.0, "reward": 0.11562500894069672, "reward_std": 0.24394547939300537, "rewards/rollout_reward_func/mean": 0.11562500894069672, "rewards/rollout_reward_func/std": 0.23706454038619995, "sampling/importance_sampling_ratio/max": 1.81676185131073, "sampling/importance_sampling_ratio/mean": 0.9273830652236938, "sampling/importance_sampling_ratio/min": 0.10416346043348312, "sampling/sampling_logp_difference/max": 1.7945609092712402, "sampling/sampling_logp_difference/mean": 0.11761051416397095, "step": 177, "step_time": 14.670890313009295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 284.1875, "completions/mean_terminated_length": 284.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26886838348582387, "epoch": 0.00356, "frac_reward_zero_std": 0.0, "grad_norm": 0.8588820695877075, "kl": 2.894058257341385, "learning_rate": 9.512249151849229e-06, "loss": -0.0295, "num_tokens": 8153906.0, "reward": 0.15000000596046448, "reward_std": 0.30666595697402954, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.31927335262298584, "sampling/importance_sampling_ratio/max": 2.9352781772613525, "sampling/importance_sampling_ratio/mean": 0.9891278743743896, "sampling/importance_sampling_ratio/min": 7.16514314547112e-09, "sampling/sampling_logp_difference/max": 18.646686553955078, "sampling/sampling_logp_difference/mean": 0.2594527006149292, "step": 178, "step_time": 13.587237147999986 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 287.96875, "completions/mean_terminated_length": 287.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18727659550495446, "epoch": 0.00358, "frac_reward_zero_std": 0.0, "grad_norm": 0.64882892370224, "kl": 0.6643412336707115, "learning_rate": 9.50551086591615e-06, "loss": -0.0139, "num_tokens": 8202334.0, "reward": 0.13124999403953552, "reward_std": 0.21503466367721558, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.21317492425441742, "sampling/importance_sampling_ratio/max": 1.2239248752593994, "sampling/importance_sampling_ratio/mean": 0.8328835964202881, "sampling/importance_sampling_ratio/min": 0.4253069758415222, "sampling/sampling_logp_difference/max": 0.9409119486808777, "sampling/sampling_logp_difference/mean": 0.11253772675991058, "step": 179, "step_time": 14.818051664002269 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009114583488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 339.375, "completions/mean_terminated_length": 339.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2421749159693718, "epoch": 0.0036, "frac_reward_zero_std": 0.0, "grad_norm": 0.8685379028320312, "kl": 1.3078749682754278, "learning_rate": 9.498729637594016e-06, "loss": -0.0383, "num_tokens": 8253480.0, "reward": 0.046875, "reward_std": 0.324290931224823, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.32526353001594543, "sampling/importance_sampling_ratio/max": 2.8200857639312744, "sampling/importance_sampling_ratio/mean": 0.8346470594406128, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.337226390838623, "sampling/sampling_logp_difference/mean": 0.1490808129310608, "step": 180, "step_time": 14.249378592001449 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 355.375, "completions/mean_terminated_length": 355.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20783221069723368, "epoch": 0.00362, "frac_reward_zero_std": 0.0, "grad_norm": 0.683068573474884, "kl": 1.4966665338724852, "learning_rate": 9.491905556331809e-06, "loss": -0.0172, "num_tokens": 8304961.0, "reward": 0.02499999850988388, "reward_std": 0.36572888493537903, "rewards/rollout_reward_func/mean": 0.02499999850988388, "rewards/rollout_reward_func/std": 0.3610021770000458, "sampling/importance_sampling_ratio/max": 1.9433902502059937, "sampling/importance_sampling_ratio/mean": 0.8584706783294678, "sampling/importance_sampling_ratio/min": 0.11236516386270523, "sampling/sampling_logp_difference/max": 1.2978825569152832, "sampling/sampling_logp_difference/mean": 0.14825409650802612, "step": 181, "step_time": 14.40429868699357 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1049.0, "completions/max_terminated_length": 1049.0, "completions/mean_length": 311.28125, "completions/mean_terminated_length": 311.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23070408194325864, "epoch": 0.00364, "frac_reward_zero_std": 0.0, "grad_norm": 0.6098201274871826, "kl": 1.5258457846939564, "learning_rate": 9.485038712143778e-06, "loss": -0.0596, "num_tokens": 8355450.0, "reward": 0.06875000894069672, "reward_std": 0.3265150785446167, "rewards/rollout_reward_func/mean": 0.06875000894069672, "rewards/rollout_reward_func/std": 0.32867035269737244, "sampling/importance_sampling_ratio/max": 1.8749408721923828, "sampling/importance_sampling_ratio/mean": 1.0045604705810547, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1683778762817383, "sampling/sampling_logp_difference/mean": 0.12254594266414642, "step": 182, "step_time": 14.376946486991073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1148.0, "completions/max_terminated_length": 1148.0, "completions/mean_length": 319.9375, "completions/mean_terminated_length": 319.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19549343595281243, "epoch": 0.00366, "frac_reward_zero_std": 0.0, "grad_norm": 2.196779727935791, "kl": 1.8154857773333788, "learning_rate": 9.478129195608238e-06, "loss": 0.058, "num_tokens": 8406572.0, "reward": 0.16562500596046448, "reward_std": 0.299069881439209, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.296875536441803, "sampling/importance_sampling_ratio/max": 2.130833148956299, "sampling/importance_sampling_ratio/mean": 1.092668056488037, "sampling/importance_sampling_ratio/min": 0.20379389822483063, "sampling/sampling_logp_difference/max": 1.445888638496399, "sampling/sampling_logp_difference/mean": 0.08730460703372955, "step": 183, "step_time": 14.228367832001823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1074.0, "completions/max_terminated_length": 1074.0, "completions/mean_length": 248.25, "completions/mean_terminated_length": 248.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17697493243031204, "epoch": 0.00368, "frac_reward_zero_std": 0.0, "grad_norm": 0.6199963092803955, "kl": 2.09570848941803, "learning_rate": 9.471177097866384e-06, "loss": 0.0263, "num_tokens": 8453711.0, "reward": 0.12812501192092896, "reward_std": 0.2529112696647644, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.2530260384082794, "sampling/importance_sampling_ratio/max": 1.404966950416565, "sampling/importance_sampling_ratio/mean": 0.9258087277412415, "sampling/importance_sampling_ratio/min": 0.20827965438365936, "sampling/sampling_logp_difference/max": 1.6009876728057861, "sampling/sampling_logp_difference/mean": 0.11607725918292999, "step": 184, "step_time": 13.976940421995096 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 885.0, "completions/max_terminated_length": 885.0, "completions/mean_length": 198.1875, "completions/mean_terminated_length": 198.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14655566099099815, "epoch": 0.0037, "frac_reward_zero_std": 0.0, "grad_norm": 0.6076487302780151, "kl": 4.162005508318543, "learning_rate": 9.464182510621085e-06, "loss": -0.0215, "num_tokens": 8498630.0, "reward": 0.12187500298023224, "reward_std": 0.2901068925857544, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.28481388092041016, "sampling/importance_sampling_ratio/max": 1.2797880172729492, "sampling/importance_sampling_ratio/mean": 0.8600012063980103, "sampling/importance_sampling_ratio/min": 0.16755259037017822, "sampling/sampling_logp_difference/max": 1.8595612049102783, "sampling/sampling_logp_difference/mean": 0.11362086236476898, "step": 185, "step_time": 12.850317891006853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1103.0, "completions/max_terminated_length": 1103.0, "completions/mean_length": 354.09375, "completions/mean_terminated_length": 354.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17531941970810294, "epoch": 0.00372, "frac_reward_zero_std": 0.0, "grad_norm": 0.7887911796569824, "kl": 2.967575753107667, "learning_rate": 9.457145526135673e-06, "loss": 0.0255, "num_tokens": 8550734.0, "reward": 0.11874999850988388, "reward_std": 0.22170543670654297, "rewards/rollout_reward_func/mean": 0.11874999850988388, "rewards/rollout_reward_func/std": 0.2558193504810333, "sampling/importance_sampling_ratio/max": 1.5071123838424683, "sampling/importance_sampling_ratio/mean": 0.9147425889968872, "sampling/importance_sampling_ratio/min": 0.1571810096502304, "sampling/sampling_logp_difference/max": 1.8762764930725098, "sampling/sampling_logp_difference/mean": 0.08050180971622467, "step": 186, "step_time": 13.969641711992153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1258.0, "completions/max_terminated_length": 1258.0, "completions/mean_length": 133.0625, "completions/mean_terminated_length": 133.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26925342762842774, "epoch": 0.00374, "frac_reward_zero_std": 0.0, "grad_norm": 2.885507345199585, "kl": 0.9689754731953144, "learning_rate": 9.450066237232736e-06, "loss": 0.1465, "num_tokens": 8591659.0, "reward": 0.19687500596046448, "reward_std": 0.2346336394548416, "rewards/rollout_reward_func/mean": 0.19687500596046448, "rewards/rollout_reward_func/std": 0.24689604341983795, "sampling/importance_sampling_ratio/max": 2.4152746200561523, "sampling/importance_sampling_ratio/mean": 1.0142945051193237, "sampling/importance_sampling_ratio/min": 0.1915445625782013, "sampling/sampling_logp_difference/max": 1.652350902557373, "sampling/sampling_logp_difference/mean": 0.05852821096777916, "step": 187, "step_time": 14.57681800399223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1643.0, "completions/max_terminated_length": 1643.0, "completions/mean_length": 230.25, "completions/mean_terminated_length": 230.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16899000853300095, "epoch": 0.00376, "frac_reward_zero_std": 0.0, "grad_norm": 0.5210801959037781, "kl": 2.505873713642359, "learning_rate": 9.442944737292872e-06, "loss": 0.0079, "num_tokens": 8637686.0, "reward": 0.140625, "reward_std": 0.23805536329746246, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.24738553166389465, "sampling/importance_sampling_ratio/max": 1.3841135501861572, "sampling/importance_sampling_ratio/mean": 0.9794542789459229, "sampling/importance_sampling_ratio/min": 0.261272668838501, "sampling/sampling_logp_difference/max": 1.3886620998382568, "sampling/sampling_logp_difference/mean": 0.06293951719999313, "step": 188, "step_time": 16.86939104799967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1146.0, "completions/max_terminated_length": 1146.0, "completions/mean_length": 298.40625, "completions/mean_terminated_length": 298.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19784045033156872, "epoch": 0.00378, "frac_reward_zero_std": 0.0, "grad_norm": 1.4076751470565796, "kl": 1.9665896501392126, "learning_rate": 9.435781120253486e-06, "loss": 0.0781, "num_tokens": 8686620.0, "reward": 0.16249999403953552, "reward_std": 0.27077269554138184, "rewards/rollout_reward_func/mean": 0.16249999403953552, "rewards/rollout_reward_func/std": 0.27090054750442505, "sampling/importance_sampling_ratio/max": 1.8493090867996216, "sampling/importance_sampling_ratio/mean": 0.9657564163208008, "sampling/importance_sampling_ratio/min": 0.2330119013786316, "sampling/sampling_logp_difference/max": 1.6303448677062988, "sampling/sampling_logp_difference/mean": 0.08504491299390793, "step": 189, "step_time": 13.90557545300544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.012369791977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012369791977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1840.0, "completions/max_terminated_length": 1840.0, "completions/mean_length": 357.25, "completions/mean_terminated_length": 357.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2026096093468368, "epoch": 0.0038, "frac_reward_zero_std": 0.0, "grad_norm": 0.44598522782325745, "kl": 1.8524528099223971, "learning_rate": 9.428575480607526e-06, "loss": 0.0033, "num_tokens": 8737107.0, "reward": 0.07500000298023224, "reward_std": 0.26133283972740173, "rewards/rollout_reward_func/mean": 0.07500000298023224, "rewards/rollout_reward_func/std": 0.27356666326522827, "sampling/importance_sampling_ratio/max": 1.60148286819458, "sampling/importance_sampling_ratio/mean": 0.9338177442550659, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.683457851409912, "sampling/sampling_logp_difference/mean": 0.09636282920837402, "step": 190, "step_time": 17.432547977005015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0031250000465661287, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0031250000465661287, "completions/clipped_ratio": 0.0, "completions/max_length": 1209.0, "completions/max_terminated_length": 1209.0, "completions/mean_length": 300.90625, "completions/mean_terminated_length": 300.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19847405375912786, "epoch": 0.00382, "frac_reward_zero_std": 0.0, "grad_norm": 1.8472954034805298, "kl": 1.7226303331553936, "learning_rate": 9.421327913402252e-06, "loss": 0.1105, "num_tokens": 8785772.0, "reward": 0.12812501192092896, "reward_std": 0.26660919189453125, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.261798232793808, "sampling/importance_sampling_ratio/max": 2.8829479217529297, "sampling/importance_sampling_ratio/mean": 1.0600647926330566, "sampling/importance_sampling_ratio/min": 0.21952277421951294, "sampling/sampling_logp_difference/max": 1.5644094944000244, "sampling/sampling_logp_difference/mean": 0.09527671337127686, "step": 191, "step_time": 14.285619940001197 }, { "clip_ratio/high_max": 0.026041666977107525, "clip_ratio/high_mean": 0.013020833488553762, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 595.0, "completions/max_terminated_length": 595.0, "completions/mean_length": 251.46875, "completions/mean_terminated_length": 251.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17835678439587355, "epoch": 0.00384, "frac_reward_zero_std": 0.0, "grad_norm": 0.7268782258033752, "kl": 2.036255467683077, "learning_rate": 9.41403851423797e-06, "loss": 0.0078, "num_tokens": 8834414.0, "reward": 0.13750000298023224, "reward_std": 0.2920565605163574, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.2981123626232147, "sampling/importance_sampling_ratio/max": 1.584600806236267, "sampling/importance_sampling_ratio/mean": 0.9801034927368164, "sampling/importance_sampling_ratio/min": 0.19539843499660492, "sampling/sampling_logp_difference/max": 1.7022894620895386, "sampling/sampling_logp_difference/mean": 0.09550048410892487, "step": 192, "step_time": 11.400059683004656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 337.84375, "completions/mean_terminated_length": 337.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19841989502310753, "epoch": 0.00386, "frac_reward_zero_std": 0.0, "grad_norm": 0.38319647312164307, "kl": 2.351551430299878, "learning_rate": 9.406707379266792e-06, "loss": -0.0167, "num_tokens": 8884421.0, "reward": 0.08750000596046448, "reward_std": 0.266537070274353, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.266094833612442, "sampling/importance_sampling_ratio/max": 1.3208733797073364, "sampling/importance_sampling_ratio/mean": 0.8083865642547607, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8145298957824707, "sampling/sampling_logp_difference/mean": 0.14600911736488342, "step": 193, "step_time": 14.909500289999414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1240.0, "completions/max_terminated_length": 1240.0, "completions/mean_length": 399.71875, "completions/mean_terminated_length": 399.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2133172433823347, "epoch": 0.00388, "frac_reward_zero_std": 0.0, "grad_norm": 0.9238452911376953, "kl": 1.5932987481355667, "learning_rate": 9.399334605191332e-06, "loss": -0.0074, "num_tokens": 8937955.0, "reward": 0.05625000596046448, "reward_std": 0.2836637496948242, "rewards/rollout_reward_func/mean": 0.05625000596046448, "rewards/rollout_reward_func/std": 0.30684396624565125, "sampling/importance_sampling_ratio/max": 1.4059948921203613, "sampling/importance_sampling_ratio/mean": 0.9193428158760071, "sampling/importance_sampling_ratio/min": 0.14666610956192017, "sampling/sampling_logp_difference/max": 1.5682001113891602, "sampling/sampling_logp_difference/mean": 0.1093364953994751, "step": 194, "step_time": 14.884564432992192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 301.09375, "completions/mean_terminated_length": 301.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2010048064403236, "epoch": 0.0039, "frac_reward_zero_std": 0.0, "grad_norm": 0.36355605721473694, "kl": 1.4385238476097584, "learning_rate": 9.391920289263475e-06, "loss": -0.0242, "num_tokens": 8987429.0, "reward": 0.12812499701976776, "reward_std": 0.2750462591648102, "rewards/rollout_reward_func/mean": 0.12812499701976776, "rewards/rollout_reward_func/std": 0.2853796184062958, "sampling/importance_sampling_ratio/max": 1.6651469469070435, "sampling/importance_sampling_ratio/mean": 0.9774003028869629, "sampling/importance_sampling_ratio/min": 0.21247626841068268, "sampling/sampling_logp_difference/max": 1.3785629272460938, "sampling/sampling_logp_difference/mean": 0.0942106693983078, "step": 195, "step_time": 14.099435215000995 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1033.0, "completions/max_terminated_length": 1033.0, "completions/mean_length": 307.875, "completions/mean_terminated_length": 307.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20142868859693408, "epoch": 0.00392, "frac_reward_zero_std": 0.0, "grad_norm": 0.4568719267845154, "kl": 1.8109531551599503, "learning_rate": 9.384464529283055e-06, "loss": -0.0273, "num_tokens": 9036715.0, "reward": 0.08437500894069672, "reward_std": 0.26862967014312744, "rewards/rollout_reward_func/mean": 0.08437500894069672, "rewards/rollout_reward_func/std": 0.2772088646888733, "sampling/importance_sampling_ratio/max": 1.5036600828170776, "sampling/importance_sampling_ratio/mean": 0.8336995244026184, "sampling/importance_sampling_ratio/min": 0.25962790846824646, "sampling/sampling_logp_difference/max": 1.2911949157714844, "sampling/sampling_logp_difference/mean": 0.14098377525806427, "step": 196, "step_time": 14.01885101899461 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 282.21875, "completions/mean_terminated_length": 282.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21314223622903228, "epoch": 0.00394, "frac_reward_zero_std": 0.0, "grad_norm": 0.6721346378326416, "kl": 1.2485673893243074, "learning_rate": 9.376967423596588e-06, "loss": -0.0322, "num_tokens": 9084815.0, "reward": 0.19687500596046448, "reward_std": 0.22541293501853943, "rewards/rollout_reward_func/mean": 0.19687500596046448, "rewards/rollout_reward_func/std": 0.2348429411649704, "sampling/importance_sampling_ratio/max": 1.9159910678863525, "sampling/importance_sampling_ratio/mean": 0.9538149833679199, "sampling/importance_sampling_ratio/min": 0.33713269233703613, "sampling/sampling_logp_difference/max": 1.4773423671722412, "sampling/sampling_logp_difference/mean": 0.11870540678501129, "step": 197, "step_time": 14.203826136006683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1128.0, "completions/max_terminated_length": 1128.0, "completions/mean_length": 302.96875, "completions/mean_terminated_length": 302.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20017840899527073, "epoch": 0.00396, "frac_reward_zero_std": 0.0, "grad_norm": 0.4997199475765228, "kl": 1.3511322401463985, "learning_rate": 9.369429071095963e-06, "loss": -0.0252, "num_tokens": 9134872.0, "reward": 0.19374999403953552, "reward_std": 0.22890913486480713, "rewards/rollout_reward_func/mean": 0.19374999403953552, "rewards/rollout_reward_func/std": 0.2500806450843811, "sampling/importance_sampling_ratio/max": 1.944703221321106, "sampling/importance_sampling_ratio/mean": 0.9071593284606934, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7340116500854492, "sampling/sampling_logp_difference/mean": 0.12762850522994995, "step": 198, "step_time": 13.622793402988464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 978.0, "completions/max_terminated_length": 978.0, "completions/mean_length": 305.5625, "completions/mean_terminated_length": 305.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2525788974016905, "epoch": 0.00398, "frac_reward_zero_std": 0.0, "grad_norm": 0.7009827494621277, "kl": 3.176251983270049, "learning_rate": 9.361849571217149e-06, "loss": -0.0001, "num_tokens": 9185204.0, "reward": 0.0624999962747097, "reward_std": 0.2608405351638794, "rewards/rollout_reward_func/mean": 0.0624999962747097, "rewards/rollout_reward_func/std": 0.26730436086654663, "sampling/importance_sampling_ratio/max": 2.4824907779693604, "sampling/importance_sampling_ratio/mean": 0.9883126020431519, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.067967176437378, "sampling/sampling_logp_difference/mean": 0.0996037945151329, "step": 199, "step_time": 13.69620001200019 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.027901785913854837, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 332.9375, "completions/mean_terminated_length": 332.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2139255264773965, "epoch": 0.004, "frac_reward_zero_std": 0.0, "grad_norm": 0.5561277270317078, "kl": 1.4806168526411057, "learning_rate": 9.35422902393887e-06, "loss": 0.007, "num_tokens": 9236355.0, "reward": 0.0312500037252903, "reward_std": 0.2793150246143341, "rewards/rollout_reward_func/mean": 0.0312500037252903, "rewards/rollout_reward_func/std": 0.28447744250297546, "sampling/importance_sampling_ratio/max": 1.6400741338729858, "sampling/importance_sampling_ratio/mean": 0.7841335535049438, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.335585117340088, "sampling/sampling_logp_difference/mean": 0.14428630471229553, "step": 200, "step_time": 14.370141540002805 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 880.0, "completions/max_terminated_length": 880.0, "completions/mean_length": 174.96875, "completions/mean_terminated_length": 174.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17061163252219558, "epoch": 0.00402, "frac_reward_zero_std": 0.0, "grad_norm": 1.1790851354599, "kl": 1.9489646535366774, "learning_rate": 9.346567529781298e-06, "loss": 0.0983, "num_tokens": 9280303.0, "reward": 0.1093750074505806, "reward_std": 0.23359882831573486, "rewards/rollout_reward_func/mean": 0.1093750074505806, "rewards/rollout_reward_func/std": 0.28211113810539246, "sampling/importance_sampling_ratio/max": 2.413576126098633, "sampling/importance_sampling_ratio/mean": 0.9778719544410706, "sampling/importance_sampling_ratio/min": 0.18778157234191895, "sampling/sampling_logp_difference/max": 1.672419548034668, "sampling/sampling_logp_difference/mean": 0.11425619572401047, "step": 201, "step_time": 12.500408270007028 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1200.0, "completions/max_terminated_length": 1200.0, "completions/mean_length": 304.375, "completions/mean_terminated_length": 304.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22460387600585818, "epoch": 0.00404, "frac_reward_zero_std": 0.0, "grad_norm": 1.2917048931121826, "kl": 2.977269198745489, "learning_rate": 9.338865189804722e-06, "loss": 0.1176, "num_tokens": 9329261.0, "reward": 0.19375000894069672, "reward_std": 0.2691294848918915, "rewards/rollout_reward_func/mean": 0.19375000894069672, "rewards/rollout_reward_func/std": 0.2816283404827118, "sampling/importance_sampling_ratio/max": 2.5205671787261963, "sampling/importance_sampling_ratio/mean": 1.0330729484558105, "sampling/importance_sampling_ratio/min": 0.2921076714992523, "sampling/sampling_logp_difference/max": 1.2302417755126953, "sampling/sampling_logp_difference/mean": 0.09723179042339325, "step": 202, "step_time": 14.76073942300718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1101.0, "completions/max_terminated_length": 1101.0, "completions/mean_length": 365.84375, "completions/mean_terminated_length": 365.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25770663283765316, "epoch": 0.00406, "frac_reward_zero_std": 0.0, "grad_norm": 0.8633249998092651, "kl": 1.5886623617261648, "learning_rate": 9.33112210560821e-06, "loss": 0.0094, "num_tokens": 9380853.0, "reward": 0.08437499403953552, "reward_std": 0.25521358847618103, "rewards/rollout_reward_func/mean": 0.08437499403953552, "rewards/rollout_reward_func/std": 0.26531705260276794, "sampling/importance_sampling_ratio/max": 2.2892327308654785, "sampling/importance_sampling_ratio/mean": 0.8513706922531128, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5028176307678223, "sampling/sampling_logp_difference/mean": 0.1356128305196762, "step": 203, "step_time": 13.705935202004184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 226.34375, "completions/mean_terminated_length": 226.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19974413700401783, "epoch": 0.00408, "frac_reward_zero_std": 0.0, "grad_norm": 1.0426015853881836, "kl": 1.3387439269572496, "learning_rate": 9.323338379328278e-06, "loss": 0.0174, "num_tokens": 9428464.0, "reward": 0.125, "reward_std": 0.2965965270996094, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.3048003017902374, "sampling/importance_sampling_ratio/max": 2.068009853363037, "sampling/importance_sampling_ratio/mean": 1.016942024230957, "sampling/importance_sampling_ratio/min": 0.2693028748035431, "sampling/sampling_logp_difference/max": 1.3227201700210571, "sampling/sampling_logp_difference/mean": 0.12460783869028091, "step": 204, "step_time": 12.942953447996842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1041.0, "completions/max_terminated_length": 1041.0, "completions/mean_length": 305.03125, "completions/mean_terminated_length": 305.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2184854275546968, "epoch": 0.0041, "frac_reward_zero_std": 0.0, "grad_norm": 0.6390209794044495, "kl": 1.361292326822877, "learning_rate": 9.315514113637534e-06, "loss": 0.0089, "num_tokens": 9478172.0, "reward": 0.125, "reward_std": 0.25315696001052856, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.2615092694759369, "sampling/importance_sampling_ratio/max": 1.603812575340271, "sampling/importance_sampling_ratio/mean": 0.8221683502197266, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.072981357574463, "sampling/sampling_logp_difference/mean": 0.1652233898639679, "step": 205, "step_time": 13.900734907987498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 242.0, "completions/mean_terminated_length": 242.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18799801170825958, "epoch": 0.00412, "frac_reward_zero_std": 0.0, "grad_norm": 0.5908008217811584, "kl": 2.4897257164120674, "learning_rate": 9.30764941174334e-06, "loss": -0.0277, "num_tokens": 9524288.0, "reward": 0.11562500149011612, "reward_std": 0.2892582416534424, "rewards/rollout_reward_func/mean": 0.11562500149011612, "rewards/rollout_reward_func/std": 0.28861111402511597, "sampling/importance_sampling_ratio/max": 1.5190266370773315, "sampling/importance_sampling_ratio/mean": 0.827454149723053, "sampling/importance_sampling_ratio/min": 0.29832547903060913, "sampling/sampling_logp_difference/max": 1.5171200037002563, "sampling/sampling_logp_difference/mean": 0.13870769739151, "step": 206, "step_time": 13.055180209004902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 248.8125, "completions/mean_terminated_length": 248.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15874540829099715, "epoch": 0.00414, "frac_reward_zero_std": 0.0, "grad_norm": 0.359067440032959, "kl": 2.7864093370735645, "learning_rate": 9.299744377386424e-06, "loss": -0.0262, "num_tokens": 9571688.0, "reward": 0.20625001192092896, "reward_std": 0.3394929766654968, "rewards/rollout_reward_func/mean": 0.20625001192092896, "rewards/rollout_reward_func/std": 0.3340248167514801, "sampling/importance_sampling_ratio/max": 2.5686123371124268, "sampling/importance_sampling_ratio/mean": 0.9217528104782104, "sampling/importance_sampling_ratio/min": 0.1262541264295578, "sampling/sampling_logp_difference/max": 1.0451254844665527, "sampling/sampling_logp_difference/mean": 0.11350060999393463, "step": 207, "step_time": 13.847455920000357 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1652.0, "completions/max_terminated_length": 1652.0, "completions/mean_length": 317.25, "completions/mean_terminated_length": 317.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20586054003797472, "epoch": 0.00416, "frac_reward_zero_std": 0.0, "grad_norm": 0.7898266315460205, "kl": 2.3433079160749912, "learning_rate": 9.291799114839533e-06, "loss": 0.0739, "num_tokens": 9621310.0, "reward": 0.15625, "reward_std": 0.24023286998271942, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.24878737330436707, "sampling/importance_sampling_ratio/max": 2.5748934745788574, "sampling/importance_sampling_ratio/mean": 1.0197396278381348, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.340238332748413, "sampling/sampling_logp_difference/mean": 0.13241785764694214, "step": 208, "step_time": 16.391436230998806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 393.5625, "completions/mean_terminated_length": 393.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22329143434762955, "epoch": 0.00418, "frac_reward_zero_std": 0.0, "grad_norm": 1.0065064430236816, "kl": 1.3258917201310396, "learning_rate": 9.283813728906054e-06, "loss": 0.002, "num_tokens": 9674842.0, "reward": 0.07187500596046448, "reward_std": 0.28695225715637207, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.29097652435302734, "sampling/importance_sampling_ratio/max": 2.33858585357666, "sampling/importance_sampling_ratio/mean": 0.8711438775062561, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.282996416091919, "sampling/sampling_logp_difference/mean": 0.13921840488910675, "step": 209, "step_time": 15.390926254010992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1075.0, "completions/max_terminated_length": 1075.0, "completions/mean_length": 244.4375, "completions/mean_terminated_length": 244.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2063495460897684, "epoch": 0.0042, "frac_reward_zero_std": 0.0, "grad_norm": 0.8726540803909302, "kl": 1.026567468419671, "learning_rate": 9.275788324918624e-06, "loss": 0.0318, "num_tokens": 9722760.0, "reward": 0.15000000596046448, "reward_std": 0.25235238671302795, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.24756883084774017, "sampling/importance_sampling_ratio/max": 2.7684085369110107, "sampling/importance_sampling_ratio/mean": 1.006212592124939, "sampling/importance_sampling_ratio/min": 0.34465140104293823, "sampling/sampling_logp_difference/max": 0.9268159866333008, "sampling/sampling_logp_difference/mean": 0.1081828773021698, "step": 210, "step_time": 14.13191688599909 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1065.0, "completions/max_terminated_length": 1065.0, "completions/mean_length": 285.71875, "completions/mean_terminated_length": 285.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23529127263464034, "epoch": 0.00422, "frac_reward_zero_std": 0.0, "grad_norm": 0.5486846566200256, "kl": 2.215908158570528, "learning_rate": 9.267723008737745e-06, "loss": -0.0125, "num_tokens": 9771490.0, "reward": 0.12812501192092896, "reward_std": 0.23861142992973328, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.24787400662899017, "sampling/importance_sampling_ratio/max": 1.8122859001159668, "sampling/importance_sampling_ratio/mean": 0.8014945983886719, "sampling/importance_sampling_ratio/min": 0.128198504447937, "sampling/sampling_logp_difference/max": 2.235147714614868, "sampling/sampling_logp_difference/mean": 0.16717278957366943, "step": 211, "step_time": 13.889812930006883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 876.0, "completions/max_terminated_length": 876.0, "completions/mean_length": 259.6875, "completions/mean_terminated_length": 259.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15939929685555398, "epoch": 0.00424, "frac_reward_zero_std": 0.0, "grad_norm": 0.6803938746452332, "kl": 1.2304437775164843, "learning_rate": 9.259617886750395e-06, "loss": 0.0207, "num_tokens": 9818310.0, "reward": 0.23750001192092896, "reward_std": 0.1589849591255188, "rewards/rollout_reward_func/mean": 0.23750001192092896, "rewards/rollout_reward_func/std": 0.16214291751384735, "sampling/importance_sampling_ratio/max": 2.361990213394165, "sampling/importance_sampling_ratio/mean": 0.9494872093200684, "sampling/importance_sampling_ratio/min": 0.13248084485530853, "sampling/sampling_logp_difference/max": 2.3392820358276367, "sampling/sampling_logp_difference/mean": 0.1061699241399765, "step": 212, "step_time": 12.49896941499901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "completions/clipped_ratio": 0.0, "completions/max_length": 1279.0, "completions/max_terminated_length": 1279.0, "completions/mean_length": 295.4375, "completions/mean_terminated_length": 295.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20028033247217536, "epoch": 0.00426, "frac_reward_zero_std": 0.0, "grad_norm": 0.5162804126739502, "kl": 1.7424583658576012, "learning_rate": 9.251473065868606e-06, "loss": 0.0162, "num_tokens": 9865985.0, "reward": 0.06875000149011612, "reward_std": 0.2993636131286621, "rewards/rollout_reward_func/mean": 0.06875000149011612, "rewards/rollout_reward_func/std": 0.2999328076839447, "sampling/importance_sampling_ratio/max": 1.3884198665618896, "sampling/importance_sampling_ratio/mean": 0.7952414751052856, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9272022247314453, "sampling/sampling_logp_difference/mean": 0.15023960173130035, "step": 213, "step_time": 15.715065467993554 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1191.0, "completions/max_terminated_length": 1191.0, "completions/mean_length": 250.53125, "completions/mean_terminated_length": 250.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23262012656778097, "epoch": 0.00428, "frac_reward_zero_std": 0.0, "grad_norm": 1.0082905292510986, "kl": 1.1856861002743244, "learning_rate": 9.243288653528075e-06, "loss": -0.0182, "num_tokens": 9913800.0, "reward": 0.140625, "reward_std": 0.21185778081417084, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.22412465512752533, "sampling/importance_sampling_ratio/max": 2.6401400566101074, "sampling/importance_sampling_ratio/mean": 0.8506566882133484, "sampling/importance_sampling_ratio/min": 6.415768183387627e-08, "sampling/sampling_logp_difference/max": 16.206947326660156, "sampling/sampling_logp_difference/mean": 0.30064499378204346, "step": 214, "step_time": 13.94374559199423 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.0252976194024086, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0292038694024086, "completions/clipped_ratio": 0.0, "completions/max_length": 1778.0, "completions/max_terminated_length": 1778.0, "completions/mean_length": 223.21875, "completions/mean_terminated_length": 223.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22658873512409627, "epoch": 0.0043, "frac_reward_zero_std": 0.0, "grad_norm": 0.8495049476623535, "kl": 5.844370843842626, "learning_rate": 9.235064757686733e-06, "loss": -0.0224, "num_tokens": 9958846.0, "reward": 0.12812501192092896, "reward_std": 0.2820037007331848, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.27384287118911743, "sampling/importance_sampling_ratio/max": 1.9493169784545898, "sampling/importance_sampling_ratio/mean": 0.7853102684020996, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2402148246765137, "sampling/sampling_logp_difference/mean": 0.13979823887348175, "step": 215, "step_time": 15.993974796994735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 282.1875, "completions/mean_terminated_length": 282.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2986710002878681, "epoch": 0.00432, "frac_reward_zero_std": 0.0, "grad_norm": 1.450805425643921, "kl": 1.4380096644163132, "learning_rate": 9.226801486823325e-06, "loss": 0.0733, "num_tokens": 10008095.0, "reward": 0.16875000298023224, "reward_std": 0.19680199027061462, "rewards/rollout_reward_func/mean": 0.16875000298023224, "rewards/rollout_reward_func/std": 0.2007044106721878, "sampling/importance_sampling_ratio/max": 2.4321787357330322, "sampling/importance_sampling_ratio/mean": 0.8948805332183838, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 15.22598648071289, "sampling/sampling_logp_difference/mean": 0.2651127278804779, "step": 216, "step_time": 14.433091602997592 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 328.96875, "completions/mean_terminated_length": 328.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2401074597146362, "epoch": 0.00434, "frac_reward_zero_std": 0.0, "grad_norm": 1.3554335832595825, "kl": 0.9870772249996662, "learning_rate": 9.21849894993598e-06, "loss": 0.0522, "num_tokens": 10057841.0, "reward": 0.1875, "reward_std": 0.2824738323688507, "rewards/rollout_reward_func/mean": 0.1875, "rewards/rollout_reward_func/std": 0.2779533267021179, "sampling/importance_sampling_ratio/max": 2.9965219497680664, "sampling/importance_sampling_ratio/mean": 0.844884991645813, "sampling/importance_sampling_ratio/min": 0.155415341258049, "sampling/sampling_logp_difference/max": 1.3453333377838135, "sampling/sampling_logp_difference/mean": 0.1403532177209854, "step": 217, "step_time": 14.41669060100321 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018229166977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1173.0, "completions/max_terminated_length": 1173.0, "completions/mean_length": 355.25, "completions/mean_terminated_length": 355.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.293045774102211, "epoch": 0.00436, "frac_reward_zero_std": 0.0, "grad_norm": 5.703181743621826, "kl": 3.140858568251133, "learning_rate": 9.21015725654077e-06, "loss": 0.1184, "num_tokens": 10109639.0, "reward": 0.0031250016763806343, "reward_std": 0.3178471326828003, "rewards/rollout_reward_func/mean": 0.0031250016763806343, "rewards/rollout_reward_func/std": 0.33261850476264954, "sampling/importance_sampling_ratio/max": 2.4256346225738525, "sampling/importance_sampling_ratio/mean": 0.9400455355644226, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4171068668365479, "sampling/sampling_logp_difference/mean": 0.14329512417316437, "step": 218, "step_time": 15.175195320989587 }, { "clip_ratio/high_max": 0.0069444444961845875, "clip_ratio/high_mean": 0.0034722222480922937, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.019097222248092294, "completions/clipped_ratio": 0.0, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 246.25, "completions/mean_terminated_length": 246.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36270489962771535, "epoch": 0.00438, "frac_reward_zero_std": 0.0, "grad_norm": 0.6323626041412354, "kl": 1.166766483336687, "learning_rate": 9.201776516670274e-06, "loss": -0.0102, "num_tokens": 10158065.0, "reward": 0.04375000298023224, "reward_std": 0.21975308656692505, "rewards/rollout_reward_func/mean": 0.04375000298023224, "rewards/rollout_reward_func/std": 0.23131811618804932, "sampling/importance_sampling_ratio/max": 2.1266989707946777, "sampling/importance_sampling_ratio/mean": 0.8632742166519165, "sampling/importance_sampling_ratio/min": 2.7041094363688956e-10, "sampling/sampling_logp_difference/max": 21.871932983398438, "sampling/sampling_logp_difference/mean": 0.46123945713043213, "step": 219, "step_time": 13.310951123999985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 975.0, "completions/max_terminated_length": 975.0, "completions/mean_length": 295.25, "completions/mean_terminated_length": 295.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26020348793827, "epoch": 0.0044, "frac_reward_zero_std": 0.0, "grad_norm": 0.8192313313484192, "kl": 1.2630127649754286, "learning_rate": 9.193356840872111e-06, "loss": 0.0653, "num_tokens": 10206906.0, "reward": 0.04375000298023224, "reward_std": 0.29200825095176697, "rewards/rollout_reward_func/mean": 0.04375000298023224, "rewards/rollout_reward_func/std": 0.3089393973350525, "sampling/importance_sampling_ratio/max": 2.594595193862915, "sampling/importance_sampling_ratio/mean": 0.8611760139465332, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4862221479415894, "sampling/sampling_logp_difference/mean": 0.16633376479148865, "step": 220, "step_time": 13.293386044999352 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 362.1875, "completions/mean_terminated_length": 362.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3117621298879385, "epoch": 0.00442, "frac_reward_zero_std": 0.0, "grad_norm": 0.6426318287849426, "kl": 1.4144716523587704, "learning_rate": 9.184898340207502e-06, "loss": 0.0409, "num_tokens": 10260190.0, "reward": 0.08750000596046448, "reward_std": 0.26764822006225586, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.29043900966644287, "sampling/importance_sampling_ratio/max": 2.379434108734131, "sampling/importance_sampling_ratio/mean": 0.8323591351509094, "sampling/importance_sampling_ratio/min": 0.23171810805797577, "sampling/sampling_logp_difference/max": 1.8273829221725464, "sampling/sampling_logp_difference/mean": 0.16994285583496094, "step": 221, "step_time": 15.731351782000274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.03125, "completions/max_length": 1266.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 281.875, "completions/mean_terminated_length": 250.1290283203125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22395596839487553, "epoch": 0.00444, "frac_reward_zero_std": 0.0, "grad_norm": 2.3224291801452637, "kl": 17.827216763049364, "learning_rate": 9.176401126249785e-06, "loss": 0.109, "num_tokens": 10307783.0, "reward": 0.140625, "reward_std": 0.20584836602210999, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.19977305829524994, "sampling/importance_sampling_ratio/max": 2.7910823822021484, "sampling/importance_sampling_ratio/mean": 1.0190736055374146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7963225841522217, "sampling/sampling_logp_difference/mean": 0.13311445713043213, "step": 222, "step_time": 15.726387438004167 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1625.0, "completions/max_terminated_length": 1625.0, "completions/mean_length": 333.25, "completions/mean_terminated_length": 333.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2945447666570544, "epoch": 0.00446, "frac_reward_zero_std": 0.0, "grad_norm": 0.5867213010787964, "kl": 1.1944441571831703, "learning_rate": 9.167865311082957e-06, "loss": 0.0415, "num_tokens": 10358265.0, "reward": 0.09687500447034836, "reward_std": 0.25071457028388977, "rewards/rollout_reward_func/mean": 0.09687500447034836, "rewards/rollout_reward_func/std": 0.2583875358104706, "sampling/importance_sampling_ratio/max": 1.7161178588867188, "sampling/importance_sampling_ratio/mean": 0.7358981966972351, "sampling/importance_sampling_ratio/min": 0.16307677328586578, "sampling/sampling_logp_difference/max": 1.0783731937408447, "sampling/sampling_logp_difference/mean": 0.16816218197345734, "step": 223, "step_time": 15.939954874997056 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 558.0, "completions/max_terminated_length": 558.0, "completions/mean_length": 204.75, "completions/mean_terminated_length": 204.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21242515067569911, "epoch": 0.00448, "frac_reward_zero_std": 0.0, "grad_norm": 0.7899563908576965, "kl": 1.439064996317029, "learning_rate": 9.159291007300193e-06, "loss": 0.0211, "num_tokens": 10404019.0, "reward": 0.16562499105930328, "reward_std": 0.25740376114845276, "rewards/rollout_reward_func/mean": 0.16562499105930328, "rewards/rollout_reward_func/std": 0.2671346068382263, "sampling/importance_sampling_ratio/max": 1.5478934049606323, "sampling/importance_sampling_ratio/mean": 0.9083219766616821, "sampling/importance_sampling_ratio/min": 0.38650694489479065, "sampling/sampling_logp_difference/max": 1.1109143495559692, "sampling/sampling_logp_difference/mean": 0.11225207895040512, "step": 224, "step_time": 11.548956477010506 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 259.03125, "completions/mean_terminated_length": 259.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1990206204354763, "epoch": 0.0045, "frac_reward_zero_std": 0.0, "grad_norm": 1.133013367652893, "kl": 1.5414435416460037, "learning_rate": 9.150678328002352e-06, "loss": 0.0864, "num_tokens": 10450067.0, "reward": 0.17812499403953552, "reward_std": 0.2694222331047058, "rewards/rollout_reward_func/mean": 0.17812499403953552, "rewards/rollout_reward_func/std": 0.29373496770858765, "sampling/importance_sampling_ratio/max": 2.718502998352051, "sampling/importance_sampling_ratio/mean": 1.0143585205078125, "sampling/importance_sampling_ratio/min": 0.19919948279857635, "sampling/sampling_logp_difference/max": 0.9963951110839844, "sampling/sampling_logp_difference/mean": 0.10360649228096008, "step": 225, "step_time": 14.059979169993312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.020833333488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 313.96875, "completions/mean_terminated_length": 313.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28669135505333543, "epoch": 0.00452, "frac_reward_zero_std": 0.0, "grad_norm": 1.2064341306686401, "kl": 1.1214723587036133, "learning_rate": 9.1420273867965e-06, "loss": 0.0884, "num_tokens": 10500082.0, "reward": 0.12812501192092896, "reward_std": 0.3166334331035614, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.33237597346305847, "sampling/importance_sampling_ratio/max": 2.3468008041381836, "sampling/importance_sampling_ratio/mean": 0.9828333854675293, "sampling/importance_sampling_ratio/min": 0.20719152688980103, "sampling/sampling_logp_difference/max": 1.614068627357483, "sampling/sampling_logp_difference/mean": 0.13137483596801758, "step": 226, "step_time": 13.272501181996631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 262.875, "completions/mean_terminated_length": 262.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3304764162749052, "epoch": 0.00454, "frac_reward_zero_std": 0.0, "grad_norm": 0.3467620313167572, "kl": 2.027207374572754, "learning_rate": 9.133338297794396e-06, "loss": -0.0068, "num_tokens": 10548230.0, "reward": 0.10625000298023224, "reward_std": 0.24578604102134705, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.24088211357593536, "sampling/importance_sampling_ratio/max": 1.6254823207855225, "sampling/importance_sampling_ratio/mean": 0.794933557510376, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5876078605651855, "sampling/sampling_logp_difference/mean": 0.1169978529214859, "step": 227, "step_time": 14.405538548995537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 298.71875, "completions/mean_terminated_length": 298.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2607615515589714, "epoch": 0.00456, "frac_reward_zero_std": 0.0, "grad_norm": 0.6827296018600464, "kl": 0.9901799988001585, "learning_rate": 9.124611175610998e-06, "loss": 0.0256, "num_tokens": 10596799.0, "reward": 0.09375, "reward_std": 0.29882362484931946, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.29939454793930054, "sampling/importance_sampling_ratio/max": 2.221890687942505, "sampling/importance_sampling_ratio/mean": 0.8756235837936401, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9941788911819458, "sampling/sampling_logp_difference/mean": 0.09268917143344879, "step": 228, "step_time": 15.40145563099577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 361.0, "completions/mean_terminated_length": 361.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24251230619847775, "epoch": 0.00458, "frac_reward_zero_std": 0.0, "grad_norm": 0.6795744299888611, "kl": 1.2100401874631643, "learning_rate": 9.11584613536295e-06, "loss": 0.0295, "num_tokens": 10647701.0, "reward": 0.11250000447034836, "reward_std": 0.30638444423675537, "rewards/rollout_reward_func/mean": 0.11250000447034836, "rewards/rollout_reward_func/std": 0.3108261525630951, "sampling/importance_sampling_ratio/max": 2.6065728664398193, "sampling/importance_sampling_ratio/mean": 0.9051994681358337, "sampling/importance_sampling_ratio/min": 0.39358511567115784, "sampling/sampling_logp_difference/max": 1.4827508926391602, "sampling/sampling_logp_difference/mean": 0.10501286387443542, "step": 229, "step_time": 15.223781991007854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 214.90625, "completions/mean_terminated_length": 214.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2508841808885336, "epoch": 0.0046, "frac_reward_zero_std": 0.0, "grad_norm": 0.5014076828956604, "kl": 1.141053500585258, "learning_rate": 9.107043292667056e-06, "loss": -0.014, "num_tokens": 10693917.0, "reward": 0.15312500298023224, "reward_std": 0.23857757449150085, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.2299850881099701, "sampling/importance_sampling_ratio/max": 1.9632930755615234, "sampling/importance_sampling_ratio/mean": 0.8457331657409668, "sampling/importance_sampling_ratio/min": 0.2062518298625946, "sampling/sampling_logp_difference/max": 1.6131895780563354, "sampling/sampling_logp_difference/mean": 0.10415861755609512, "step": 230, "step_time": 11.866753926995443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 171.21875, "completions/mean_terminated_length": 171.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22451560106128454, "epoch": 0.00462, "frac_reward_zero_std": 0.0, "grad_norm": 0.34701234102249146, "kl": 0.9150925008580089, "learning_rate": 9.098202763638763e-06, "loss": 0.0022, "num_tokens": 10737266.0, "reward": 0.15937501192092896, "reward_std": 0.1714901626110077, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.1793525516986847, "sampling/importance_sampling_ratio/max": 1.3944315910339355, "sampling/importance_sampling_ratio/mean": 0.854866623878479, "sampling/importance_sampling_ratio/min": 0.2559617757797241, "sampling/sampling_logp_difference/max": 1.3607499599456787, "sampling/sampling_logp_difference/mean": 0.08816426992416382, "step": 231, "step_time": 13.024097676003294 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014062500093132257, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 296.78125, "completions/mean_terminated_length": 296.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3108947053551674, "epoch": 0.00464, "frac_reward_zero_std": 0.0, "grad_norm": 1.0716001987457275, "kl": 0.6376080401241779, "learning_rate": 9.089324664890625e-06, "loss": 0.0252, "num_tokens": 10787160.0, "reward": 0.109375, "reward_std": 0.25519683957099915, "rewards/rollout_reward_func/mean": 0.109375, "rewards/rollout_reward_func/std": 0.2728101909160614, "sampling/importance_sampling_ratio/max": 1.5334792137145996, "sampling/importance_sampling_ratio/mean": 0.9189778566360474, "sampling/importance_sampling_ratio/min": 0.4723767042160034, "sampling/sampling_logp_difference/max": 1.2562217712402344, "sampling/sampling_logp_difference/mean": 0.09062428027391434, "step": 232, "step_time": 13.838409851006872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 268.09375, "completions/mean_terminated_length": 268.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.245048092212528, "epoch": 0.00466, "frac_reward_zero_std": 0.0, "grad_norm": 0.8532366752624512, "kl": 1.9912269115447998, "learning_rate": 9.080409113530767e-06, "loss": 0.0577, "num_tokens": 10835473.0, "reward": 0.06875000149011612, "reward_std": 0.2677779495716095, "rewards/rollout_reward_func/mean": 0.06875000149011612, "rewards/rollout_reward_func/std": 0.268132746219635, "sampling/importance_sampling_ratio/max": 2.369253635406494, "sampling/importance_sampling_ratio/mean": 0.8640143871307373, "sampling/importance_sampling_ratio/min": 0.08825299143791199, "sampling/sampling_logp_difference/max": 2.082698345184326, "sampling/sampling_logp_difference/mean": 0.13722990453243256, "step": 233, "step_time": 14.44520044800447 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1307.0, "completions/max_terminated_length": 1307.0, "completions/mean_length": 280.84375, "completions/mean_terminated_length": 280.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23398237070068717, "epoch": 0.00468, "frac_reward_zero_std": 0.0, "grad_norm": 0.3764919936656952, "kl": 0.84406578540802, "learning_rate": 9.071456227161335e-06, "loss": 0.0312, "num_tokens": 10883692.0, "reward": 0.17812499403953552, "reward_std": 0.27232086658477783, "rewards/rollout_reward_func/mean": 0.17812499403953552, "rewards/rollout_reward_func/std": 0.2732532322406769, "sampling/importance_sampling_ratio/max": 1.7436747550964355, "sampling/importance_sampling_ratio/mean": 0.7954779863357544, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.714820146560669, "sampling/sampling_logp_difference/mean": 0.12941089272499084, "step": 234, "step_time": 14.512654798996664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 266.40625, "completions/mean_terminated_length": 266.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29055723641067743, "epoch": 0.0047, "frac_reward_zero_std": 0.0, "grad_norm": 0.560286819934845, "kl": 0.7556242765858769, "learning_rate": 9.062466123876954e-06, "loss": 0.0016, "num_tokens": 10932609.0, "reward": 0.109375, "reward_std": 0.283163845539093, "rewards/rollout_reward_func/mean": 0.109375, "rewards/rollout_reward_func/std": 0.28438884019851685, "sampling/importance_sampling_ratio/max": 1.767192006111145, "sampling/importance_sampling_ratio/mean": 0.7880735993385315, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 12.053220748901367, "sampling/sampling_logp_difference/mean": 0.22610825300216675, "step": 235, "step_time": 14.686156242991274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 254.5, "completions/mean_terminated_length": 254.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25080798799172044, "epoch": 0.00472, "frac_reward_zero_std": 0.0, "grad_norm": 0.6688145399093628, "kl": 1.2235719542950392, "learning_rate": 9.053438922263165e-06, "loss": 0.0192, "num_tokens": 10981164.0, "reward": 0.012499995529651642, "reward_std": 0.24434086680412292, "rewards/rollout_reward_func/mean": 0.012499995529651642, "rewards/rollout_reward_func/std": 0.2624327838420868, "sampling/importance_sampling_ratio/max": 2.7707979679107666, "sampling/importance_sampling_ratio/mean": 0.9572605490684509, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7356326580047607, "sampling/sampling_logp_difference/mean": 0.11393360048532486, "step": 236, "step_time": 15.030656848000945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 280.09375, "completions/mean_terminated_length": 280.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22500681574456394, "epoch": 0.00474, "frac_reward_zero_std": 0.0, "grad_norm": 0.3464186191558838, "kl": 0.5731271635740995, "learning_rate": 9.044374741394859e-06, "loss": -0.0064, "num_tokens": 11029345.0, "reward": 0.18437501788139343, "reward_std": 0.2517842650413513, "rewards/rollout_reward_func/mean": 0.18437501788139343, "rewards/rollout_reward_func/std": 0.2829674482345581, "sampling/importance_sampling_ratio/max": 1.3508076667785645, "sampling/importance_sampling_ratio/mean": 0.8336701989173889, "sampling/importance_sampling_ratio/min": 0.11234523355960846, "sampling/sampling_logp_difference/max": 2.1975154876708984, "sampling/sampling_logp_difference/mean": 0.11373690515756607, "step": 237, "step_time": 13.475671670999873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 204.5625, "completions/mean_terminated_length": 204.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19055656623095274, "epoch": 0.00476, "frac_reward_zero_std": 0.0, "grad_norm": 0.5948528051376343, "kl": 0.6027458645403385, "learning_rate": 9.035273700834707e-06, "loss": -0.0162, "num_tokens": 11073373.0, "reward": 0.24375000596046448, "reward_std": 0.18181614577770233, "rewards/rollout_reward_func/mean": 0.24375000596046448, "rewards/rollout_reward_func/std": 0.18480591475963593, "sampling/importance_sampling_ratio/max": 1.528883457183838, "sampling/importance_sampling_ratio/mean": 0.905658483505249, "sampling/importance_sampling_ratio/min": 0.15651099383831024, "sampling/sampling_logp_difference/max": 1.9000506401062012, "sampling/sampling_logp_difference/mean": 0.08751663565635681, "step": 238, "step_time": 13.419543620002514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 273.9375, "completions/mean_terminated_length": 273.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2613520002923906, "epoch": 0.00478, "frac_reward_zero_std": 0.0, "grad_norm": 0.42548125982284546, "kl": 2.279174257069826, "learning_rate": 9.026135920631592e-06, "loss": -0.0144, "num_tokens": 11121224.0, "reward": 0.0625, "reward_std": 0.29533421993255615, "rewards/rollout_reward_func/mean": 0.0625, "rewards/rollout_reward_func/std": 0.2991924583911896, "sampling/importance_sampling_ratio/max": 1.2994014024734497, "sampling/importance_sampling_ratio/mean": 0.8256770372390747, "sampling/importance_sampling_ratio/min": 0.1486884206533432, "sampling/sampling_logp_difference/max": 1.9585418701171875, "sampling/sampling_logp_difference/mean": 0.12413070350885391, "step": 239, "step_time": 14.191121549996751 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 234.96875, "completions/mean_terminated_length": 234.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2088916003704071, "epoch": 0.0048, "frac_reward_zero_std": 0.0, "grad_norm": 0.5850512385368347, "kl": 1.2292664982378483, "learning_rate": 9.016961521319006e-06, "loss": 0.03, "num_tokens": 11167385.0, "reward": 0.19062501192092896, "reward_std": 0.25557073950767517, "rewards/rollout_reward_func/mean": 0.19062501192092896, "rewards/rollout_reward_func/std": 0.27981486916542053, "sampling/importance_sampling_ratio/max": 2.2186403274536133, "sampling/importance_sampling_ratio/mean": 0.9179697036743164, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0044994354248047, "sampling/sampling_logp_difference/mean": 0.08129704743623734, "step": 240, "step_time": 12.23244027700639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 326.3125, "completions/mean_terminated_length": 326.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2178045604377985, "epoch": 0.00482, "frac_reward_zero_std": 0.0, "grad_norm": 0.8047565817832947, "kl": 1.913765287026763, "learning_rate": 9.00775062391348e-06, "loss": 0.041, "num_tokens": 11216552.0, "reward": 0.21875, "reward_std": 0.22909818589687347, "rewards/rollout_reward_func/mean": 0.21875, "rewards/rollout_reward_func/std": 0.22921675443649292, "sampling/importance_sampling_ratio/max": 2.3245410919189453, "sampling/importance_sampling_ratio/mean": 1.0341999530792236, "sampling/importance_sampling_ratio/min": 0.062130339443683624, "sampling/sampling_logp_difference/max": 2.810288429260254, "sampling/sampling_logp_difference/mean": 0.0707245022058487, "step": 241, "step_time": 14.589793486007693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 270.28125, "completions/mean_terminated_length": 270.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21010618656873703, "epoch": 0.00484, "frac_reward_zero_std": 0.0, "grad_norm": 0.3430674374103546, "kl": 2.194681294262409, "learning_rate": 8.998503349912977e-06, "loss": -0.0123, "num_tokens": 11265243.0, "reward": 0.10625000298023224, "reward_std": 0.20965354144573212, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.21089288592338562, "sampling/importance_sampling_ratio/max": 1.8399115800857544, "sampling/importance_sampling_ratio/mean": 0.8789221048355103, "sampling/importance_sampling_ratio/min": 0.11708051711320877, "sampling/sampling_logp_difference/max": 2.1570234298706055, "sampling/sampling_logp_difference/mean": 0.11512541025876999, "step": 242, "step_time": 12.284718313007033 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1661.0, "completions/max_terminated_length": 1661.0, "completions/mean_length": 390.5625, "completions/mean_terminated_length": 390.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2558925556950271, "epoch": 0.00486, "frac_reward_zero_std": 0.0, "grad_norm": 0.6241450905799866, "kl": 0.8101275460794568, "learning_rate": 8.989219821295294e-06, "loss": -0.0193, "num_tokens": 11318441.0, "reward": 0.05937499552965164, "reward_std": 0.2665151059627533, "rewards/rollout_reward_func/mean": 0.05937499552965164, "rewards/rollout_reward_func/std": 0.26379287242889404, "sampling/importance_sampling_ratio/max": 1.1327202320098877, "sampling/importance_sampling_ratio/mean": 0.8297796249389648, "sampling/importance_sampling_ratio/min": 0.31479716300964355, "sampling/sampling_logp_difference/max": 0.9537475109100342, "sampling/sampling_logp_difference/mean": 0.09165732562541962, "step": 243, "step_time": 16.012585102005687 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 293.71875, "completions/mean_terminated_length": 293.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19049517717212439, "epoch": 0.00488, "frac_reward_zero_std": 0.0, "grad_norm": 0.5668641328811646, "kl": 0.5522784776985645, "learning_rate": 8.979900160516447e-06, "loss": 0.0059, "num_tokens": 11366985.0, "reward": 0.19062499701976776, "reward_std": 0.23356138169765472, "rewards/rollout_reward_func/mean": 0.19062499701976776, "rewards/rollout_reward_func/std": 0.23467117547988892, "sampling/importance_sampling_ratio/max": 1.0368391275405884, "sampling/importance_sampling_ratio/mean": 0.9170701503753662, "sampling/importance_sampling_ratio/min": 0.3176731467247009, "sampling/sampling_logp_difference/max": 0.7299132347106934, "sampling/sampling_logp_difference/mean": 0.04389491304755211, "step": 244, "step_time": 16.223489684995002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1163.0, "completions/max_terminated_length": 1163.0, "completions/mean_length": 320.375, "completions/mean_terminated_length": 320.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2188603999093175, "epoch": 0.0049, "frac_reward_zero_std": 0.0, "grad_norm": 0.9451175928115845, "kl": 3.3567973766475916, "learning_rate": 8.970544490509064e-06, "loss": 0.0302, "num_tokens": 11417302.0, "reward": 0.006250003352761269, "reward_std": 0.3306097388267517, "rewards/rollout_reward_func/mean": 0.006250003352761269, "rewards/rollout_reward_func/std": 0.3281792402267456, "sampling/importance_sampling_ratio/max": 1.497641921043396, "sampling/importance_sampling_ratio/mean": 0.8270469307899475, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.606154441833496, "sampling/sampling_logp_difference/mean": 0.11343221366405487, "step": 245, "step_time": 13.95993570399878 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 333.25, "completions/mean_terminated_length": 333.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26862423960119486, "epoch": 0.00492, "frac_reward_zero_std": 0.0, "grad_norm": 0.46726545691490173, "kl": 1.1651468500494957, "learning_rate": 8.961152934680752e-06, "loss": -0.028, "num_tokens": 11468064.0, "reward": 0.08750000596046448, "reward_std": 0.3330591917037964, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.327009916305542, "sampling/importance_sampling_ratio/max": 1.494116187095642, "sampling/importance_sampling_ratio/mean": 0.8637710213661194, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.818669557571411, "sampling/sampling_logp_difference/mean": 0.10313041508197784, "step": 246, "step_time": 13.754982435995771 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.028125000186264515, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.028125000186264515, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 265.6875, "completions/mean_terminated_length": 265.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21249007829464972, "epoch": 0.00494, "frac_reward_zero_std": 0.0, "grad_norm": 0.4071352183818817, "kl": 2.1285723708570004, "learning_rate": 8.951725616912489e-06, "loss": -0.0082, "num_tokens": 11514851.0, "reward": 0.11250000447034836, "reward_std": 0.2824678122997284, "rewards/rollout_reward_func/mean": 0.11250000447034836, "rewards/rollout_reward_func/std": 0.29594025015830994, "sampling/importance_sampling_ratio/max": 1.1209293603897095, "sampling/importance_sampling_ratio/mean": 0.7627503871917725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8217759132385254, "sampling/sampling_logp_difference/mean": 0.15044867992401123, "step": 247, "step_time": 15.701818518995424 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 300.1875, "completions/mean_terminated_length": 300.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1743924873881042, "epoch": 0.00496, "frac_reward_zero_std": 0.0, "grad_norm": 0.4717790484428406, "kl": 1.2589963767677546, "learning_rate": 8.942262661556962e-06, "loss": 0.012, "num_tokens": 11564108.0, "reward": 0.11875000596046448, "reward_std": 0.30125606060028076, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.2934087812900543, "sampling/importance_sampling_ratio/max": 1.96142578125, "sampling/importance_sampling_ratio/mean": 0.9102701544761658, "sampling/importance_sampling_ratio/min": 0.21407827734947205, "sampling/sampling_logp_difference/max": 1.470252513885498, "sampling/sampling_logp_difference/mean": 0.07780726253986359, "step": 248, "step_time": 13.553423921988724 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 312.46875, "completions/mean_terminated_length": 312.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2547385520301759, "epoch": 0.00498, "frac_reward_zero_std": 0.0, "grad_norm": 0.4644928276538849, "kl": 0.886600324884057, "learning_rate": 8.932764193436953e-06, "loss": 0.0313, "num_tokens": 11613886.0, "reward": 0.21250000596046448, "reward_std": 0.22455862164497375, "rewards/rollout_reward_func/mean": 0.21250000596046448, "rewards/rollout_reward_func/std": 0.22106780111789703, "sampling/importance_sampling_ratio/max": 1.1246027946472168, "sampling/importance_sampling_ratio/mean": 0.8265763521194458, "sampling/importance_sampling_ratio/min": 0.24103274941444397, "sampling/sampling_logp_difference/max": 1.4331419467926025, "sampling/sampling_logp_difference/mean": 0.09090910851955414, "step": 249, "step_time": 13.546325824001542 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1625.0, "completions/max_terminated_length": 1625.0, "completions/mean_length": 403.5625, "completions/mean_terminated_length": 403.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2481313543394208, "epoch": 0.005, "frac_reward_zero_std": 0.0, "grad_norm": 0.4882728159427643, "kl": 0.7002610955387354, "learning_rate": 8.923230337843675e-06, "loss": 0.0071, "num_tokens": 11667468.0, "reward": 0.10000000894069672, "reward_std": 0.25056689977645874, "rewards/rollout_reward_func/mean": 0.10000000894069672, "rewards/rollout_reward_func/std": 0.25272706151008606, "sampling/importance_sampling_ratio/max": 1.9346073865890503, "sampling/importance_sampling_ratio/mean": 0.8332518339157104, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7700960636138916, "sampling/sampling_logp_difference/mean": 0.11612936854362488, "step": 250, "step_time": 17.13030613699084 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1055.0, "completions/max_terminated_length": 1055.0, "completions/mean_length": 265.4375, "completions/mean_terminated_length": 265.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16780805820599198, "epoch": 0.00502, "frac_reward_zero_std": 0.0, "grad_norm": 0.5547589659690857, "kl": 1.030711356550455, "learning_rate": 8.913661220535132e-06, "loss": -0.0214, "num_tokens": 11715117.0, "reward": 0.125, "reward_std": 0.2966882586479187, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.285114586353302, "sampling/importance_sampling_ratio/max": 1.2418763637542725, "sampling/importance_sampling_ratio/mean": 0.794096827507019, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.992415189743042, "sampling/sampling_logp_difference/mean": 0.15195003151893616, "step": 251, "step_time": 13.550987809008802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 264.8125, "completions/mean_terminated_length": 264.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19867516355589032, "epoch": 0.00504, "frac_reward_zero_std": 0.0, "grad_norm": 0.7124246954917908, "kl": 0.9218555372208357, "learning_rate": 8.904056967734447e-06, "loss": 0.0291, "num_tokens": 11764214.0, "reward": 0.11562499403953552, "reward_std": 0.21685349941253662, "rewards/rollout_reward_func/mean": 0.11562499403953552, "rewards/rollout_reward_func/std": 0.2591666281223297, "sampling/importance_sampling_ratio/max": 2.5638983249664307, "sampling/importance_sampling_ratio/mean": 0.8779603242874146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2460877895355225, "sampling/sampling_logp_difference/mean": 0.10331167280673981, "step": 252, "step_time": 11.438115192009718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 187.75, "completions/mean_terminated_length": 187.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18405043706297874, "epoch": 0.00506, "frac_reward_zero_std": 0.0, "grad_norm": 0.414483904838562, "kl": 1.204677214846015, "learning_rate": 8.894417706128209e-06, "loss": -0.0247, "num_tokens": 11808552.0, "reward": 0.19374999403953552, "reward_std": 0.28756970167160034, "rewards/rollout_reward_func/mean": 0.19374999403953552, "rewards/rollout_reward_func/std": 0.2804805636405945, "sampling/importance_sampling_ratio/max": 1.612493872642517, "sampling/importance_sampling_ratio/mean": 0.7858084440231323, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.632309913635254, "sampling/sampling_logp_difference/mean": 0.13271963596343994, "step": 253, "step_time": 13.99165296400679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1607.0, "completions/max_terminated_length": 1607.0, "completions/mean_length": 304.59375, "completions/mean_terminated_length": 304.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20358472131192684, "epoch": 0.00508, "frac_reward_zero_std": 0.0, "grad_norm": 0.726687490940094, "kl": 0.6220174506306648, "learning_rate": 8.884743562864792e-06, "loss": -0.0006, "num_tokens": 11856836.0, "reward": 0.1312500238418579, "reward_std": 0.2759251594543457, "rewards/rollout_reward_func/mean": 0.1312500238418579, "rewards/rollout_reward_func/std": 0.29009175300598145, "sampling/importance_sampling_ratio/max": 1.3717708587646484, "sampling/importance_sampling_ratio/mean": 0.8577007055282593, "sampling/importance_sampling_ratio/min": 0.14042796194553375, "sampling/sampling_logp_difference/max": 1.2132182121276855, "sampling/sampling_logp_difference/mean": 0.1037416160106659, "step": 254, "step_time": 15.800385763996019 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 283.59375, "completions/mean_terminated_length": 283.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21910392539575696, "epoch": 0.0051, "frac_reward_zero_std": 0.0, "grad_norm": 0.34424033761024475, "kl": 0.42227220721542835, "learning_rate": 8.875034665552686e-06, "loss": -0.0336, "num_tokens": 11905713.0, "reward": 0.09062499552965164, "reward_std": 0.26840758323669434, "rewards/rollout_reward_func/mean": 0.09062499552965164, "rewards/rollout_reward_func/std": 0.275164932012558, "sampling/importance_sampling_ratio/max": 1.8230204582214355, "sampling/importance_sampling_ratio/mean": 0.8300085663795471, "sampling/importance_sampling_ratio/min": 0.08976902812719345, "sampling/sampling_logp_difference/max": 2.418689489364624, "sampling/sampling_logp_difference/mean": 0.13790036737918854, "step": 255, "step_time": 13.87210195699663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 290.5625, "completions/mean_terminated_length": 290.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15108935977332294, "epoch": 0.00512, "frac_reward_zero_std": 0.0, "grad_norm": 0.42553597688674927, "kl": 0.9493621699512005, "learning_rate": 8.865291142258813e-06, "loss": -0.016, "num_tokens": 11954062.0, "reward": 0.15312500298023224, "reward_std": 0.2893424332141876, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.31723031401634216, "sampling/importance_sampling_ratio/max": 1.0838916301727295, "sampling/importance_sampling_ratio/mean": 0.7507939338684082, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4228830337524414, "sampling/sampling_logp_difference/mean": 0.1446072906255722, "step": 256, "step_time": 14.541502656004013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1319.0, "completions/max_terminated_length": 1319.0, "completions/mean_length": 357.0, "completions/mean_terminated_length": 368.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27110095880925655, "epoch": 0.00514, "frac_reward_zero_std": 0.0, "grad_norm": 0.5970364809036255, "kl": 0.9003786705434322, "learning_rate": 8.855513121506826e-06, "loss": 0.0195, "num_tokens": 12006842.0, "reward": 0.09375, "reward_std": 0.29222536087036133, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.3181980550289154, "sampling/importance_sampling_ratio/max": 1.2532402276992798, "sampling/importance_sampling_ratio/mean": 0.8340620398521423, "sampling/importance_sampling_ratio/min": 0.08483715355396271, "sampling/sampling_logp_difference/max": 2.475433826446533, "sampling/sampling_logp_difference/mean": 0.1088268831372261, "step": 257, "step_time": 14.82128008699874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 637.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 221.09375, "completions/mean_terminated_length": 221.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14513791631907225, "epoch": 0.00516, "frac_reward_zero_std": 0.0, "grad_norm": 0.7368667125701904, "kl": 0.9309267904609442, "learning_rate": 8.845700732275435e-06, "loss": 0.0315, "num_tokens": 12054003.0, "reward": 0.06875000894069672, "reward_std": 0.32750391960144043, "rewards/rollout_reward_func/mean": 0.06875000894069672, "rewards/rollout_reward_func/std": 0.32372578978538513, "sampling/importance_sampling_ratio/max": 2.1085925102233887, "sampling/importance_sampling_ratio/mean": 1.0847861766815186, "sampling/importance_sampling_ratio/min": 0.12458830326795578, "sampling/sampling_logp_difference/max": 2.0880260467529297, "sampling/sampling_logp_difference/mean": 0.0752631351351738, "step": 258, "step_time": 12.17732539901408 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 273.96875, "completions/mean_terminated_length": 273.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17423954885452986, "epoch": 0.00518, "frac_reward_zero_std": 0.0, "grad_norm": 2.37615966796875, "kl": 0.8582821693271399, "learning_rate": 8.835854103996683e-06, "loss": 0.0301, "num_tokens": 12102686.0, "reward": 0.20000000298023224, "reward_std": 0.24123111367225647, "rewards/rollout_reward_func/mean": 0.20000000298023224, "rewards/rollout_reward_func/std": 0.2514474093914032, "sampling/importance_sampling_ratio/max": 2.2952916622161865, "sampling/importance_sampling_ratio/mean": 0.8525100946426392, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1867940425872803, "sampling/sampling_logp_difference/mean": 0.12905652821063995, "step": 259, "step_time": 13.48747096999432 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 369.125, "completions/mean_terminated_length": 369.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17012645909562707, "epoch": 0.0052, "frac_reward_zero_std": 0.0, "grad_norm": 0.4083753824234009, "kl": 1.0122621646150947, "learning_rate": 8.825973366554254e-06, "loss": 0.0602, "num_tokens": 12155526.0, "reward": 0.13437500596046448, "reward_std": 0.2775416076183319, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.27545782923698425, "sampling/importance_sampling_ratio/max": 1.5333976745605469, "sampling/importance_sampling_ratio/mean": 0.7242650985717773, "sampling/importance_sampling_ratio/min": 0.06108579412102699, "sampling/sampling_logp_difference/max": 2.7084453105926514, "sampling/sampling_logp_difference/mean": 0.1870715171098709, "step": 260, "step_time": 14.428000286996394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1241.0, "completions/max_terminated_length": 1241.0, "completions/mean_length": 317.375, "completions/mean_terminated_length": 317.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.189278845442459, "epoch": 0.00522, "frac_reward_zero_std": 0.0, "grad_norm": 0.5270984768867493, "kl": 1.0934006609022617, "learning_rate": 8.816058650281758e-06, "loss": 0.0243, "num_tokens": 12205472.0, "reward": 0.06562500447034836, "reward_std": 0.2749374210834503, "rewards/rollout_reward_func/mean": 0.06562500447034836, "rewards/rollout_reward_func/std": 0.2858031988143921, "sampling/importance_sampling_ratio/max": 1.8268331289291382, "sampling/importance_sampling_ratio/mean": 0.8291357755661011, "sampling/importance_sampling_ratio/min": 0.036935631185770035, "sampling/sampling_logp_difference/max": 2.1470284461975098, "sampling/sampling_logp_difference/mean": 0.14029653370380402, "step": 261, "step_time": 15.727412991993333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 216.625, "completions/mean_terminated_length": 216.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16008019354194403, "epoch": 0.00524, "frac_reward_zero_std": 0.0, "grad_norm": 0.42382732033729553, "kl": 1.0920081548392773, "learning_rate": 8.806110085960999e-06, "loss": -0.0093, "num_tokens": 12251488.0, "reward": 0.12187500298023224, "reward_std": 0.1903342604637146, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.22394464910030365, "sampling/importance_sampling_ratio/max": 2.269397497177124, "sampling/importance_sampling_ratio/mean": 0.915541410446167, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7869205474853516, "sampling/sampling_logp_difference/mean": 0.09877824038267136, "step": 262, "step_time": 14.015961688994139 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 856.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 235.59375, "completions/mean_terminated_length": 235.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14935008948668838, "epoch": 0.00526, "frac_reward_zero_std": 0.0, "grad_norm": 0.618134617805481, "kl": 1.0617250502109528, "learning_rate": 8.79612780482027e-06, "loss": 0.0093, "num_tokens": 12298618.0, "reward": 0.08124999701976776, "reward_std": 0.2906920909881592, "rewards/rollout_reward_func/mean": 0.08124999701976776, "rewards/rollout_reward_func/std": 0.2822004556655884, "sampling/importance_sampling_ratio/max": 1.8191931247711182, "sampling/importance_sampling_ratio/mean": 0.8626056909561157, "sampling/importance_sampling_ratio/min": 0.16777178645133972, "sampling/sampling_logp_difference/max": 1.794406533241272, "sampling/sampling_logp_difference/mean": 0.11286862194538116, "step": 263, "step_time": 12.08967777900034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 249.03125, "completions/mean_terminated_length": 249.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12909581745043397, "epoch": 0.00528, "frac_reward_zero_std": 0.0, "grad_norm": 0.32283273339271545, "kl": 0.8272107467055321, "learning_rate": 8.786111938532606e-06, "loss": 0.0248, "num_tokens": 12344497.0, "reward": 0.15000000596046448, "reward_std": 0.28463464975357056, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.27708157896995544, "sampling/importance_sampling_ratio/max": 1.4503145217895508, "sampling/importance_sampling_ratio/mean": 0.932274580001831, "sampling/importance_sampling_ratio/min": 0.19853517413139343, "sampling/sampling_logp_difference/max": 1.4301996231079102, "sampling/sampling_logp_difference/mean": 0.06518541276454926, "step": 264, "step_time": 14.533617085995502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 297.625, "completions/mean_terminated_length": 297.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10737295227590948, "epoch": 0.0053, "frac_reward_zero_std": 0.0, "grad_norm": 10.359028816223145, "kl": 5.561798799782991, "learning_rate": 8.776062619214053e-06, "loss": -0.0033, "num_tokens": 12393241.0, "reward": 0.15937501192092896, "reward_std": 0.25426751375198364, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.25885525345802307, "sampling/importance_sampling_ratio/max": 1.646299123764038, "sampling/importance_sampling_ratio/mean": 0.7999939322471619, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.770822286605835, "sampling/sampling_logp_difference/mean": 0.14839936792850494, "step": 265, "step_time": 13.835846354999376 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1283.0, "completions/max_terminated_length": 1283.0, "completions/mean_length": 299.375, "completions/mean_terminated_length": 299.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10825809370726347, "epoch": 0.00532, "frac_reward_zero_std": 0.0, "grad_norm": 0.5201862454414368, "kl": 0.7040472906082869, "learning_rate": 8.765979979421928e-06, "loss": -0.0206, "num_tokens": 12442578.0, "reward": 0.13437500596046448, "reward_std": 0.29400336742401123, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.29028284549713135, "sampling/importance_sampling_ratio/max": 1.161351203918457, "sampling/importance_sampling_ratio/mean": 0.8082857728004456, "sampling/importance_sampling_ratio/min": 0.04167695716023445, "sampling/sampling_logp_difference/max": 3.135960340499878, "sampling/sampling_logp_difference/mean": 0.13666987419128418, "step": 266, "step_time": 14.60024784599591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 319.1875, "completions/mean_terminated_length": 319.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10002906527370214, "epoch": 0.00534, "frac_reward_zero_std": 0.0, "grad_norm": 0.7348783016204834, "kl": 3.5733442064374685, "learning_rate": 8.755864152153064e-06, "loss": -0.0078, "num_tokens": 12492875.0, "reward": 0.13124999403953552, "reward_std": 0.2875002920627594, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.2934087812900543, "sampling/importance_sampling_ratio/max": 1.6275535821914673, "sampling/importance_sampling_ratio/mean": 0.765316903591156, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3223845958709717, "sampling/sampling_logp_difference/mean": 0.14551764726638794, "step": 267, "step_time": 13.785100602002785 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 284.40625, "completions/mean_terminated_length": 284.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08655306801665574, "epoch": 0.00536, "frac_reward_zero_std": 0.0, "grad_norm": 0.5976717472076416, "kl": 1.48380565084517, "learning_rate": 8.745715270842064e-06, "loss": 0.0226, "num_tokens": 12541536.0, "reward": 0.10312499105930328, "reward_std": 0.2690797448158264, "rewards/rollout_reward_func/mean": 0.10312499105930328, "rewards/rollout_reward_func/std": 0.30319201946258545, "sampling/importance_sampling_ratio/max": 1.8710891008377075, "sampling/importance_sampling_ratio/mean": 0.9287987947463989, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3523972034454346, "sampling/sampling_logp_difference/mean": 0.07793505489826202, "step": 268, "step_time": 14.549229683987505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0096726194024086, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0096726194024086, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 347.71875, "completions/mean_terminated_length": 347.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1284567720722407, "epoch": 0.00538, "frac_reward_zero_std": 0.0, "grad_norm": 0.8882115483283997, "kl": 1.0056309960782528, "learning_rate": 8.73553346935953e-06, "loss": -0.0076, "num_tokens": 12592705.0, "reward": 0.09999999403953552, "reward_std": 0.30248749256134033, "rewards/rollout_reward_func/mean": 0.09999999403953552, "rewards/rollout_reward_func/std": 0.3121207356452942, "sampling/importance_sampling_ratio/max": 1.4022977352142334, "sampling/importance_sampling_ratio/mean": 0.8441144227981567, "sampling/importance_sampling_ratio/min": 0.07720368355512619, "sampling/sampling_logp_difference/max": 2.5661888122558594, "sampling/sampling_logp_difference/mean": 0.12107457220554352, "step": 269, "step_time": 13.93140025100729 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 329.71875, "completions/mean_terminated_length": 329.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14167554082814604, "epoch": 0.0054, "frac_reward_zero_std": 0.0, "grad_norm": 0.23620980978012085, "kl": 1.420529505237937, "learning_rate": 8.725318882010307e-06, "loss": 0.0014, "num_tokens": 12642941.0, "reward": 0.10000000149011612, "reward_std": 0.26379942893981934, "rewards/rollout_reward_func/mean": 0.10000000149011612, "rewards/rollout_reward_func/std": 0.3162277638912201, "sampling/importance_sampling_ratio/max": 1.479229211807251, "sampling/importance_sampling_ratio/mean": 0.8932013511657715, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.494600772857666, "sampling/sampling_logp_difference/mean": 0.09553386270999908, "step": 270, "step_time": 15.147682188002364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 264.8125, "completions/mean_terminated_length": 264.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09806300012860447, "epoch": 0.00542, "frac_reward_zero_std": 0.0, "grad_norm": 0.6663671135902405, "kl": 2.5959088858217, "learning_rate": 8.715071643531708e-06, "loss": 0.0046, "num_tokens": 12691763.0, "reward": 0.08437500149011612, "reward_std": 0.3030031621456146, "rewards/rollout_reward_func/mean": 0.08437500149011612, "rewards/rollout_reward_func/std": 0.308073490858078, "sampling/importance_sampling_ratio/max": 1.9125280380249023, "sampling/importance_sampling_ratio/mean": 0.9449977278709412, "sampling/importance_sampling_ratio/min": 0.24910277128219604, "sampling/sampling_logp_difference/max": 1.370012879371643, "sampling/sampling_logp_difference/mean": 0.05016843229532242, "step": 271, "step_time": 13.961785532996146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1261.0, "completions/max_terminated_length": 1261.0, "completions/mean_length": 390.3125, "completions/mean_terminated_length": 390.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1543547313194722, "epoch": 0.00544, "frac_reward_zero_std": 0.0, "grad_norm": 0.3668195307254791, "kl": 0.6463566143065691, "learning_rate": 8.704791889091737e-06, "loss": 0.0061, "num_tokens": 12744569.0, "reward": 0.06562499701976776, "reward_std": 0.28692227602005005, "rewards/rollout_reward_func/mean": 0.06562499701976776, "rewards/rollout_reward_func/std": 0.2869296371936798, "sampling/importance_sampling_ratio/max": 1.4275906085968018, "sampling/importance_sampling_ratio/mean": 0.9054564833641052, "sampling/importance_sampling_ratio/min": 0.15792059898376465, "sampling/sampling_logp_difference/max": 1.8566393852233887, "sampling/sampling_logp_difference/mean": 0.07140199840068817, "step": 272, "step_time": 15.030382283996005 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 310.71875, "completions/mean_terminated_length": 310.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11486212362069637, "epoch": 0.00546, "frac_reward_zero_std": 0.0, "grad_norm": 0.6154648065567017, "kl": 0.5646991468966007, "learning_rate": 8.6944797542873e-06, "loss": -0.0194, "num_tokens": 12794379.0, "reward": 0.22812500596046448, "reward_std": 0.2904698848724365, "rewards/rollout_reward_func/mean": 0.22812500596046448, "rewards/rollout_reward_func/std": 0.27967074513435364, "sampling/importance_sampling_ratio/max": 1.4429833889007568, "sampling/importance_sampling_ratio/mean": 0.9365019798278809, "sampling/importance_sampling_ratio/min": 0.275563508272171, "sampling/sampling_logp_difference/max": 1.297104835510254, "sampling/sampling_logp_difference/mean": 0.05179084092378616, "step": 273, "step_time": 14.410506149997673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 311.28125, "completions/mean_terminated_length": 311.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09207304450683296, "epoch": 0.00548, "frac_reward_zero_std": 0.0, "grad_norm": 0.3823791444301605, "kl": 1.0575015526264906, "learning_rate": 8.68413537514243e-06, "loss": 0.003, "num_tokens": 12843396.0, "reward": 0.11562499403953552, "reward_std": 0.30025336146354675, "rewards/rollout_reward_func/mean": 0.11562499403953552, "rewards/rollout_reward_func/std": 0.30702459812164307, "sampling/importance_sampling_ratio/max": 1.3572819232940674, "sampling/importance_sampling_ratio/mean": 0.9045702219009399, "sampling/importance_sampling_ratio/min": 0.2319292277097702, "sampling/sampling_logp_difference/max": 1.4640123844146729, "sampling/sampling_logp_difference/mean": 0.0561479814350605, "step": 274, "step_time": 13.791416688996833 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01302083395421505, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 231.53125, "completions/mean_terminated_length": 231.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12306687200907618, "epoch": 0.0055, "frac_reward_zero_std": 0.0, "grad_norm": 0.7242388129234314, "kl": 7.125034134835005, "learning_rate": 8.673758888106481e-06, "loss": 0.019, "num_tokens": 12889986.0, "reward": 0.14687500894069672, "reward_std": 0.17995718121528625, "rewards/rollout_reward_func/mean": 0.14687500894069672, "rewards/rollout_reward_func/std": 0.17957723140716553, "sampling/importance_sampling_ratio/max": 2.802408456802368, "sampling/importance_sampling_ratio/mean": 0.9565543532371521, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8500878810882568, "sampling/sampling_logp_difference/mean": 0.08483188599348068, "step": 275, "step_time": 14.202964511008759 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 332.0625, "completions/mean_terminated_length": 332.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1483266082359478, "epoch": 0.00552, "frac_reward_zero_std": 0.0, "grad_norm": 0.5265971422195435, "kl": 1.516592474654317, "learning_rate": 8.663350430052328e-06, "loss": -0.0264, "num_tokens": 12940612.0, "reward": 0.14375001192092896, "reward_std": 0.27444297075271606, "rewards/rollout_reward_func/mean": 0.14375001192092896, "rewards/rollout_reward_func/std": 0.2758418321609497, "sampling/importance_sampling_ratio/max": 1.609852910041809, "sampling/importance_sampling_ratio/mean": 0.8914026021957397, "sampling/importance_sampling_ratio/min": 0.18611541390419006, "sampling/sampling_logp_difference/max": 1.2881243228912354, "sampling/sampling_logp_difference/mean": 0.07248716056346893, "step": 276, "step_time": 14.256014362996211 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 220.5625, "completions/mean_terminated_length": 220.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08957538334652781, "epoch": 0.00554, "frac_reward_zero_std": 0.0, "grad_norm": 3.57395339012146, "kl": 17.33286767732352, "learning_rate": 8.652910138274573e-06, "loss": 0.0134, "num_tokens": 12985554.0, "reward": 0.13750001788139343, "reward_std": 0.2839033603668213, "rewards/rollout_reward_func/mean": 0.13750001788139343, "rewards/rollout_reward_func/std": 0.28026485443115234, "sampling/importance_sampling_ratio/max": 1.1575825214385986, "sampling/importance_sampling_ratio/mean": 0.9009212851524353, "sampling/importance_sampling_ratio/min": 0.17454297840595245, "sampling/sampling_logp_difference/max": 1.7457141876220703, "sampling/sampling_logp_difference/mean": 0.07006211578845978, "step": 277, "step_time": 13.396135915998457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 303.28125, "completions/mean_terminated_length": 303.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08479212503880262, "epoch": 0.00556, "frac_reward_zero_std": 0.0, "grad_norm": 0.2855192720890045, "kl": 0.6068007480353117, "learning_rate": 8.642438150487718e-06, "loss": 0.0054, "num_tokens": 13034352.0, "reward": 0.21875, "reward_std": 0.2537975311279297, "rewards/rollout_reward_func/mean": 0.21875, "rewards/rollout_reward_func/std": 0.25707724690437317, "sampling/importance_sampling_ratio/max": 1.0504711866378784, "sampling/importance_sampling_ratio/mean": 0.9377983212471008, "sampling/importance_sampling_ratio/min": 0.3289322257041931, "sampling/sampling_logp_difference/max": 1.12937593460083, "sampling/sampling_logp_difference/mean": 0.03378678113222122, "step": 278, "step_time": 14.655666208993352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 303.0625, "completions/mean_terminated_length": 303.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1885401306208223, "epoch": 0.00558, "frac_reward_zero_std": 0.0, "grad_norm": 0.45387983322143555, "kl": 0.9142861003056169, "learning_rate": 8.631934604824366e-06, "loss": 0.0087, "num_tokens": 13083496.0, "reward": 0.09062500298023224, "reward_std": 0.23952189087867737, "rewards/rollout_reward_func/mean": 0.09062500298023224, "rewards/rollout_reward_func/std": 0.23329253494739532, "sampling/importance_sampling_ratio/max": 1.5007729530334473, "sampling/importance_sampling_ratio/mean": 0.9255662560462952, "sampling/importance_sampling_ratio/min": 0.23706291615962982, "sampling/sampling_logp_difference/max": 1.4393117427825928, "sampling/sampling_logp_difference/mean": 0.06815847009420395, "step": 279, "step_time": 13.9161696340052 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 285.125, "completions/mean_terminated_length": 285.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1548420733306557, "epoch": 0.0056, "frac_reward_zero_std": 0.0, "grad_norm": 0.35010793805122375, "kl": 0.6897830329835415, "learning_rate": 8.62139963983338e-06, "loss": -0.0372, "num_tokens": 13133766.0, "reward": 0.09375, "reward_std": 0.30950433015823364, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.30684396624565125, "sampling/importance_sampling_ratio/max": 1.0335659980773926, "sampling/importance_sampling_ratio/mean": 0.8736499547958374, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0131332874298096, "sampling/sampling_logp_difference/mean": 0.05709452927112579, "step": 280, "step_time": 13.487252722999983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 326.40625, "completions/mean_terminated_length": 326.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1325820756610483, "epoch": 0.00562, "frac_reward_zero_std": 0.0, "grad_norm": 0.9491053223609924, "kl": 10.295347064733505, "learning_rate": 8.610833394478073e-06, "loss": 0.0046, "num_tokens": 13184866.0, "reward": 0.10312500596046448, "reward_std": 0.225444033741951, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.2334652990102768, "sampling/importance_sampling_ratio/max": 1.3653637170791626, "sampling/importance_sampling_ratio/mean": 0.9162615537643433, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9902029037475586, "sampling/sampling_logp_difference/mean": 0.05956553667783737, "step": 281, "step_time": 14.394511859001796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 291.59375, "completions/mean_terminated_length": 291.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19095565332099795, "epoch": 0.00564, "frac_reward_zero_std": 0.0, "grad_norm": 0.31640616059303284, "kl": 1.1578752407804132, "learning_rate": 8.60023600813436e-06, "loss": -0.0419, "num_tokens": 13233949.0, "reward": 0.07187500596046448, "reward_std": 0.35307905077934265, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.346628338098526, "sampling/importance_sampling_ratio/max": 1.3238170146942139, "sampling/importance_sampling_ratio/mean": 0.8209272623062134, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8257290124893188, "sampling/sampling_logp_difference/mean": 0.11165203154087067, "step": 282, "step_time": 13.496410758994898 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 275.84375, "completions/mean_terminated_length": 275.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16047131083905697, "epoch": 0.00566, "frac_reward_zero_std": 0.0, "grad_norm": 0.4401945471763611, "kl": 0.6654954310506582, "learning_rate": 8.589607620588937e-06, "loss": 0.0206, "num_tokens": 13283007.0, "reward": 0.10625000298023224, "reward_std": 0.2194211632013321, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.23546454310417175, "sampling/importance_sampling_ratio/max": 2.156156063079834, "sampling/importance_sampling_ratio/mean": 0.9546959400177002, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3110320568084717, "sampling/sampling_logp_difference/mean": 0.06736572086811066, "step": 283, "step_time": 13.43887627598815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 286.28125, "completions/mean_terminated_length": 286.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17149520851671696, "epoch": 0.00568, "frac_reward_zero_std": 0.0, "grad_norm": 0.792773962020874, "kl": 0.6537107424810529, "learning_rate": 8.578948372037416e-06, "loss": -0.0012, "num_tokens": 13331277.0, "reward": 0.08125000447034836, "reward_std": 0.3658958673477173, "rewards/rollout_reward_func/mean": 0.08125000447034836, "rewards/rollout_reward_func/std": 0.3736804723739624, "sampling/importance_sampling_ratio/max": 1.5076568126678467, "sampling/importance_sampling_ratio/mean": 0.9082181453704834, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.74648380279541, "sampling/sampling_logp_difference/mean": 0.08482630550861359, "step": 284, "step_time": 16.051442683012283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1868.0, "completions/max_terminated_length": 1868.0, "completions/mean_length": 432.65625, "completions/mean_terminated_length": 432.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21592833660542965, "epoch": 0.0057, "frac_reward_zero_std": 0.0, "grad_norm": 0.31317824125289917, "kl": 1.6251688189804554, "learning_rate": 8.568258403082492e-06, "loss": -0.025, "num_tokens": 13385728.0, "reward": 0.07499999552965164, "reward_std": 0.2878904938697815, "rewards/rollout_reward_func/mean": 0.07499999552965164, "rewards/rollout_reward_func/std": 0.30900463461875916, "sampling/importance_sampling_ratio/max": 1.0697437524795532, "sampling/importance_sampling_ratio/mean": 0.7952969074249268, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3148040771484375, "sampling/sampling_logp_difference/mean": 0.09557053446769714, "step": 285, "step_time": 16.724279587000638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 347.25, "completions/mean_terminated_length": 347.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19478257955051959, "epoch": 0.00572, "frac_reward_zero_std": 0.0, "grad_norm": 0.6409355401992798, "kl": 1.1060534752905369, "learning_rate": 8.557537854732082e-06, "loss": 0.0573, "num_tokens": 13435180.0, "reward": 0.09062500298023224, "reward_std": 0.26181453466415405, "rewards/rollout_reward_func/mean": 0.09062500298023224, "rewards/rollout_reward_func/std": 0.28438884019851685, "sampling/importance_sampling_ratio/max": 2.1853156089782715, "sampling/importance_sampling_ratio/mean": 0.9878605604171753, "sampling/importance_sampling_ratio/min": 0.07072144746780396, "sampling/sampling_logp_difference/max": 2.672297239303589, "sampling/sampling_logp_difference/mean": 0.09242834150791168, "step": 286, "step_time": 15.132884185997682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 614.0, "completions/max_terminated_length": 583.0, "completions/mean_length": 168.90625, "completions/mean_terminated_length": 154.5483856201172, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21618563449010253, "epoch": 0.00574, "frac_reward_zero_std": 0.0, "grad_norm": 1.00492525100708, "kl": 0.8431749455630779, "learning_rate": 8.546786868397465e-06, "loss": 0.0508, "num_tokens": 13478608.0, "reward": 0.10625000298023224, "reward_std": 0.2537144124507904, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.25895261764526367, "sampling/importance_sampling_ratio/max": 1.29150390625, "sampling/importance_sampling_ratio/mean": 0.9067602157592773, "sampling/importance_sampling_ratio/min": 0.27579548954963684, "sampling/sampling_logp_difference/max": 1.313395380973816, "sampling/sampling_logp_difference/mean": 0.07010505348443985, "step": 287, "step_time": 11.868028813005367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 374.75, "completions/mean_terminated_length": 386.32257080078125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.46697093825787306, "epoch": 0.00576, "frac_reward_zero_std": 0.0, "grad_norm": 2.0873677730560303, "kl": 1.0922772772610188, "learning_rate": 8.536005585891422e-06, "loss": 0.0375, "num_tokens": 13530750.0, "reward": -0.031249992549419403, "reward_std": 0.4089580774307251, "rewards/rollout_reward_func/mean": -0.031249992549419403, "rewards/rollout_reward_func/std": 0.4313800036907196, "sampling/importance_sampling_ratio/max": 2.2862393856048584, "sampling/importance_sampling_ratio/mean": 0.8907387852668762, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 16.75587272644043, "sampling/sampling_logp_difference/mean": 0.19678056240081787, "step": 288, "step_time": 14.130388941001002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1715.0, "completions/max_terminated_length": 1715.0, "completions/mean_length": 476.78125, "completions/mean_terminated_length": 476.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26523103984072804, "epoch": 0.00578, "frac_reward_zero_std": 0.0, "grad_norm": 2.2680776119232178, "kl": 1.2422049986198545, "learning_rate": 8.525194149426354e-06, "loss": 0.0611, "num_tokens": 13585905.0, "reward": -0.09062499552965164, "reward_std": 0.4153372347354889, "rewards/rollout_reward_func/mean": -0.09062499552965164, "rewards/rollout_reward_func/std": 0.4191845953464508, "sampling/importance_sampling_ratio/max": 2.3383443355560303, "sampling/importance_sampling_ratio/mean": 0.9233583211898804, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.097090721130371, "sampling/sampling_logp_difference/mean": 0.12902361154556274, "step": 289, "step_time": 16.945172658004594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 308.46875, "completions/mean_terminated_length": 308.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24431125866249204, "epoch": 0.0058, "frac_reward_zero_std": 0.0, "grad_norm": 0.46173736453056335, "kl": 1.4752323236316442, "learning_rate": 8.514352701612418e-06, "loss": 0.0343, "num_tokens": 13635294.0, "reward": 0.02187500149011612, "reward_std": 0.3423672914505005, "rewards/rollout_reward_func/mean": 0.02187500149011612, "rewards/rollout_reward_func/std": 0.3480214774608612, "sampling/importance_sampling_ratio/max": 1.8345601558685303, "sampling/importance_sampling_ratio/mean": 0.8767729997634888, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1500372886657715, "sampling/sampling_logp_difference/mean": 0.0843380019068718, "step": 290, "step_time": 15.735747185990476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 233.0625, "completions/mean_terminated_length": 233.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23634860711172223, "epoch": 0.00582, "frac_reward_zero_std": 0.0, "grad_norm": 1.185630440711975, "kl": 0.7821237649768591, "learning_rate": 8.503481385455637e-06, "loss": 0.0475, "num_tokens": 13680982.0, "reward": 0.0937500074505806, "reward_std": 0.32356518507003784, "rewards/rollout_reward_func/mean": 0.0937500074505806, "rewards/rollout_reward_func/std": 0.35463517904281616, "sampling/importance_sampling_ratio/max": 2.183959484100342, "sampling/importance_sampling_ratio/mean": 0.9878873825073242, "sampling/importance_sampling_ratio/min": 0.2493574470281601, "sampling/sampling_logp_difference/max": 1.4016598463058472, "sampling/sampling_logp_difference/mean": 0.0824345201253891, "step": 291, "step_time": 13.527004396004486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2160.0, "completions/max_terminated_length": 2160.0, "completions/mean_length": 398.59375, "completions/mean_terminated_length": 398.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22910352144390345, "epoch": 0.00584, "frac_reward_zero_std": 0.0, "grad_norm": 1.8139842748641968, "kl": 1.3336749151349068, "learning_rate": 8.492580344356023e-06, "loss": 0.195, "num_tokens": 13733887.0, "reward": 0.02500000223517418, "reward_std": 0.36490991711616516, "rewards/rollout_reward_func/mean": 0.02500000223517418, "rewards/rollout_reward_func/std": 0.3618946373462677, "sampling/importance_sampling_ratio/max": 2.985208034515381, "sampling/importance_sampling_ratio/mean": 0.9970783591270447, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2463698387145996, "sampling/sampling_logp_difference/mean": 0.07330955564975739, "step": 292, "step_time": 19.69029654399492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 353.28125, "completions/mean_terminated_length": 353.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21503473911434412, "epoch": 0.00586, "frac_reward_zero_std": 0.0, "grad_norm": 0.5777910947799683, "kl": 1.7097720550373197, "learning_rate": 8.481649722105677e-06, "loss": 0.0375, "num_tokens": 13786383.0, "reward": 0.08124999701976776, "reward_std": 0.2883167862892151, "rewards/rollout_reward_func/mean": 0.08124999701976776, "rewards/rollout_reward_func/std": 0.27990493178367615, "sampling/importance_sampling_ratio/max": 2.64475154876709, "sampling/importance_sampling_ratio/mean": 1.0068066120147705, "sampling/importance_sampling_ratio/min": 0.5115438103675842, "sampling/sampling_logp_difference/max": 0.9339767694473267, "sampling/sampling_logp_difference/mean": 0.055581316351890564, "step": 293, "step_time": 13.653863606981758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 289.03125, "completions/mean_terminated_length": 289.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2187310801818967, "epoch": 0.00588, "frac_reward_zero_std": 0.0, "grad_norm": 0.5913192629814148, "kl": 2.239577528089285, "learning_rate": 8.470689662886895e-06, "loss": -0.0066, "num_tokens": 13836647.0, "reward": 0.08750000596046448, "reward_std": 0.3142547309398651, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.32897689938545227, "sampling/importance_sampling_ratio/max": 1.4554392099380493, "sampling/importance_sampling_ratio/mean": 0.974961519241333, "sampling/importance_sampling_ratio/min": 0.42712852358818054, "sampling/sampling_logp_difference/max": 0.8648066520690918, "sampling/sampling_logp_difference/mean": 0.05581973120570183, "step": 294, "step_time": 15.051872520994948 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1679.0, "completions/max_terminated_length": 1679.0, "completions/mean_length": 322.5625, "completions/mean_terminated_length": 322.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20597264310345054, "epoch": 0.0059, "frac_reward_zero_std": 0.0, "grad_norm": 1.8888969421386719, "kl": 0.9934553559869528, "learning_rate": 8.459700311270268e-06, "loss": 0.0826, "num_tokens": 13885302.0, "reward": 0.09687500447034836, "reward_std": 0.3387308120727539, "rewards/rollout_reward_func/mean": 0.09687500447034836, "rewards/rollout_reward_func/std": 0.35694435238838196, "sampling/importance_sampling_ratio/max": 1.8751204013824463, "sampling/importance_sampling_ratio/mean": 0.9889767169952393, "sampling/importance_sampling_ratio/min": 0.2865862548351288, "sampling/sampling_logp_difference/max": 1.0618081092834473, "sampling/sampling_logp_difference/mean": 0.0709524005651474, "step": 295, "step_time": 16.29427546200168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 262.65625, "completions/mean_terminated_length": 262.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15729254065081477, "epoch": 0.00592, "frac_reward_zero_std": 0.0, "grad_norm": 0.5052852630615234, "kl": 0.6131757972761989, "learning_rate": 8.44868181221277e-06, "loss": 0.0068, "num_tokens": 13933173.0, "reward": 0.15937499701976776, "reward_std": 0.21032994985580444, "rewards/rollout_reward_func/mean": 0.15937499701976776, "rewards/rollout_reward_func/std": 0.21229828894138336, "sampling/importance_sampling_ratio/max": 1.5150598287582397, "sampling/importance_sampling_ratio/mean": 1.004104495048523, "sampling/importance_sampling_ratio/min": 0.49155154824256897, "sampling/sampling_logp_difference/max": 0.726284921169281, "sampling/sampling_logp_difference/mean": 0.036122601479291916, "step": 296, "step_time": 11.25803814399842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 784.0, "completions/max_terminated_length": 784.0, "completions/mean_length": 231.46875, "completions/mean_terminated_length": 231.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19885633746162057, "epoch": 0.00594, "frac_reward_zero_std": 0.0, "grad_norm": 0.550079882144928, "kl": 1.9640582855790854, "learning_rate": 8.437634311055855e-06, "loss": -0.0302, "num_tokens": 13981090.0, "reward": 0.046875, "reward_std": 0.2561854422092438, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.2839631736278534, "sampling/importance_sampling_ratio/max": 1.4452333450317383, "sampling/importance_sampling_ratio/mean": 0.9498964548110962, "sampling/importance_sampling_ratio/min": 0.3756193220615387, "sampling/sampling_logp_difference/max": 0.9791855812072754, "sampling/sampling_logp_difference/mean": 0.05220041796565056, "step": 297, "step_time": 12.106304902001284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1279.0, "completions/max_terminated_length": 1279.0, "completions/mean_length": 231.09375, "completions/mean_terminated_length": 231.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18008441710844636, "epoch": 0.00596, "frac_reward_zero_std": 0.0, "grad_norm": 0.5716245770454407, "kl": 2.8720622351393104, "learning_rate": 8.42655795352353e-06, "loss": 0.0015, "num_tokens": 14026814.0, "reward": 0.11249999701976776, "reward_std": 0.27080005407333374, "rewards/rollout_reward_func/mean": 0.11249999701976776, "rewards/rollout_reward_func/std": 0.2779533267021179, "sampling/importance_sampling_ratio/max": 1.394458532333374, "sampling/importance_sampling_ratio/mean": 0.939422607421875, "sampling/importance_sampling_ratio/min": 0.3705349862575531, "sampling/sampling_logp_difference/max": 0.9926671981811523, "sampling/sampling_logp_difference/mean": 0.06413313746452332, "step": 298, "step_time": 15.438975100005337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 182.15625, "completions/mean_terminated_length": 182.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12661381578072906, "epoch": 0.00598, "frac_reward_zero_std": 0.0, "grad_norm": 0.6365452408790588, "kl": 0.9224259480834007, "learning_rate": 8.415452885720438e-06, "loss": 0.0404, "num_tokens": 14071378.0, "reward": 0.15000000596046448, "reward_std": 0.2362779825925827, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.23827336728572845, "sampling/importance_sampling_ratio/max": 2.550424814224243, "sampling/importance_sampling_ratio/mean": 1.0911773443222046, "sampling/importance_sampling_ratio/min": 0.6210566163063049, "sampling/sampling_logp_difference/max": 0.9111095666885376, "sampling/sampling_logp_difference/mean": 0.028788050636649132, "step": 299, "step_time": 12.340228081007808 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 333.09375, "completions/mean_terminated_length": 333.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17813771567307413, "epoch": 0.006, "frac_reward_zero_std": 0.0, "grad_norm": 0.40849074721336365, "kl": 0.6372355427592993, "learning_rate": 8.40431925412993e-06, "loss": 0.0057, "num_tokens": 14121651.0, "reward": 0.14687499403953552, "reward_std": 0.22501882910728455, "rewards/rollout_reward_func/mean": 0.14687499403953552, "rewards/rollout_reward_func/std": 0.24094487726688385, "sampling/importance_sampling_ratio/max": 1.2720229625701904, "sampling/importance_sampling_ratio/mean": 0.9606152772903442, "sampling/importance_sampling_ratio/min": 0.5463894009590149, "sampling/sampling_logp_difference/max": 0.6108841896057129, "sampling/sampling_logp_difference/mean": 0.03428518772125244, "step": 300, "step_time": 12.384232350996172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 283.1875, "completions/mean_terminated_length": 283.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16754126362502575, "epoch": 0.00602, "frac_reward_zero_std": 0.0, "grad_norm": 0.5618429183959961, "kl": 2.267330214381218, "learning_rate": 8.393157205612137e-06, "loss": 0.0011, "num_tokens": 14171142.0, "reward": 0.046875, "reward_std": 0.26623624563217163, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.2577625513076782, "sampling/importance_sampling_ratio/max": 1.4663163423538208, "sampling/importance_sampling_ratio/mean": 0.9587970972061157, "sampling/importance_sampling_ratio/min": 0.4767524003982544, "sampling/sampling_logp_difference/max": 0.7408490180969238, "sampling/sampling_logp_difference/mean": 0.03465832769870758, "step": 301, "step_time": 14.48022267300621 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 312.125, "completions/mean_terminated_length": 312.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18502550246194005, "epoch": 0.00604, "frac_reward_zero_std": 0.0, "grad_norm": 0.6219044923782349, "kl": 0.5516458377242088, "learning_rate": 8.38196688740202e-06, "loss": -0.0148, "num_tokens": 14221477.0, "reward": 0.10625000298023224, "reward_std": 0.23608489334583282, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.2474873960018158, "sampling/importance_sampling_ratio/max": 1.295870065689087, "sampling/importance_sampling_ratio/mean": 1.000359296798706, "sampling/importance_sampling_ratio/min": 0.30508872866630554, "sampling/sampling_logp_difference/max": 0.7871609926223755, "sampling/sampling_logp_difference/mean": 0.030340932309627533, "step": 302, "step_time": 13.94245038099325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 269.0, "completions/mean_terminated_length": 269.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14319319603964686, "epoch": 0.00606, "frac_reward_zero_std": 0.0, "grad_norm": 0.292540580034256, "kl": 1.0625294242054224, "learning_rate": 8.370748447107445e-06, "loss": -0.0014, "num_tokens": 14270567.0, "reward": 0.12187500298023224, "reward_std": 0.25436103343963623, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.276772141456604, "sampling/importance_sampling_ratio/max": 1.2497745752334595, "sampling/importance_sampling_ratio/mean": 0.951073408126831, "sampling/importance_sampling_ratio/min": 0.38412606716156006, "sampling/sampling_logp_difference/max": 0.7353606224060059, "sampling/sampling_logp_difference/mean": 0.039242420345544815, "step": 303, "step_time": 13.324797768997087 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 307.125, "completions/mean_terminated_length": 307.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28879768028855324, "epoch": 0.00608, "frac_reward_zero_std": 0.0, "grad_norm": 0.49717310070991516, "kl": 0.7646013125777245, "learning_rate": 8.359502032707219e-06, "loss": -0.0085, "num_tokens": 14321174.0, "reward": 0.09375, "reward_std": 0.27067074179649353, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.26993128657341003, "sampling/importance_sampling_ratio/max": 1.5502017736434937, "sampling/importance_sampling_ratio/mean": 1.0182225704193115, "sampling/importance_sampling_ratio/min": 0.26666685938835144, "sampling/sampling_logp_difference/max": 1.3350040912628174, "sampling/sampling_logp_difference/mean": 0.039807382971048355, "step": 304, "step_time": 14.597993265000696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 295.15625, "completions/mean_terminated_length": 295.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1678864995483309, "epoch": 0.0061, "frac_reward_zero_std": 0.0, "grad_norm": 0.6580269932746887, "kl": 0.5199214098975062, "learning_rate": 8.34822779254915e-06, "loss": -0.0332, "num_tokens": 14369750.0, "reward": 0.11875000596046448, "reward_std": 0.23417791724205017, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.29559943079948425, "sampling/importance_sampling_ratio/max": 1.5731744766235352, "sampling/importance_sampling_ratio/mean": 1.0244005918502808, "sampling/importance_sampling_ratio/min": 0.5350822806358337, "sampling/sampling_logp_difference/max": 0.6253455281257629, "sampling/sampling_logp_difference/mean": 0.02669564262032509, "step": 305, "step_time": 13.394058127993048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 266.21875, "completions/mean_terminated_length": 266.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15954297059215605, "epoch": 0.00612, "frac_reward_zero_std": 0.0, "grad_norm": 0.5072585344314575, "kl": 0.4800307769328356, "learning_rate": 8.336925875348093e-06, "loss": 0.0188, "num_tokens": 14417427.0, "reward": 0.16562500596046448, "reward_std": 0.2506566047668457, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.2858031988143921, "sampling/importance_sampling_ratio/max": 1.6140754222869873, "sampling/importance_sampling_ratio/mean": 1.0252490043640137, "sampling/importance_sampling_ratio/min": 0.5343079566955566, "sampling/sampling_logp_difference/max": 0.6268399953842163, "sampling/sampling_logp_difference/mean": 0.021396465599536896, "step": 306, "step_time": 14.552868120990752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 273.78125, "completions/mean_terminated_length": 273.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14114367053844035, "epoch": 0.00614, "frac_reward_zero_std": 0.0, "grad_norm": 0.5151171088218689, "kl": 0.5161200240254402, "learning_rate": 8.32559643018397e-06, "loss": 0.0103, "num_tokens": 14465738.0, "reward": 0.10312500596046448, "reward_std": 0.2836323082447052, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.3010566234588623, "sampling/importance_sampling_ratio/max": 1.1390100717544556, "sampling/importance_sampling_ratio/mean": 0.9719565510749817, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6797733306884766, "sampling/sampling_logp_difference/mean": 0.030519019812345505, "step": 307, "step_time": 14.660837739997078 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1229.0, "completions/max_terminated_length": 1229.0, "completions/mean_length": 344.71875, "completions/mean_terminated_length": 344.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14819699036888778, "epoch": 0.00616, "frac_reward_zero_std": 0.0, "grad_norm": 0.6026591062545776, "kl": 0.570689694955945, "learning_rate": 8.314239606499828e-06, "loss": 0.0076, "num_tokens": 14516505.0, "reward": 0.15312500298023224, "reward_std": 0.2418186515569687, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.2565080523490906, "sampling/importance_sampling_ratio/max": 1.1381521224975586, "sampling/importance_sampling_ratio/mean": 0.9366252422332764, "sampling/importance_sampling_ratio/min": 0.262248158454895, "sampling/sampling_logp_difference/max": 0.7786623239517212, "sampling/sampling_logp_difference/mean": 0.044950105249881744, "step": 308, "step_time": 14.343477529000666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 385.625, "completions/mean_terminated_length": 385.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15787953068502247, "epoch": 0.00618, "frac_reward_zero_std": 0.0, "grad_norm": 0.43700599670410156, "kl": 3.072397139854729, "learning_rate": 8.302855554099842e-06, "loss": 0.0117, "num_tokens": 14568324.0, "reward": 0.06875000149011612, "reward_std": 0.2510361969470978, "rewards/rollout_reward_func/mean": 0.06875000149011612, "rewards/rollout_reward_func/std": 0.25328487157821655, "sampling/importance_sampling_ratio/max": 1.11003839969635, "sampling/importance_sampling_ratio/mean": 0.9320287108421326, "sampling/importance_sampling_ratio/min": 0.29081714153289795, "sampling/sampling_logp_difference/max": 1.2351493835449219, "sampling/sampling_logp_difference/mean": 0.038997262716293335, "step": 309, "step_time": 16.74517717100025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 375.1875, "completions/mean_terminated_length": 375.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1669954638928175, "epoch": 0.0062, "frac_reward_zero_std": 0.0, "grad_norm": 0.4276646673679352, "kl": 0.5945504326373339, "learning_rate": 8.29144442314736e-06, "loss": -0.0057, "num_tokens": 14621390.0, "reward": 0.11562500149011612, "reward_std": 0.2433062046766281, "rewards/rollout_reward_func/mean": 0.11562500149011612, "rewards/rollout_reward_func/std": 0.25413912534713745, "sampling/importance_sampling_ratio/max": 1.387615442276001, "sampling/importance_sampling_ratio/mean": 0.9811346530914307, "sampling/importance_sampling_ratio/min": 0.4094439148902893, "sampling/sampling_logp_difference/max": 0.9043400287628174, "sampling/sampling_logp_difference/mean": 0.03610975295305252, "step": 310, "step_time": 14.426628960005473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 381.375, "completions/mean_terminated_length": 381.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15547594777308404, "epoch": 0.00622, "frac_reward_zero_std": 0.0, "grad_norm": 0.5240262746810913, "kl": 0.5006210394203663, "learning_rate": 8.280006364162915e-06, "loss": -0.0024, "num_tokens": 14673838.0, "reward": 0.11250001192092896, "reward_std": 0.29372426867485046, "rewards/rollout_reward_func/mean": 0.11250001192092896, "rewards/rollout_reward_func/std": 0.2991924583911896, "sampling/importance_sampling_ratio/max": 1.1892790794372559, "sampling/importance_sampling_ratio/mean": 0.9480119943618774, "sampling/importance_sampling_ratio/min": 0.39683768153190613, "sampling/sampling_logp_difference/max": 0.9346085786819458, "sampling/sampling_logp_difference/mean": 0.03648554906249046, "step": 311, "step_time": 13.902743968003051 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 333.875, "completions/mean_terminated_length": 333.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1956492569297552, "epoch": 0.00624, "frac_reward_zero_std": 0.0, "grad_norm": 0.6292077302932739, "kl": 0.9453918691724539, "learning_rate": 8.268541528022238e-06, "loss": 0.0145, "num_tokens": 14724573.0, "reward": 0.09687499701976776, "reward_std": 0.2775682806968689, "rewards/rollout_reward_func/mean": 0.09687499701976776, "rewards/rollout_reward_func/std": 0.2989059388637543, "sampling/importance_sampling_ratio/max": 1.4727391004562378, "sampling/importance_sampling_ratio/mean": 1.0022692680358887, "sampling/importance_sampling_ratio/min": 0.6113867163658142, "sampling/sampling_logp_difference/max": 0.4920368194580078, "sampling/sampling_logp_difference/mean": 0.03165724128484726, "step": 312, "step_time": 14.429881931002456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 361.15625, "completions/mean_terminated_length": 361.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20086467964574695, "epoch": 0.00626, "frac_reward_zero_std": 0.0, "grad_norm": 0.4361870288848877, "kl": 2.029970448464155, "learning_rate": 8.257050065954267e-06, "loss": -0.0114, "num_tokens": 14776500.0, "reward": 0.09999999403953552, "reward_std": 0.2819060981273651, "rewards/rollout_reward_func/mean": 0.09999999403953552, "rewards/rollout_reward_func/std": 0.27474328875541687, "sampling/importance_sampling_ratio/max": 1.2208510637283325, "sampling/importance_sampling_ratio/mean": 0.8949195742607117, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6974027156829834, "sampling/sampling_logp_difference/mean": 0.059182047843933105, "step": 313, "step_time": 15.45992461700007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1270.0, "completions/max_terminated_length": 1270.0, "completions/mean_length": 232.0625, "completions/mean_terminated_length": 232.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15042984229512513, "epoch": 0.00628, "frac_reward_zero_std": 0.0, "grad_norm": 0.708594799041748, "kl": 0.5480386782437563, "learning_rate": 8.245532129539153e-06, "loss": 0.011, "num_tokens": 14822719.0, "reward": 0.15625, "reward_std": 0.2571547031402588, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.2513672411441803, "sampling/importance_sampling_ratio/max": 1.2406352758407593, "sampling/importance_sampling_ratio/mean": 0.9795144200325012, "sampling/importance_sampling_ratio/min": 0.5501402020454407, "sampling/sampling_logp_difference/max": 0.6145000457763672, "sampling/sampling_logp_difference/mean": 0.027305174618959427, "step": 314, "step_time": 14.34870960799526 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 335.0, "completions/mean_terminated_length": 335.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1577788747381419, "epoch": 0.0063, "frac_reward_zero_std": 0.0, "grad_norm": 0.39623165130615234, "kl": 0.7412934061139822, "learning_rate": 8.233987870706267e-06, "loss": 0.0411, "num_tokens": 14873904.0, "reward": 0.08437500149011612, "reward_std": 0.25073790550231934, "rewards/rollout_reward_func/mean": 0.08437500149011612, "rewards/rollout_reward_func/std": 0.2490101307630539, "sampling/importance_sampling_ratio/max": 1.97944176197052, "sampling/importance_sampling_ratio/mean": 1.0138933658599854, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7231369018554688, "sampling/sampling_logp_difference/mean": 0.05353935435414314, "step": 315, "step_time": 14.61906946599629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 288.0625, "completions/mean_terminated_length": 288.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12056317133828998, "epoch": 0.00632, "frac_reward_zero_std": 0.0, "grad_norm": 0.2674139440059662, "kl": 1.2725303256884217, "learning_rate": 8.222417441732182e-06, "loss": 0.0264, "num_tokens": 14922705.0, "reward": 0.109375, "reward_std": 0.1799824833869934, "rewards/rollout_reward_func/mean": 0.109375, "rewards/rollout_reward_func/std": 0.18898604810237885, "sampling/importance_sampling_ratio/max": 1.149341344833374, "sampling/importance_sampling_ratio/mean": 0.9236140847206116, "sampling/importance_sampling_ratio/min": 0.3534582555294037, "sampling/sampling_logp_difference/max": 0.7318165302276611, "sampling/sampling_logp_difference/mean": 0.042304836213588715, "step": 316, "step_time": 13.378363542993611 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 250.4375, "completions/mean_terminated_length": 250.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17152168601751328, "epoch": 0.00634, "frac_reward_zero_std": 0.0, "grad_norm": 0.4030093848705292, "kl": 3.546752866357565, "learning_rate": 8.210820995238682e-06, "loss": 0.0115, "num_tokens": 14970737.0, "reward": 0.10312500596046448, "reward_std": 0.23738861083984375, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.27648061513900757, "sampling/importance_sampling_ratio/max": 1.1638432741165161, "sampling/importance_sampling_ratio/mean": 0.9615205526351929, "sampling/importance_sampling_ratio/min": 0.4723591208457947, "sampling/sampling_logp_difference/max": 0.7610911130905151, "sampling/sampling_logp_difference/mean": 0.03497634083032608, "step": 317, "step_time": 13.50851987700662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 335.1875, "completions/mean_terminated_length": 335.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17815036792308092, "epoch": 0.00636, "frac_reward_zero_std": 0.0, "grad_norm": 0.6301501989364624, "kl": 0.7164746914058924, "learning_rate": 8.199198684190737e-06, "loss": 0.0273, "num_tokens": 15020470.0, "reward": 0.10000000894069672, "reward_std": 0.27073952555656433, "rewards/rollout_reward_func/mean": 0.10000000894069672, "rewards/rollout_reward_func/std": 0.26880860328674316, "sampling/importance_sampling_ratio/max": 1.6828030347824097, "sampling/importance_sampling_ratio/mean": 0.9415502548217773, "sampling/importance_sampling_ratio/min": 0.2189161628484726, "sampling/sampling_logp_difference/max": 1.5259954929351807, "sampling/sampling_logp_difference/mean": 0.06628520786762238, "step": 318, "step_time": 14.03266311200423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 379.0, "completions/mean_terminated_length": 379.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13796880887821317, "epoch": 0.00638, "frac_reward_zero_std": 0.0, "grad_norm": 0.27696987986564636, "kl": 1.0807942319661379, "learning_rate": 8.187550661894482e-06, "loss": -0.0161, "num_tokens": 15072303.0, "reward": 0.21562500298023224, "reward_std": 0.29885706305503845, "rewards/rollout_reward_func/mean": 0.21562500298023224, "rewards/rollout_reward_func/std": 0.3243945837020874, "sampling/importance_sampling_ratio/max": 1.6379611492156982, "sampling/importance_sampling_ratio/mean": 0.9182924032211304, "sampling/importance_sampling_ratio/min": 0.31302404403686523, "sampling/sampling_logp_difference/max": 1.1698098182678223, "sampling/sampling_logp_difference/mean": 0.05031592398881912, "step": 319, "step_time": 13.95372171600684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1240.0, "completions/max_terminated_length": 1240.0, "completions/mean_length": 328.78125, "completions/mean_terminated_length": 328.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23904289514757693, "epoch": 0.0064, "frac_reward_zero_std": 0.0, "grad_norm": 1.4106658697128296, "kl": 0.8405934758484364, "learning_rate": 8.175877081995205e-06, "loss": 0.1122, "num_tokens": 15122717.0, "reward": -0.02812500298023224, "reward_std": 0.3314964771270752, "rewards/rollout_reward_func/mean": -0.02812500298023224, "rewards/rollout_reward_func/std": 0.3294515013694763, "sampling/importance_sampling_ratio/max": 2.670628547668457, "sampling/importance_sampling_ratio/mean": 1.0171080827713013, "sampling/importance_sampling_ratio/min": 0.5313318967819214, "sampling/sampling_logp_difference/max": 0.9619457721710205, "sampling/sampling_logp_difference/mean": 0.06061507761478424, "step": 320, "step_time": 14.386914999005967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 246.65625, "completions/mean_terminated_length": 246.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15907831001095474, "epoch": 0.00642, "frac_reward_zero_std": 0.0, "grad_norm": 0.5396546125411987, "kl": 1.802431421354413, "learning_rate": 8.16417809847532e-06, "loss": -0.0041, "num_tokens": 15169061.0, "reward": 0.17812499403953552, "reward_std": 0.2814944386482239, "rewards/rollout_reward_func/mean": 0.17812499403953552, "rewards/rollout_reward_func/std": 0.2744312286376953, "sampling/importance_sampling_ratio/max": 1.853975534439087, "sampling/importance_sampling_ratio/mean": 0.938116192817688, "sampling/importance_sampling_ratio/min": 0.16502806544303894, "sampling/sampling_logp_difference/max": 1.8111921548843384, "sampling/sampling_logp_difference/mean": 0.07156681269407272, "step": 321, "step_time": 14.21060776700324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 218.84375, "completions/mean_terminated_length": 218.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14793619373813272, "epoch": 0.00644, "frac_reward_zero_std": 0.0, "grad_norm": 0.4553847312927246, "kl": 2.7993745636194944, "learning_rate": 8.152453865652327e-06, "loss": -0.0094, "num_tokens": 15214348.0, "reward": 0.16562500596046448, "reward_std": 0.26067671179771423, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.2924968898296356, "sampling/importance_sampling_ratio/max": 1.1415505409240723, "sampling/importance_sampling_ratio/mean": 0.9022704362869263, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1821939945220947, "sampling/sampling_logp_difference/mean": 0.05482861399650574, "step": 322, "step_time": 13.111208406993683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 278.40625, "completions/mean_terminated_length": 278.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13128780107945204, "epoch": 0.00646, "frac_reward_zero_std": 0.0, "grad_norm": 0.46980318427085876, "kl": 1.0428384058177471, "learning_rate": 8.140704538176782e-06, "loss": -0.0301, "num_tokens": 15262289.0, "reward": 0.21562500298023224, "reward_std": 0.2539578378200531, "rewards/rollout_reward_func/mean": 0.21562500298023224, "rewards/rollout_reward_func/std": 0.27837011218070984, "sampling/importance_sampling_ratio/max": 1.5126523971557617, "sampling/importance_sampling_ratio/mean": 0.9792702198028564, "sampling/importance_sampling_ratio/min": 0.429586261510849, "sampling/sampling_logp_difference/max": 0.8549462556838989, "sampling/sampling_logp_difference/mean": 0.03700302168726921, "step": 323, "step_time": 13.7758975680008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 283.46875, "completions/mean_terminated_length": 283.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12310619791969657, "epoch": 0.00648, "frac_reward_zero_std": 0.0, "grad_norm": 0.6854049563407898, "kl": 0.9398440178483725, "learning_rate": 8.128930271030256e-06, "loss": 0.0767, "num_tokens": 15310330.0, "reward": 0.13750000298023224, "reward_std": 0.2863636612892151, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.3045356273651123, "sampling/importance_sampling_ratio/max": 2.884281635284424, "sampling/importance_sampling_ratio/mean": 1.0874714851379395, "sampling/importance_sampling_ratio/min": 0.44521602988243103, "sampling/sampling_logp_difference/max": 1.0430808067321777, "sampling/sampling_logp_difference/mean": 0.05290799215435982, "step": 324, "step_time": 14.259939848998329 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 386.75, "completions/mean_terminated_length": 386.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1460539244581014, "epoch": 0.0065, "frac_reward_zero_std": 0.0, "grad_norm": 0.4736587405204773, "kl": 0.7307285955175757, "learning_rate": 8.117131219523293e-06, "loss": 0.0336, "num_tokens": 15362185.0, "reward": 0.08437499403953552, "reward_std": 0.31785619258880615, "rewards/rollout_reward_func/mean": 0.08437499403953552, "rewards/rollout_reward_func/std": 0.3091188073158264, "sampling/importance_sampling_ratio/max": 2.3569395542144775, "sampling/importance_sampling_ratio/mean": 0.9758723974227905, "sampling/importance_sampling_ratio/min": 0.3088628351688385, "sampling/sampling_logp_difference/max": 1.2162129878997803, "sampling/sampling_logp_difference/mean": 0.06482815742492676, "step": 325, "step_time": 13.877573229998234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 250.59375, "completions/mean_terminated_length": 250.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14547241479158401, "epoch": 0.00652, "frac_reward_zero_std": 0.0, "grad_norm": 0.3537554144859314, "kl": 0.5309886410832405, "learning_rate": 8.10530753929336e-06, "loss": 0.0035, "num_tokens": 15409388.0, "reward": 0.125, "reward_std": 0.28176149725914, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.2839809060096741, "sampling/importance_sampling_ratio/max": 1.0646660327911377, "sampling/importance_sampling_ratio/mean": 0.9347048997879028, "sampling/importance_sampling_ratio/min": 0.48115602135658264, "sampling/sampling_logp_difference/max": 0.7378919124603271, "sampling/sampling_logp_difference/mean": 0.034664321690797806, "step": 326, "step_time": 14.108651892001944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 300.28125, "completions/mean_terminated_length": 300.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1433143289759755, "epoch": 0.00654, "frac_reward_zero_std": 0.0, "grad_norm": 0.7838330864906311, "kl": 1.6480639055371284, "learning_rate": 8.093459386302788e-06, "loss": -0.0113, "num_tokens": 15459833.0, "reward": 0.16562499105930328, "reward_std": 0.24346080422401428, "rewards/rollout_reward_func/mean": 0.16562499105930328, "rewards/rollout_reward_func/std": 0.26102694869041443, "sampling/importance_sampling_ratio/max": 1.6010336875915527, "sampling/importance_sampling_ratio/mean": 0.9811586737632751, "sampling/importance_sampling_ratio/min": 0.5337523818016052, "sampling/sampling_logp_difference/max": 0.5962823033332825, "sampling/sampling_logp_difference/mean": 0.033645447343587875, "step": 327, "step_time": 14.297025722000399 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 357.09375, "completions/mean_terminated_length": 357.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17110442742705345, "epoch": 0.00656, "frac_reward_zero_std": 0.0, "grad_norm": 0.6039538979530334, "kl": 3.3760250732302666, "learning_rate": 8.081586916836728e-06, "loss": 0.0095, "num_tokens": 15511094.0, "reward": 0.08124999701976776, "reward_std": 0.268825888633728, "rewards/rollout_reward_func/mean": 0.08124999701976776, "rewards/rollout_reward_func/std": 0.275256484746933, "sampling/importance_sampling_ratio/max": 2.1263318061828613, "sampling/importance_sampling_ratio/mean": 0.9172385931015015, "sampling/importance_sampling_ratio/min": 0.19680993258953094, "sampling/sampling_logp_difference/max": 1.6255393028259277, "sampling/sampling_logp_difference/mean": 0.07483972609043121, "step": 328, "step_time": 13.912840037999558 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 242.15625, "completions/mean_terminated_length": 242.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10364327183924615, "epoch": 0.00658, "frac_reward_zero_std": 0.0, "grad_norm": 0.256538063287735, "kl": 0.6668004542589188, "learning_rate": 8.069690287501078e-06, "loss": 0.0377, "num_tokens": 15557669.0, "reward": 0.18125000596046448, "reward_std": 0.23181653022766113, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.24943484365940094, "sampling/importance_sampling_ratio/max": 1.6683754920959473, "sampling/importance_sampling_ratio/mean": 1.0185294151306152, "sampling/importance_sampling_ratio/min": 0.3334856927394867, "sampling/sampling_logp_difference/max": 1.098230242729187, "sampling/sampling_logp_difference/mean": 0.036697451025247574, "step": 329, "step_time": 15.05272591200628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 297.9375, "completions/mean_terminated_length": 297.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1417910943273455, "epoch": 0.0066, "frac_reward_zero_std": 0.0, "grad_norm": 0.7617589831352234, "kl": 0.7079621385782957, "learning_rate": 8.05776965522042e-06, "loss": 0.0481, "num_tokens": 15606798.0, "reward": 0.15625, "reward_std": 0.23788253962993622, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.23270845413208008, "sampling/importance_sampling_ratio/max": 2.15647554397583, "sampling/importance_sampling_ratio/mean": 1.0018967390060425, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5710179805755615, "sampling/sampling_logp_difference/mean": 0.0574718713760376, "step": 330, "step_time": 13.658581972005777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 275.1875, "completions/mean_terminated_length": 275.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12755008996464312, "epoch": 0.00662, "frac_reward_zero_std": 0.0, "grad_norm": 0.7499209046363831, "kl": 5.574789585545659, "learning_rate": 8.045825177235952e-06, "loss": 0.0327, "num_tokens": 15655091.0, "reward": 0.09687499701976776, "reward_std": 0.2230929434299469, "rewards/rollout_reward_func/mean": 0.09687499701976776, "rewards/rollout_reward_func/std": 0.26577261090278625, "sampling/importance_sampling_ratio/max": 2.049574136734009, "sampling/importance_sampling_ratio/mean": 0.9909869432449341, "sampling/importance_sampling_ratio/min": 0.6445604562759399, "sampling/sampling_logp_difference/max": 0.7083028554916382, "sampling/sampling_logp_difference/mean": 0.030763696879148483, "step": 331, "step_time": 13.834946259998105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 263.6875, "completions/mean_terminated_length": 263.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1541876953560859, "epoch": 0.00664, "frac_reward_zero_std": 0.0, "grad_norm": 0.38272297382354736, "kl": 1.3280393378809094, "learning_rate": 8.033857011103411e-06, "loss": 0.0132, "num_tokens": 15701542.0, "reward": 0.08749999850988388, "reward_std": 0.29606884717941284, "rewards/rollout_reward_func/mean": 0.08749999850988388, "rewards/rollout_reward_func/std": 0.3045355975627899, "sampling/importance_sampling_ratio/max": 1.328816533088684, "sampling/importance_sampling_ratio/mean": 0.9532124996185303, "sampling/importance_sampling_ratio/min": 0.38350871205329895, "sampling/sampling_logp_difference/max": 1.0967206954956055, "sampling/sampling_logp_difference/mean": 0.04232606664299965, "step": 332, "step_time": 17.11770109101417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 229.65625, "completions/mean_terminated_length": 229.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08164019044488668, "epoch": 0.00666, "frac_reward_zero_std": 0.0, "grad_norm": 0.1854148954153061, "kl": 4.45452862419188, "learning_rate": 8.021865314691006e-06, "loss": -0.0078, "num_tokens": 15748523.0, "reward": 0.08749999850988388, "reward_std": 0.23511545360088348, "rewards/rollout_reward_func/mean": 0.08749999850988388, "rewards/rollout_reward_func/std": 0.24724285304546356, "sampling/importance_sampling_ratio/max": 1.1426554918289185, "sampling/importance_sampling_ratio/mean": 0.9648246169090271, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3725619316101074, "sampling/sampling_logp_difference/mean": 0.0326923169195652, "step": 333, "step_time": 12.284416409005644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 254.09375, "completions/mean_terminated_length": 254.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1130609365645796, "epoch": 0.00668, "frac_reward_zero_std": 0.0, "grad_norm": 2.232028007507324, "kl": 7.151356576010585, "learning_rate": 8.009850246177311e-06, "loss": 0.0273, "num_tokens": 15796390.0, "reward": 0.09375, "reward_std": 0.24072201550006866, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.2526472806930542, "sampling/importance_sampling_ratio/max": 2.704064130783081, "sampling/importance_sampling_ratio/mean": 1.0320379734039307, "sampling/importance_sampling_ratio/min": 0.3011828064918518, "sampling/sampling_logp_difference/max": 1.207045078277588, "sampling/sampling_logp_difference/mean": 0.039058439433574677, "step": 334, "step_time": 13.396486765996087 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 242.34375, "completions/mean_terminated_length": 242.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14204370533116162, "epoch": 0.0067, "frac_reward_zero_std": 0.0, "grad_norm": 0.5981636047363281, "kl": 0.6733309701085091, "learning_rate": 7.997811964049207e-06, "loss": 0.0372, "num_tokens": 15841704.0, "reward": 0.10000000894069672, "reward_std": 0.32711920142173767, "rewards/rollout_reward_func/mean": 0.10000000894069672, "rewards/rollout_reward_func/std": 0.32824066281318665, "sampling/importance_sampling_ratio/max": 1.6049689054489136, "sampling/importance_sampling_ratio/mean": 1.003380298614502, "sampling/importance_sampling_ratio/min": 0.2977748513221741, "sampling/sampling_logp_difference/max": 1.2114707231521606, "sampling/sampling_logp_difference/mean": 0.042678624391555786, "step": 335, "step_time": 14.897865048002132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 397.15625, "completions/mean_terminated_length": 397.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1849997374229133, "epoch": 0.00672, "frac_reward_zero_std": 0.0, "grad_norm": 0.6307199001312256, "kl": 1.4733572229743004, "learning_rate": 7.98575062709977e-06, "loss": 0.0239, "num_tokens": 15894962.0, "reward": 0.05000000819563866, "reward_std": 0.24568292498588562, "rewards/rollout_reward_func/mean": 0.05000000819563866, "rewards/rollout_reward_func/std": 0.29512161016464233, "sampling/importance_sampling_ratio/max": 1.6875823736190796, "sampling/importance_sampling_ratio/mean": 0.9792556762695312, "sampling/importance_sampling_ratio/min": 0.22683024406433105, "sampling/sampling_logp_difference/max": 1.4836034774780273, "sampling/sampling_logp_difference/mean": 0.056426648050546646, "step": 336, "step_time": 13.927318055008072 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 253.25, "completions/mean_terminated_length": 253.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10719312611036003, "epoch": 0.00674, "frac_reward_zero_std": 0.0, "grad_norm": 1.0870271921157837, "kl": 5.07798844575882, "learning_rate": 7.97366639442619e-06, "loss": 0.0218, "num_tokens": 15942883.0, "reward": 0.11562500149011612, "reward_std": 0.26310643553733826, "rewards/rollout_reward_func/mean": 0.11562500149011612, "rewards/rollout_reward_func/std": 0.2616441547870636, "sampling/importance_sampling_ratio/max": 1.2926524877548218, "sampling/importance_sampling_ratio/mean": 0.9687758684158325, "sampling/importance_sampling_ratio/min": 0.419436514377594, "sampling/sampling_logp_difference/max": 0.8688173294067383, "sampling/sampling_logp_difference/mean": 0.029333725571632385, "step": 337, "step_time": 13.678563578996545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 242.59375, "completions/mean_terminated_length": 242.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14829913713037968, "epoch": 0.00676, "frac_reward_zero_std": 0.0, "grad_norm": 0.3676522672176361, "kl": 0.6040768101811409, "learning_rate": 7.961559425427666e-06, "loss": 0.0155, "num_tokens": 15989470.0, "reward": 0.14687499403953552, "reward_std": 0.2664281725883484, "rewards/rollout_reward_func/mean": 0.14687499403953552, "rewards/rollout_reward_func/std": 0.2839631736278534, "sampling/importance_sampling_ratio/max": 1.4211432933807373, "sampling/importance_sampling_ratio/mean": 1.0236256122589111, "sampling/importance_sampling_ratio/min": 0.5029776096343994, "sampling/sampling_logp_difference/max": 0.6872051954269409, "sampling/sampling_logp_difference/mean": 0.029248224571347237, "step": 338, "step_time": 14.508840381997288 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 865.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 208.5625, "completions/mean_terminated_length": 208.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13674956862814724, "epoch": 0.00678, "frac_reward_zero_std": 0.0, "grad_norm": 0.3180796802043915, "kl": 0.6863129492849112, "learning_rate": 7.949429879803298e-06, "loss": -0.0105, "num_tokens": 16035829.0, "reward": 0.16562500596046448, "reward_std": 0.224996417760849, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.2336379587650299, "sampling/importance_sampling_ratio/max": 1.061179757118225, "sampling/importance_sampling_ratio/mean": 0.9612832069396973, "sampling/importance_sampling_ratio/min": 0.3924678564071655, "sampling/sampling_logp_difference/max": 0.9353060722351074, "sampling/sampling_logp_difference/mean": 0.02742069773375988, "step": 339, "step_time": 12.189815847999853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 263.53125, "completions/mean_terminated_length": 263.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15030353190377355, "epoch": 0.0068, "frac_reward_zero_std": 0.0, "grad_norm": 0.5084715485572815, "kl": 0.7216615434736013, "learning_rate": 7.937277917549998e-06, "loss": 0.0111, "num_tokens": 16082717.0, "reward": 0.15000000596046448, "reward_std": 0.22871172428131104, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.24230079352855682, "sampling/importance_sampling_ratio/max": 1.7840887308120728, "sampling/importance_sampling_ratio/mean": 0.9790428876876831, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2914540767669678, "sampling/sampling_logp_difference/mean": 0.0430787019431591, "step": 340, "step_time": 14.31518670999867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 838.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 222.875, "completions/mean_terminated_length": 222.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1532949097454548, "epoch": 0.00682, "frac_reward_zero_std": 0.0, "grad_norm": 0.38562238216400146, "kl": 1.3198019936680794, "learning_rate": 7.925103698960361e-06, "loss": 0.0045, "num_tokens": 16129826.0, "reward": 0.13124999403953552, "reward_std": 0.30158719420433044, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.3094610273838043, "sampling/importance_sampling_ratio/max": 1.3084156513214111, "sampling/importance_sampling_ratio/mean": 0.9962409734725952, "sampling/importance_sampling_ratio/min": 0.19950342178344727, "sampling/sampling_logp_difference/max": 1.1084604263305664, "sampling/sampling_logp_difference/mean": 0.03978833928704262, "step": 341, "step_time": 12.02986465799404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 309.3125, "completions/mean_terminated_length": 309.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1504993331618607, "epoch": 0.00684, "frac_reward_zero_std": 0.0, "grad_norm": 0.3445810079574585, "kl": 3.6810469832271338, "learning_rate": 7.91290738462056e-06, "loss": -0.0022, "num_tokens": 16179312.0, "reward": 0.12187500298023224, "reward_std": 0.278766393661499, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.26848965883255005, "sampling/importance_sampling_ratio/max": 1.2422165870666504, "sampling/importance_sampling_ratio/mean": 0.9181807041168213, "sampling/importance_sampling_ratio/min": 0.29561373591423035, "sampling/sampling_logp_difference/max": 1.230506181716919, "sampling/sampling_logp_difference/mean": 0.04694633185863495, "step": 342, "step_time": 13.563491343993519 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 260.28125, "completions/mean_terminated_length": 260.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18829140905290842, "epoch": 0.00686, "frac_reward_zero_std": 0.0, "grad_norm": 0.5527353882789612, "kl": 0.5326665407046676, "learning_rate": 7.900689135408226e-06, "loss": -0.0042, "num_tokens": 16227141.0, "reward": 0.13125000894069672, "reward_std": 0.2453775852918625, "rewards/rollout_reward_func/mean": 0.13125000894069672, "rewards/rollout_reward_func/std": 0.2822004556655884, "sampling/importance_sampling_ratio/max": 2.564089059829712, "sampling/importance_sampling_ratio/mean": 1.0021693706512451, "sampling/importance_sampling_ratio/min": 0.5478589534759521, "sampling/sampling_logp_difference/max": 0.8967628479003906, "sampling/sampling_logp_difference/mean": 0.046906132251024246, "step": 343, "step_time": 14.451820623009553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 275.75, "completions/mean_terminated_length": 275.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17421740829013288, "epoch": 0.00688, "frac_reward_zero_std": 0.0, "grad_norm": 0.5056030750274658, "kl": 0.9127302542328835, "learning_rate": 7.888449112490325e-06, "loss": -0.0004, "num_tokens": 16274462.0, "reward": 0.13125000894069672, "reward_std": 0.2699360251426697, "rewards/rollout_reward_func/mean": 0.13125000894069672, "rewards/rollout_reward_func/std": 0.2693331241607666, "sampling/importance_sampling_ratio/max": 1.2400987148284912, "sampling/importance_sampling_ratio/mean": 0.9521600008010864, "sampling/importance_sampling_ratio/min": 0.23201285302639008, "sampling/sampling_logp_difference/max": 0.995079517364502, "sampling/sampling_logp_difference/mean": 0.04664310812950134, "step": 344, "step_time": 13.52221478699721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 251.8125, "completions/mean_terminated_length": 251.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18110323417931795, "epoch": 0.0069, "frac_reward_zero_std": 0.0, "grad_norm": 0.27416282892227173, "kl": 0.9315435336902738, "learning_rate": 7.876187477321033e-06, "loss": -0.0152, "num_tokens": 16322291.0, "reward": 0.13750000298023224, "reward_std": 0.257915198802948, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.2648797929286957, "sampling/importance_sampling_ratio/max": 1.2193236351013184, "sampling/importance_sampling_ratio/mean": 0.9697474241256714, "sampling/importance_sampling_ratio/min": 0.48108676075935364, "sampling/sampling_logp_difference/max": 0.7317228317260742, "sampling/sampling_logp_difference/mean": 0.0346609428524971, "step": 345, "step_time": 13.690804405003291 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 255.09375, "completions/mean_terminated_length": 255.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.195125340949744, "epoch": 0.00692, "frac_reward_zero_std": 0.0, "grad_norm": 0.3259126842021942, "kl": 1.2407248336821795, "learning_rate": 7.86390439163961e-06, "loss": 0.0054, "num_tokens": 16368751.0, "reward": 0.125, "reward_std": 0.2815924882888794, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.2828426957130432, "sampling/importance_sampling_ratio/max": 1.220151662826538, "sampling/importance_sampling_ratio/mean": 0.9765691161155701, "sampling/importance_sampling_ratio/min": 0.445153146982193, "sampling/sampling_logp_difference/max": 0.8175921440124512, "sampling/sampling_logp_difference/mean": 0.033037323504686356, "step": 346, "step_time": 14.805724254998495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 206.78125, "completions/mean_terminated_length": 206.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22204948868602514, "epoch": 0.00694, "frac_reward_zero_std": 0.0, "grad_norm": 1.2198553085327148, "kl": 1.7398968152701855, "learning_rate": 7.851600017468257e-06, "loss": -0.0094, "num_tokens": 16414481.0, "reward": 0.13749998807907104, "reward_std": 0.2622029185295105, "rewards/rollout_reward_func/mean": 0.13749998807907104, "rewards/rollout_reward_func/std": 0.2624328136444092, "sampling/importance_sampling_ratio/max": 1.6768569946289062, "sampling/importance_sampling_ratio/mean": 1.0194387435913086, "sampling/importance_sampling_ratio/min": 0.5885978937149048, "sampling/sampling_logp_difference/max": 0.5367549657821655, "sampling/sampling_logp_difference/mean": 0.04484236240386963, "step": 347, "step_time": 13.190873046991328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 306.46875, "completions/mean_terminated_length": 306.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1800998910330236, "epoch": 0.00696, "frac_reward_zero_std": 0.0, "grad_norm": 0.6144630312919617, "kl": 0.7150081470608711, "learning_rate": 7.839274517109987e-06, "loss": 0.0159, "num_tokens": 16464163.0, "reward": 0.11250000447034836, "reward_std": 0.2780687212944031, "rewards/rollout_reward_func/mean": 0.11250000447034836, "rewards/rollout_reward_func/std": 0.2720887064933777, "sampling/importance_sampling_ratio/max": 1.687119483947754, "sampling/importance_sampling_ratio/mean": 1.0201795101165771, "sampling/importance_sampling_ratio/min": 0.5678369402885437, "sampling/sampling_logp_difference/max": 0.6729409694671631, "sampling/sampling_logp_difference/mean": 0.033709026873111725, "step": 348, "step_time": 13.715107119987806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 344.15625, "completions/mean_terminated_length": 344.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23169474489986897, "epoch": 0.00698, "frac_reward_zero_std": 0.0, "grad_norm": 0.4008009433746338, "kl": 0.5314976433292031, "learning_rate": 7.826928053146482e-06, "loss": 0.0193, "num_tokens": 16514536.0, "reward": 0.14687500894069672, "reward_std": 0.258269727230072, "rewards/rollout_reward_func/mean": 0.14687500894069672, "rewards/rollout_reward_func/std": 0.3110693097114563, "sampling/importance_sampling_ratio/max": 1.2526017427444458, "sampling/importance_sampling_ratio/mean": 0.9801144003868103, "sampling/importance_sampling_ratio/min": 0.2601584792137146, "sampling/sampling_logp_difference/max": 1.346492052078247, "sampling/sampling_logp_difference/mean": 0.03666267544031143, "step": 349, "step_time": 14.365615960996365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 354.96875, "completions/mean_terminated_length": 354.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20111668249592185, "epoch": 0.007, "frac_reward_zero_std": 0.0, "grad_norm": 0.4952649176120758, "kl": 2.4047500984743237, "learning_rate": 7.814560788435947e-06, "loss": -0.001, "num_tokens": 16565496.0, "reward": 0.16875001788139343, "reward_std": 0.2709559202194214, "rewards/rollout_reward_func/mean": 0.16875001788139343, "rewards/rollout_reward_func/std": 0.268132746219635, "sampling/importance_sampling_ratio/max": 1.7873241901397705, "sampling/importance_sampling_ratio/mean": 0.9137978553771973, "sampling/importance_sampling_ratio/min": 0.15646058320999146, "sampling/sampling_logp_difference/max": 1.8624587059020996, "sampling/sampling_logp_difference/mean": 0.06508858501911163, "step": 350, "step_time": 13.83393298199735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 325.46875, "completions/mean_terminated_length": 325.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23271273216232657, "epoch": 0.00702, "frac_reward_zero_std": 0.0, "grad_norm": 0.6716303825378418, "kl": 1.075387821532786, "learning_rate": 7.802172886110968e-06, "loss": -0.0154, "num_tokens": 16616231.0, "reward": 0.11249999701976776, "reward_std": 0.335310697555542, "rewards/rollout_reward_func/mean": 0.11249999701976776, "rewards/rollout_reward_func/std": 0.3471078872680664, "sampling/importance_sampling_ratio/max": 1.6925852298736572, "sampling/importance_sampling_ratio/mean": 0.8914604187011719, "sampling/importance_sampling_ratio/min": 0.3037126958370209, "sampling/sampling_logp_difference/max": 1.1927995681762695, "sampling/sampling_logp_difference/mean": 0.0673644170165062, "step": 351, "step_time": 13.914974929997697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 257.75, "completions/mean_terminated_length": 257.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21109020663425326, "epoch": 0.00704, "frac_reward_zero_std": 0.0, "grad_norm": 0.6646309494972229, "kl": 1.0312822479754686, "learning_rate": 7.789764509576347e-06, "loss": -0.0036, "num_tokens": 16663732.0, "reward": 0.125, "reward_std": 0.3100542426109314, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.29946190118789673, "sampling/importance_sampling_ratio/max": 1.5002620220184326, "sampling/importance_sampling_ratio/mean": 0.9664345979690552, "sampling/importance_sampling_ratio/min": 0.5054823160171509, "sampling/sampling_logp_difference/max": 0.6842690706253052, "sampling/sampling_logp_difference/mean": 0.0420808270573616, "step": 352, "step_time": 14.520815480991587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 227.5625, "completions/mean_terminated_length": 227.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19502560188993812, "epoch": 0.00706, "frac_reward_zero_std": 0.0, "grad_norm": 1.1449531316757202, "kl": 0.9981586150825024, "learning_rate": 7.77733582250696e-06, "loss": 0.0803, "num_tokens": 16709393.0, "reward": 0.07187500596046448, "reward_std": 0.29813605546951294, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.31339389085769653, "sampling/importance_sampling_ratio/max": 2.7152981758117676, "sampling/importance_sampling_ratio/mean": 1.0992059707641602, "sampling/importance_sampling_ratio/min": 0.6221949458122253, "sampling/sampling_logp_difference/max": 0.9870095252990723, "sampling/sampling_logp_difference/mean": 0.039351724088191986, "step": 353, "step_time": 13.45049884999753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 331.96875, "completions/mean_terminated_length": 331.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18001684686169028, "epoch": 0.00708, "frac_reward_zero_std": 0.0, "grad_norm": 0.7031246423721313, "kl": 1.0345863811671734, "learning_rate": 7.764886988845588e-06, "loss": 0.0234, "num_tokens": 16758550.0, "reward": 0.15312500298023224, "reward_std": 0.3425571918487549, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.338863730430603, "sampling/importance_sampling_ratio/max": 2.759425401687622, "sampling/importance_sampling_ratio/mean": 1.007016658782959, "sampling/importance_sampling_ratio/min": 0.22781121730804443, "sampling/sampling_logp_difference/max": 1.4863321781158447, "sampling/sampling_logp_difference/mean": 0.06858152896165848, "step": 354, "step_time": 14.231821381010377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 176.0, "completions/mean_terminated_length": 176.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13259309832938015, "epoch": 0.0071, "frac_reward_zero_std": 0.0, "grad_norm": 0.7383409738540649, "kl": 2.856666462495923, "learning_rate": 7.752418172800769e-06, "loss": 0.0056, "num_tokens": 16801836.0, "reward": 0.12187500298023224, "reward_std": 0.2922535836696625, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.2813955545425415, "sampling/importance_sampling_ratio/max": 1.6178687810897827, "sampling/importance_sampling_ratio/mean": 0.9267283082008362, "sampling/importance_sampling_ratio/min": 0.3716124892234802, "sampling/sampling_logp_difference/max": 1.0040745735168457, "sampling/sampling_logp_difference/mean": 0.055288251489400864, "step": 355, "step_time": 12.41517841199311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 275.59375, "completions/mean_terminated_length": 275.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13701789174228907, "epoch": 0.00712, "frac_reward_zero_std": 0.0, "grad_norm": 0.2861834168434143, "kl": 0.6057512750849128, "learning_rate": 7.73992953884461e-06, "loss": -0.0108, "num_tokens": 16850282.0, "reward": 0.13125000894069672, "reward_std": 0.2967778444290161, "rewards/rollout_reward_func/mean": 0.13125000894069672, "rewards/rollout_reward_func/std": 0.31769075989723206, "sampling/importance_sampling_ratio/max": 1.0569508075714111, "sampling/importance_sampling_ratio/mean": 0.9159774780273438, "sampling/importance_sampling_ratio/min": 0.20064468681812286, "sampling/sampling_logp_difference/max": 1.611046552658081, "sampling/sampling_logp_difference/mean": 0.045928388833999634, "step": 356, "step_time": 13.76182181299373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 259.84375, "completions/mean_terminated_length": 259.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09888907242566347, "epoch": 0.00714, "frac_reward_zero_std": 0.0, "grad_norm": 0.43075689673423767, "kl": 0.484033590182662, "learning_rate": 7.727421251710637e-06, "loss": 0.0297, "num_tokens": 16898080.0, "reward": 0.13124999403953552, "reward_std": 0.2259330153465271, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.23201432824134827, "sampling/importance_sampling_ratio/max": 1.1277868747711182, "sampling/importance_sampling_ratio/mean": 0.9925730228424072, "sampling/importance_sampling_ratio/min": 0.6607670783996582, "sampling/sampling_logp_difference/max": 0.4247422218322754, "sampling/sampling_logp_difference/mean": 0.015792664140462875, "step": 357, "step_time": 14.81657720899966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1688.0, "completions/max_terminated_length": 1688.0, "completions/mean_length": 406.25, "completions/mean_terminated_length": 406.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16163807199336588, "epoch": 0.00716, "frac_reward_zero_std": 0.0, "grad_norm": 0.5321156978607178, "kl": 0.5070551130920649, "learning_rate": 7.714893476391613e-06, "loss": -0.0187, "num_tokens": 16950481.0, "reward": 0.21562498807907104, "reward_std": 0.2901393175125122, "rewards/rollout_reward_func/mean": 0.21562498807907104, "rewards/rollout_reward_func/std": 0.3006545305252075, "sampling/importance_sampling_ratio/max": 1.4020521640777588, "sampling/importance_sampling_ratio/mean": 0.9501447081565857, "sampling/importance_sampling_ratio/min": 0.24182164669036865, "sampling/sampling_logp_difference/max": 1.4195103645324707, "sampling/sampling_logp_difference/mean": 0.04664336144924164, "step": 358, "step_time": 16.039447531005862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 293.71875, "completions/mean_terminated_length": 293.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13415430474560708, "epoch": 0.00718, "frac_reward_zero_std": 0.0, "grad_norm": 0.5629164576530457, "kl": 0.5575214643031359, "learning_rate": 7.702346378137364e-06, "loss": 0.0105, "num_tokens": 17000159.0, "reward": 0.10000000149011612, "reward_std": 0.2896307408809662, "rewards/rollout_reward_func/mean": 0.10000000149011612, "rewards/rollout_reward_func/std": 0.2873684763908386, "sampling/importance_sampling_ratio/max": 1.1150399446487427, "sampling/importance_sampling_ratio/mean": 1.0075020790100098, "sampling/importance_sampling_ratio/min": 0.7635613083839417, "sampling/sampling_logp_difference/max": 0.29587578773498535, "sampling/sampling_logp_difference/mean": 0.016666172072291374, "step": 359, "step_time": 13.630476420999912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 370.15625, "completions/mean_terminated_length": 370.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13159402727615088, "epoch": 0.0072, "frac_reward_zero_std": 0.0, "grad_norm": 0.4278484284877777, "kl": 0.4323490699753165, "learning_rate": 7.689780122452598e-06, "loss": 0.021, "num_tokens": 17052603.0, "reward": 0.16562500596046448, "reward_std": 0.28141912817955017, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.28916940093040466, "sampling/importance_sampling_ratio/max": 1.4965965747833252, "sampling/importance_sampling_ratio/mean": 1.0019240379333496, "sampling/importance_sampling_ratio/min": 0.43911221623420715, "sampling/sampling_logp_difference/max": 0.8290810585021973, "sampling/sampling_logp_difference/mean": 0.02083296701312065, "step": 360, "step_time": 14.7494099239957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 270.6875, "completions/mean_terminated_length": 270.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09651543654035777, "epoch": 0.00722, "frac_reward_zero_std": 0.0, "grad_norm": 0.5869027972221375, "kl": 1.4208320155739784, "learning_rate": 7.67719487509472e-06, "loss": 0.0094, "num_tokens": 17100785.0, "reward": 0.109375, "reward_std": 0.289956271648407, "rewards/rollout_reward_func/mean": 0.109375, "rewards/rollout_reward_func/std": 0.28211110830307007, "sampling/importance_sampling_ratio/max": 1.1568388938903809, "sampling/importance_sampling_ratio/mean": 0.9655015468597412, "sampling/importance_sampling_ratio/min": 0.5394048690795898, "sampling/sampling_logp_difference/max": 0.6228456497192383, "sampling/sampling_logp_difference/mean": 0.023033645004034042, "step": 361, "step_time": 13.26250687600259 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 368.46875, "completions/mean_terminated_length": 368.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10852962580975145, "epoch": 0.00724, "frac_reward_zero_std": 0.0, "grad_norm": 0.714699387550354, "kl": 0.7891108486801386, "learning_rate": 7.664590802071653e-06, "loss": 0.0556, "num_tokens": 17151957.0, "reward": 0.05000000074505806, "reward_std": 0.29797494411468506, "rewards/rollout_reward_func/mean": 0.05000000074505806, "rewards/rollout_reward_func/std": 0.3026762306690216, "sampling/importance_sampling_ratio/max": 2.350419044494629, "sampling/importance_sampling_ratio/mean": 0.975537121295929, "sampling/importance_sampling_ratio/min": 0.3290459215641022, "sampling/sampling_logp_difference/max": 0.845980167388916, "sampling/sampling_logp_difference/mean": 0.044246695935726166, "step": 362, "step_time": 14.29610509399572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 287.5625, "completions/mean_terminated_length": 287.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11149514222051948, "epoch": 0.00726, "frac_reward_zero_std": 0.0, "grad_norm": 0.5364643335342407, "kl": 0.9391241483390331, "learning_rate": 7.651968069639637e-06, "loss": 0.0044, "num_tokens": 17199696.0, "reward": 0.11562500894069672, "reward_std": 0.30536502599716187, "rewards/rollout_reward_func/mean": 0.11562500894069672, "rewards/rollout_reward_func/std": 0.3059721291065216, "sampling/importance_sampling_ratio/max": 1.0903398990631104, "sampling/importance_sampling_ratio/mean": 0.9437516927719116, "sampling/importance_sampling_ratio/min": 0.20900097489356995, "sampling/sampling_logp_difference/max": 1.5808799266815186, "sampling/sampling_logp_difference/mean": 0.0395551398396492, "step": 363, "step_time": 14.318083118003415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 329.25, "completions/mean_terminated_length": 329.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1169445556588471, "epoch": 0.00728, "frac_reward_zero_std": 0.0, "grad_norm": 0.5228849053382874, "kl": 1.3151628095656633, "learning_rate": 7.63932684430105e-06, "loss": 0.0318, "num_tokens": 17250301.0, "reward": 0.1875, "reward_std": 0.26810190081596375, "rewards/rollout_reward_func/mean": 0.1875, "rewards/rollout_reward_func/std": 0.2779533267021179, "sampling/importance_sampling_ratio/max": 2.2077434062957764, "sampling/importance_sampling_ratio/mean": 1.003431797027588, "sampling/importance_sampling_ratio/min": 0.2688505947589874, "sampling/sampling_logp_difference/max": 1.3202855587005615, "sampling/sampling_logp_difference/mean": 0.03911362588405609, "step": 364, "step_time": 13.769931537997763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 311.03125, "completions/mean_terminated_length": 311.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10441038524731994, "epoch": 0.0073, "frac_reward_zero_std": 0.0, "grad_norm": 0.46914565563201904, "kl": 0.5010304357856512, "learning_rate": 7.626667292802197e-06, "loss": 0.013, "num_tokens": 17300393.0, "reward": 0.04375000298023224, "reward_std": 0.2783769369125366, "rewards/rollout_reward_func/mean": 0.04375000298023224, "rewards/rollout_reward_func/std": 0.2793281078338623, "sampling/importance_sampling_ratio/max": 1.1726404428482056, "sampling/importance_sampling_ratio/mean": 0.9661462903022766, "sampling/importance_sampling_ratio/min": 0.6245051622390747, "sampling/sampling_logp_difference/max": 0.4823770523071289, "sampling/sampling_logp_difference/mean": 0.022345153614878654, "step": 365, "step_time": 14.250368742999854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 213.09375, "completions/mean_terminated_length": 213.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10612591239623725, "epoch": 0.00732, "frac_reward_zero_std": 0.0, "grad_norm": 0.18755598366260529, "kl": 0.5865590758621693, "learning_rate": 7.613989582131121e-06, "loss": -0.008, "num_tokens": 17346307.0, "reward": 0.19374999403953552, "reward_std": 0.18056723475456238, "rewards/rollout_reward_func/mean": 0.19374999403953552, "rewards/rollout_reward_func/std": 0.21393020451068878, "sampling/importance_sampling_ratio/max": 1.4324060678482056, "sampling/importance_sampling_ratio/mean": 0.9917494654655457, "sampling/importance_sampling_ratio/min": 0.6617348194122314, "sampling/sampling_logp_difference/max": 0.4128122329711914, "sampling/sampling_logp_difference/mean": 0.01652798056602478, "step": 366, "step_time": 14.701296433002426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 253.5625, "completions/mean_terminated_length": 253.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09833640931174159, "epoch": 0.00734, "frac_reward_zero_std": 0.0, "grad_norm": 0.3875458836555481, "kl": 0.5065928669646382, "learning_rate": 7.6012938795153966e-06, "loss": 0.0169, "num_tokens": 17393007.0, "reward": 0.11562500894069672, "reward_std": 0.27120262384414673, "rewards/rollout_reward_func/mean": 0.11562500894069672, "rewards/rollout_reward_func/std": 0.2829674184322357, "sampling/importance_sampling_ratio/max": 1.1884077787399292, "sampling/importance_sampling_ratio/mean": 0.9584481120109558, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1676108837127686, "sampling/sampling_logp_difference/mean": 0.027113694697618484, "step": 367, "step_time": 13.396241301008558 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 235.875, "completions/mean_terminated_length": 235.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07517069461755455, "epoch": 0.00736, "frac_reward_zero_std": 0.0, "grad_norm": 0.6619123220443726, "kl": 0.44842331390827894, "learning_rate": 7.588580352419923e-06, "loss": 0.0155, "num_tokens": 17438974.0, "reward": 0.13750000298023224, "reward_std": 0.25267016887664795, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.24854415655136108, "sampling/importance_sampling_ratio/max": 1.7657581567764282, "sampling/importance_sampling_ratio/mean": 1.0068531036376953, "sampling/importance_sampling_ratio/min": 0.49651405215263367, "sampling/sampling_logp_difference/max": 0.7096864581108093, "sampling/sampling_logp_difference/mean": 0.025692272931337357, "step": 368, "step_time": 14.468939615999261 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 184.0, "completions/mean_terminated_length": 184.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09941251133568585, "epoch": 0.00738, "frac_reward_zero_std": 0.0, "grad_norm": 0.33838391304016113, "kl": 2.451954941265285, "learning_rate": 7.575849168544717e-06, "loss": 0.0273, "num_tokens": 17483272.0, "reward": 0.171875, "reward_std": 0.22869853675365448, "rewards/rollout_reward_func/mean": 0.171875, "rewards/rollout_reward_func/std": 0.25429773330688477, "sampling/importance_sampling_ratio/max": 1.785421371459961, "sampling/importance_sampling_ratio/mean": 1.0122437477111816, "sampling/importance_sampling_ratio/min": 0.2549613118171692, "sampling/sampling_logp_difference/max": 1.3754477500915527, "sampling/sampling_logp_difference/mean": 0.03229237347841263, "step": 369, "step_time": 12.380387230994529 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 226.84375, "completions/mean_terminated_length": 226.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09128652117215097, "epoch": 0.0074, "frac_reward_zero_std": 0.0, "grad_norm": 0.4286957085132599, "kl": 0.5713847707957029, "learning_rate": 7.5631004958227e-06, "loss": -0.0115, "num_tokens": 17529132.0, "reward": 0.16250000894069672, "reward_std": 0.26760002970695496, "rewards/rollout_reward_func/mean": 0.16250000894069672, "rewards/rollout_reward_func/std": 0.2624327838420868, "sampling/importance_sampling_ratio/max": 1.031201720237732, "sampling/importance_sampling_ratio/mean": 0.9479809403419495, "sampling/importance_sampling_ratio/min": 0.3794257342815399, "sampling/sampling_logp_difference/max": 0.9795695543289185, "sampling/sampling_logp_difference/mean": 0.028119951486587524, "step": 370, "step_time": 13.749819085991476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 243.15625, "completions/mean_terminated_length": 243.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11612931994022802, "epoch": 0.00742, "frac_reward_zero_std": 0.0, "grad_norm": 0.5137542486190796, "kl": 0.5479308217763901, "learning_rate": 7.550334502417483e-06, "loss": 0.0142, "num_tokens": 17576297.0, "reward": 0.10625000298023224, "reward_std": 0.2344377338886261, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.22423706948757172, "sampling/importance_sampling_ratio/max": 1.2215169668197632, "sampling/importance_sampling_ratio/mean": 0.9972527027130127, "sampling/importance_sampling_ratio/min": 0.7208196520805359, "sampling/sampling_logp_difference/max": 0.3430686593055725, "sampling/sampling_logp_difference/mean": 0.013314547948539257, "step": 371, "step_time": 14.456323236991011 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 293.28125, "completions/mean_terminated_length": 293.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09071052737999707, "epoch": 0.00744, "frac_reward_zero_std": 0.0, "grad_norm": 0.320112407207489, "kl": 0.6775121493265033, "learning_rate": 7.537551356721149e-06, "loss": 0.0254, "num_tokens": 17625761.0, "reward": 0.13437500596046448, "reward_std": 0.24210934340953827, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.2659243047237396, "sampling/importance_sampling_ratio/max": 2.1823675632476807, "sampling/importance_sampling_ratio/mean": 1.013767957687378, "sampling/importance_sampling_ratio/min": 0.23116648197174072, "sampling/sampling_logp_difference/max": 1.4747945070266724, "sampling/sampling_logp_difference/mean": 0.02823324128985405, "step": 372, "step_time": 14.184909737992712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 251.03125, "completions/mean_terminated_length": 251.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0929576363414526, "epoch": 0.00746, "frac_reward_zero_std": 0.0, "grad_norm": 0.5035810470581055, "kl": 1.1048030871897936, "learning_rate": 7.5247512273520325e-06, "loss": 0.0226, "num_tokens": 17672839.0, "reward": 0.16249999403953552, "reward_std": 0.23348405957221985, "rewards/rollout_reward_func/mean": 0.16249999403953552, "rewards/rollout_reward_func/std": 0.26365911960601807, "sampling/importance_sampling_ratio/max": 1.413117527961731, "sampling/importance_sampling_ratio/mean": 0.9886705875396729, "sampling/importance_sampling_ratio/min": 0.27861422300338745, "sampling/sampling_logp_difference/max": 1.2779011726379395, "sampling/sampling_logp_difference/mean": 0.03340921550989151, "step": 373, "step_time": 13.55277306899734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 262.6875, "completions/mean_terminated_length": 262.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15349876252003014, "epoch": 0.00748, "frac_reward_zero_std": 0.0, "grad_norm": 0.4859195053577423, "kl": 0.48013456352055073, "learning_rate": 7.511934283152491e-06, "loss": 0.0188, "num_tokens": 17720532.0, "reward": 0.07187500596046448, "reward_std": 0.23794089257717133, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.27851492166519165, "sampling/importance_sampling_ratio/max": 1.9761769771575928, "sampling/importance_sampling_ratio/mean": 0.9625829458236694, "sampling/importance_sampling_ratio/min": 0.3341844081878662, "sampling/sampling_logp_difference/max": 1.09596848487854, "sampling/sampling_logp_difference/mean": 0.04982834681868553, "step": 374, "step_time": 13.53618699299841 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 158.34375, "completions/mean_terminated_length": 158.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07370924076531082, "epoch": 0.0075, "frac_reward_zero_std": 0.0, "grad_norm": 0.2999086081981659, "kl": 0.6670220475643873, "learning_rate": 7.499100693186684e-06, "loss": 0.0231, "num_tokens": 17761955.0, "reward": 0.1093750074505806, "reward_std": 0.29416757822036743, "rewards/rollout_reward_func/mean": 0.1093750074505806, "rewards/rollout_reward_func/std": 0.29111507534980774, "sampling/importance_sampling_ratio/max": 1.9321786165237427, "sampling/importance_sampling_ratio/mean": 1.0002299547195435, "sampling/importance_sampling_ratio/min": 0.34558433294296265, "sampling/sampling_logp_difference/max": 1.0724411010742188, "sampling/sampling_logp_difference/mean": 0.03153427317738533, "step": 375, "step_time": 12.326751252996473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 280.78125, "completions/mean_terminated_length": 280.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13259030901826918, "epoch": 0.00752, "frac_reward_zero_std": 0.0, "grad_norm": 0.49899327754974365, "kl": 0.41866188310086727, "learning_rate": 7.486250626738338e-06, "loss": 0.0001, "num_tokens": 17811057.0, "reward": 0.10312500596046448, "reward_std": 0.3139907717704773, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.32871633768081665, "sampling/importance_sampling_ratio/max": 1.182113766670227, "sampling/importance_sampling_ratio/mean": 0.9781641960144043, "sampling/importance_sampling_ratio/min": 0.6414632797241211, "sampling/sampling_logp_difference/max": 0.4439428448677063, "sampling/sampling_logp_difference/mean": 0.01746557094156742, "step": 376, "step_time": 13.798964780999086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1661.0, "completions/max_terminated_length": 1661.0, "completions/mean_length": 373.9375, "completions/mean_terminated_length": 373.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1501863608136773, "epoch": 0.00754, "frac_reward_zero_std": 0.0, "grad_norm": 0.8156258463859558, "kl": 0.7684100549668074, "learning_rate": 7.473384253308518e-06, "loss": -0.006, "num_tokens": 17862785.0, "reward": 0.15000000596046448, "reward_std": 0.28571605682373047, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.29946187138557434, "sampling/importance_sampling_ratio/max": 1.7271428108215332, "sampling/importance_sampling_ratio/mean": 0.9929789304733276, "sampling/importance_sampling_ratio/min": 0.5437683463096619, "sampling/sampling_logp_difference/max": 0.7254538536071777, "sampling/sampling_logp_difference/mean": 0.03198392316699028, "step": 377, "step_time": 16.6907017580088 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1859.0, "completions/max_terminated_length": 1859.0, "completions/mean_length": 377.875, "completions/mean_terminated_length": 377.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1331361811608076, "epoch": 0.00756, "frac_reward_zero_std": 0.0, "grad_norm": 0.5963050127029419, "kl": 0.6521645560860634, "learning_rate": 7.460501742613385e-06, "loss": 0.03, "num_tokens": 17914866.0, "reward": 0.08125000447034836, "reward_std": 0.3121488094329834, "rewards/rollout_reward_func/mean": 0.08125000447034836, "rewards/rollout_reward_func/std": 0.31153884530067444, "sampling/importance_sampling_ratio/max": 2.023238182067871, "sampling/importance_sampling_ratio/mean": 1.0326992273330688, "sampling/importance_sampling_ratio/min": 0.5313096642494202, "sampling/sampling_logp_difference/max": 0.6891207695007324, "sampling/sampling_logp_difference/mean": 0.03008503094315529, "step": 378, "step_time": 16.531622790000256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 295.40625, "completions/mean_terminated_length": 295.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11254425090737641, "epoch": 0.00758, "frac_reward_zero_std": 0.0, "grad_norm": 0.738817036151886, "kl": 0.7150264494121075, "learning_rate": 7.447603264581964e-06, "loss": 0.053, "num_tokens": 17964387.0, "reward": 0.11875000596046448, "reward_std": 0.23681005835533142, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.2415507733821869, "sampling/importance_sampling_ratio/max": 1.8587769269943237, "sampling/importance_sampling_ratio/mean": 1.083001971244812, "sampling/importance_sampling_ratio/min": 0.6122754812240601, "sampling/sampling_logp_difference/max": 0.6003620624542236, "sampling/sampling_logp_difference/mean": 0.03208732604980469, "step": 379, "step_time": 14.2362671209994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 322.53125, "completions/mean_terminated_length": 322.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13298122922424227, "epoch": 0.0076, "frac_reward_zero_std": 0.0, "grad_norm": 0.43373507261276245, "kl": 0.6784664038568735, "learning_rate": 7.4346889893539e-06, "loss": 0.037, "num_tokens": 18014745.0, "reward": 0.06562499701976776, "reward_std": 0.3021978735923767, "rewards/rollout_reward_func/mean": 0.06562499701976776, "rewards/rollout_reward_func/std": 0.34604620933532715, "sampling/importance_sampling_ratio/max": 1.893566370010376, "sampling/importance_sampling_ratio/mean": 1.0321263074874878, "sampling/importance_sampling_ratio/min": 0.5478686094284058, "sampling/sampling_logp_difference/max": 0.6278266906738281, "sampling/sampling_logp_difference/mean": 0.0342879444360733, "step": 380, "step_time": 15.216632846004359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 393.65625, "completions/mean_terminated_length": 393.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19200700579676777, "epoch": 0.00762, "frac_reward_zero_std": 0.0, "grad_norm": 0.6740888357162476, "kl": 0.9991927165538073, "learning_rate": 7.421759087277214e-06, "loss": 0.0371, "num_tokens": 18067528.0, "reward": 0.006250001490116119, "reward_std": 0.34901371598243713, "rewards/rollout_reward_func/mean": 0.006250001490116119, "rewards/rollout_reward_func/std": 0.36272937059402466, "sampling/importance_sampling_ratio/max": 2.0229413509368896, "sampling/importance_sampling_ratio/mean": 1.0134100914001465, "sampling/importance_sampling_ratio/min": 0.5109431743621826, "sampling/sampling_logp_difference/max": 0.6896032094955444, "sampling/sampling_logp_difference/mean": 0.04521987959742546, "step": 381, "step_time": 14.53694314599852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1279.0, "completions/max_terminated_length": 1279.0, "completions/mean_length": 312.375, "completions/mean_terminated_length": 312.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16983593814074993, "epoch": 0.00764, "frac_reward_zero_std": 0.0, "grad_norm": 0.437542587518692, "kl": 1.1556322183459997, "learning_rate": 7.408813728906053e-06, "loss": 0.0178, "num_tokens": 18118215.0, "reward": 0.04062499850988388, "reward_std": 0.3134784400463104, "rewards/rollout_reward_func/mean": 0.04062499850988388, "rewards/rollout_reward_func/std": 0.3046514093875885, "sampling/importance_sampling_ratio/max": 1.7166211605072021, "sampling/importance_sampling_ratio/mean": 0.9688358902931213, "sampling/importance_sampling_ratio/min": 0.18513019382953644, "sampling/sampling_logp_difference/max": 1.6966173648834229, "sampling/sampling_logp_difference/mean": 0.050606705248355865, "step": 382, "step_time": 14.510476395986188 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 260.6875, "completions/mean_terminated_length": 260.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1321803741157055, "epoch": 0.00766, "frac_reward_zero_std": 0.0, "grad_norm": 0.5272294878959656, "kl": 1.1764957942068577, "learning_rate": 7.395853084998448e-06, "loss": 0.0047, "num_tokens": 18166220.0, "reward": 0.09375, "reward_std": 0.277470201253891, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.2793281078338623, "sampling/importance_sampling_ratio/max": 1.084223985671997, "sampling/importance_sampling_ratio/mean": 0.9772982597351074, "sampling/importance_sampling_ratio/min": 0.30870577692985535, "sampling/sampling_logp_difference/max": 1.1873226165771484, "sampling/sampling_logp_difference/mean": 0.03337405249476433, "step": 383, "step_time": 14.384553773001244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 356.09375, "completions/mean_terminated_length": 356.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14878913550637662, "epoch": 0.00768, "frac_reward_zero_std": 0.0, "grad_norm": 0.30398157238960266, "kl": 0.6286350358277559, "learning_rate": 7.382877326514051e-06, "loss": 0.0264, "num_tokens": 18217707.0, "reward": 0.13125000894069672, "reward_std": 0.3136027455329895, "rewards/rollout_reward_func/mean": 0.13125000894069672, "rewards/rollout_reward_func/std": 0.3156534433364868, "sampling/importance_sampling_ratio/max": 1.5708404779434204, "sampling/importance_sampling_ratio/mean": 0.9861130714416504, "sampling/importance_sampling_ratio/min": 0.25286564230918884, "sampling/sampling_logp_difference/max": 1.3748478889465332, "sampling/sampling_logp_difference/mean": 0.030599601566791534, "step": 384, "step_time": 15.552861782001855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1571.0, "completions/max_terminated_length": 1571.0, "completions/mean_length": 276.0, "completions/mean_terminated_length": 276.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12935176363680512, "epoch": 0.0077, "frac_reward_zero_std": 0.0, "grad_norm": 0.47632381319999695, "kl": 0.5183544475585222, "learning_rate": 7.369886624611889e-06, "loss": 0.0134, "num_tokens": 18265796.0, "reward": 0.25312501192092896, "reward_std": 0.27846860885620117, "rewards/rollout_reward_func/mean": 0.25312501192092896, "rewards/rollout_reward_func/std": 0.27938222885131836, "sampling/importance_sampling_ratio/max": 1.2854385375976562, "sampling/importance_sampling_ratio/mean": 1.009049892425537, "sampling/importance_sampling_ratio/min": 0.7163835763931274, "sampling/sampling_logp_difference/max": 0.3427008390426636, "sampling/sampling_logp_difference/mean": 0.017918720841407776, "step": 385, "step_time": 15.736926034995122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 197.5, "completions/mean_terminated_length": 197.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14303193101659417, "epoch": 0.00772, "frac_reward_zero_std": 0.0, "grad_norm": 0.4453886151313782, "kl": 0.6439152490347624, "learning_rate": 7.356881150648103e-06, "loss": -0.0027, "num_tokens": 18310147.0, "reward": 0.125, "reward_std": 0.2769929766654968, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.3016085922718048, "sampling/importance_sampling_ratio/max": 1.328711986541748, "sampling/importance_sampling_ratio/mean": 0.9872516989707947, "sampling/importance_sampling_ratio/min": 0.5461665987968445, "sampling/sampling_logp_difference/max": 0.7366452217102051, "sampling/sampling_logp_difference/mean": 0.046146031469106674, "step": 386, "step_time": 14.097552009003266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 298.21875, "completions/mean_terminated_length": 298.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12271801137831062, "epoch": 0.00774, "frac_reward_zero_std": 0.0, "grad_norm": 0.6522309184074402, "kl": 0.8072170065715909, "learning_rate": 7.343861076173684e-06, "loss": 0.0247, "num_tokens": 18358843.0, "reward": 0.11562500894069672, "reward_std": 0.25444871187210083, "rewards/rollout_reward_func/mean": 0.11562500894069672, "rewards/rollout_reward_func/std": 0.3193838596343994, "sampling/importance_sampling_ratio/max": 1.3253720998764038, "sampling/importance_sampling_ratio/mean": 1.026924729347229, "sampling/importance_sampling_ratio/min": 0.5320571064949036, "sampling/sampling_logp_difference/max": 0.7306840419769287, "sampling/sampling_logp_difference/mean": 0.023614440113306046, "step": 387, "step_time": 15.15532105100283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 307.5, "completions/mean_terminated_length": 307.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18359232530929148, "epoch": 0.00776, "frac_reward_zero_std": 0.0, "grad_norm": 0.5262148976325989, "kl": 2.6110937846824527, "learning_rate": 7.330826572932217e-06, "loss": -0.0013, "num_tokens": 18409777.0, "reward": 0.04062499850988388, "reward_std": 0.3058238923549652, "rewards/rollout_reward_func/mean": 0.04062499850988388, "rewards/rollout_reward_func/std": 0.2960594594478607, "sampling/importance_sampling_ratio/max": 1.4947482347488403, "sampling/importance_sampling_ratio/mean": 0.964730441570282, "sampling/importance_sampling_ratio/min": 0.275413453578949, "sampling/sampling_logp_difference/max": 1.1447193622589111, "sampling/sampling_logp_difference/mean": 0.0471656434237957, "step": 388, "step_time": 14.40827811100462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 339.59375, "completions/mean_terminated_length": 339.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22981098829768598, "epoch": 0.00778, "frac_reward_zero_std": 0.0, "grad_norm": 1.0086994171142578, "kl": 1.0981447538360953, "learning_rate": 7.3177778128576124e-06, "loss": -0.0527, "num_tokens": 18461693.0, "reward": 0.20000000298023224, "reward_std": 0.3178083896636963, "rewards/rollout_reward_func/mean": 0.20000000298023224, "rewards/rollout_reward_func/std": 0.32128772139549255, "sampling/importance_sampling_ratio/max": 1.1385289430618286, "sampling/importance_sampling_ratio/mean": 0.9423784017562866, "sampling/importance_sampling_ratio/min": 0.465571790933609, "sampling/sampling_logp_difference/max": 1.0929222106933594, "sampling/sampling_logp_difference/mean": 0.046855803579092026, "step": 389, "step_time": 14.591438490988367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 236.125, "completions/mean_terminated_length": 236.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15177763812243938, "epoch": 0.0078, "frac_reward_zero_std": 0.0, "grad_norm": 0.3402908146381378, "kl": 3.3142778631299734, "learning_rate": 7.304714968071834e-06, "loss": 0.0001, "num_tokens": 18507884.0, "reward": 0.12187501043081284, "reward_std": 0.2727051377296448, "rewards/rollout_reward_func/mean": 0.12187501043081284, "rewards/rollout_reward_func/std": 0.28253960609436035, "sampling/importance_sampling_ratio/max": 1.4480620622634888, "sampling/importance_sampling_ratio/mean": 0.9816348552703857, "sampling/importance_sampling_ratio/min": 0.544724702835083, "sampling/sampling_logp_difference/max": 0.588902473449707, "sampling/sampling_logp_difference/mean": 0.03617984056472778, "step": 390, "step_time": 14.619039413992141 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 339.25, "completions/mean_terminated_length": 339.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16975308349356055, "epoch": 0.00782, "frac_reward_zero_std": 0.0, "grad_norm": 0.7774699926376343, "kl": 0.7502375394105911, "learning_rate": 7.291638210882638e-06, "loss": 0.0388, "num_tokens": 18558306.0, "reward": 0.0781250074505806, "reward_std": 0.3274831771850586, "rewards/rollout_reward_func/mean": 0.0781250074505806, "rewards/rollout_reward_func/std": 0.35626593232154846, "sampling/importance_sampling_ratio/max": 1.8446351289749146, "sampling/importance_sampling_ratio/mean": 0.9685207605361938, "sampling/importance_sampling_ratio/min": 0.5069882273674011, "sampling/sampling_logp_difference/max": 0.7009825706481934, "sampling/sampling_logp_difference/mean": 0.04682367295026779, "step": 391, "step_time": 15.584837965998304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 244.125, "completions/mean_terminated_length": 244.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14497264428064227, "epoch": 0.00784, "frac_reward_zero_std": 0.0, "grad_norm": 0.49724528193473816, "kl": 0.5891287578269839, "learning_rate": 7.278547713781288e-06, "loss": 0.001, "num_tokens": 18605482.0, "reward": 0.16249999403953552, "reward_std": 0.2822359502315521, "rewards/rollout_reward_func/mean": 0.16249999403953552, "rewards/rollout_reward_func/std": 0.27911147475242615, "sampling/importance_sampling_ratio/max": 1.1088054180145264, "sampling/importance_sampling_ratio/mean": 0.9842108488082886, "sampling/importance_sampling_ratio/min": 0.6295710206031799, "sampling/sampling_logp_difference/max": 0.54526686668396, "sampling/sampling_logp_difference/mean": 0.022502433508634567, "step": 392, "step_time": 13.613746321003418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 210.28125, "completions/mean_terminated_length": 210.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11393500119447708, "epoch": 0.00786, "frac_reward_zero_std": 0.0, "grad_norm": 0.5300108194351196, "kl": 1.0779993738979101, "learning_rate": 7.2654436494402955e-06, "loss": 0.0244, "num_tokens": 18650486.0, "reward": 0.0781250074505806, "reward_std": 0.24297727644443512, "rewards/rollout_reward_func/mean": 0.0781250074505806, "rewards/rollout_reward_func/std": 0.25238779187202454, "sampling/importance_sampling_ratio/max": 1.2181355953216553, "sampling/importance_sampling_ratio/mean": 0.9484555721282959, "sampling/importance_sampling_ratio/min": 0.424344003200531, "sampling/sampling_logp_difference/max": 0.8654251098632812, "sampling/sampling_logp_difference/mean": 0.03744788095355034, "step": 393, "step_time": 12.32575660299699 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 294.59375, "completions/mean_terminated_length": 294.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13736594805959612, "epoch": 0.00788, "frac_reward_zero_std": 0.0, "grad_norm": 0.6204705834388733, "kl": 0.48456821590662, "learning_rate": 7.252326190711121e-06, "loss": 0.0234, "num_tokens": 18698098.0, "reward": 0.203125, "reward_std": 0.17880073189735413, "rewards/rollout_reward_func/mean": 0.203125, "rewards/rollout_reward_func/std": 0.19424688816070557, "sampling/importance_sampling_ratio/max": 1.1250211000442505, "sampling/importance_sampling_ratio/mean": 0.9788303375244141, "sampling/importance_sampling_ratio/min": 0.4655840992927551, "sampling/sampling_logp_difference/max": 0.7719060182571411, "sampling/sampling_logp_difference/mean": 0.02158144675195217, "step": 394, "step_time": 16.637480254001275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 351.1875, "completions/mean_terminated_length": 351.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13233594014309347, "epoch": 0.0079, "frac_reward_zero_std": 0.0, "grad_norm": 0.7134262919425964, "kl": 0.4545518374070525, "learning_rate": 7.239195510621918e-06, "loss": 0.0263, "num_tokens": 18749067.0, "reward": 0.10312500596046448, "reward_std": 0.2915126383304596, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.29346030950546265, "sampling/importance_sampling_ratio/max": 1.0935494899749756, "sampling/importance_sampling_ratio/mean": 0.9817126989364624, "sampling/importance_sampling_ratio/min": 0.638982355594635, "sampling/sampling_logp_difference/max": 0.4663839340209961, "sampling/sampling_logp_difference/mean": 0.019303416833281517, "step": 395, "step_time": 13.917065496993018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 286.96875, "completions/mean_terminated_length": 289.4838562011719, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12756893574260175, "epoch": 0.00792, "frac_reward_zero_std": 0.0, "grad_norm": 0.8050448894500732, "kl": 0.7589465910568833, "learning_rate": 7.22605178237523e-06, "loss": 0.0291, "num_tokens": 18797765.0, "reward": 0.13437500596046448, "reward_std": 0.2784811556339264, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.30754950642585754, "sampling/importance_sampling_ratio/max": 1.7753589153289795, "sampling/importance_sampling_ratio/mean": 0.9981502294540405, "sampling/importance_sampling_ratio/min": 0.4005560874938965, "sampling/sampling_logp_difference/max": 0.9345738887786865, "sampling/sampling_logp_difference/mean": 0.032888978719711304, "step": 396, "step_time": 12.720439157001238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1624.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 353.65625, "completions/mean_terminated_length": 312.6773986816406, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16929736523889005, "epoch": 0.00794, "frac_reward_zero_std": 0.0, "grad_norm": 2.3770945072174072, "kl": 0.5452912757173181, "learning_rate": 7.212895179345718e-06, "loss": 0.3707, "num_tokens": 18848381.0, "reward": 0.05937499925494194, "reward_std": 0.34974026679992676, "rewards/rollout_reward_func/mean": 0.05937499925494194, "rewards/rollout_reward_func/std": 0.3490626811981201, "sampling/importance_sampling_ratio/max": 1.8632622957229614, "sampling/importance_sampling_ratio/mean": 1.055851697921753, "sampling/importance_sampling_ratio/min": 0.7124574184417725, "sampling/sampling_logp_difference/max": 0.6252243518829346, "sampling/sampling_logp_difference/mean": 0.030456505715847015, "step": 397, "step_time": 16.844937311998365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 356.8125, "completions/mean_terminated_length": 356.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19735085451975465, "epoch": 0.00796, "frac_reward_zero_std": 0.0, "grad_norm": 0.6873713731765747, "kl": 1.66576511785388, "learning_rate": 7.199725875077871e-06, "loss": -0.0192, "num_tokens": 18900310.0, "reward": 0.05625000223517418, "reward_std": 0.2718425393104553, "rewards/rollout_reward_func/mean": 0.05625000223517418, "rewards/rollout_reward_func/std": 0.2601953446865082, "sampling/importance_sampling_ratio/max": 1.2429790496826172, "sampling/importance_sampling_ratio/mean": 0.8534765243530273, "sampling/importance_sampling_ratio/min": 0.2262081652879715, "sampling/sampling_logp_difference/max": 1.4931964874267578, "sampling/sampling_logp_difference/mean": 0.0670192539691925, "step": 398, "step_time": 13.95730953300881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 327.0, "completions/mean_terminated_length": 327.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18373733572661877, "epoch": 0.00798, "frac_reward_zero_std": 0.0, "grad_norm": 0.6549598574638367, "kl": 1.066187015734613, "learning_rate": 7.186544043283715e-06, "loss": 0.0107, "num_tokens": 18949874.0, "reward": 0.08437500149011612, "reward_std": 0.2964494228363037, "rewards/rollout_reward_func/mean": 0.08437500149011612, "rewards/rollout_reward_func/std": 0.32239964604377747, "sampling/importance_sampling_ratio/max": 1.6425048112869263, "sampling/importance_sampling_ratio/mean": 0.9919752478599548, "sampling/importance_sampling_ratio/min": 0.5404369235038757, "sampling/sampling_logp_difference/max": 0.7691411972045898, "sampling/sampling_logp_difference/mean": 0.03566209226846695, "step": 399, "step_time": 15.60421826799211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 264.1875, "completions/mean_terminated_length": 264.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14305081195198, "epoch": 0.008, "frac_reward_zero_std": 0.0, "grad_norm": 0.8413283824920654, "kl": 0.6161659108474851, "learning_rate": 7.173349857840521e-06, "loss": 0.0516, "num_tokens": 18997995.0, "reward": 0.09687500447034836, "reward_std": 0.2853497266769409, "rewards/rollout_reward_func/mean": 0.09687500447034836, "rewards/rollout_reward_func/std": 0.2741372287273407, "sampling/importance_sampling_ratio/max": 2.030982255935669, "sampling/importance_sampling_ratio/mean": 0.9792234897613525, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.097689151763916, "sampling/sampling_logp_difference/mean": 0.05201077461242676, "step": 400, "step_time": 15.992881406997185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1706.0, "completions/max_terminated_length": 1706.0, "completions/mean_length": 299.03125, "completions/mean_terminated_length": 299.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14827432250604033, "epoch": 0.00802, "frac_reward_zero_std": 0.0, "grad_norm": 0.4282774329185486, "kl": 0.7191561087965965, "learning_rate": 7.160143492788516e-06, "loss": 0.0059, "num_tokens": 19047541.0, "reward": 0.11874999850988388, "reward_std": 0.2286359965801239, "rewards/rollout_reward_func/mean": 0.11874999850988388, "rewards/rollout_reward_func/std": 0.25707724690437317, "sampling/importance_sampling_ratio/max": 1.2459561824798584, "sampling/importance_sampling_ratio/mean": 0.928447425365448, "sampling/importance_sampling_ratio/min": 0.3235457241535187, "sampling/sampling_logp_difference/max": 1.1285066604614258, "sampling/sampling_logp_difference/mean": 0.047848694026470184, "step": 401, "step_time": 15.997779942994384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 263.78125, "completions/mean_terminated_length": 263.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11097659333609045, "epoch": 0.00804, "frac_reward_zero_std": 0.0, "grad_norm": 0.40753597021102905, "kl": 0.6414584908634424, "learning_rate": 7.146925122328581e-06, "loss": 0.0381, "num_tokens": 19094560.0, "reward": 0.15937501192092896, "reward_std": 0.18226271867752075, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.20455990731716156, "sampling/importance_sampling_ratio/max": 1.0969654321670532, "sampling/importance_sampling_ratio/mean": 0.9810306429862976, "sampling/importance_sampling_ratio/min": 0.27251189947128296, "sampling/sampling_logp_difference/max": 1.3001508712768555, "sampling/sampling_logp_difference/mean": 0.021769173443317413, "step": 402, "step_time": 15.427999314011686 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 364.78125, "completions/mean_terminated_length": 364.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14474882534705102, "epoch": 0.00806, "frac_reward_zero_std": 0.0, "grad_norm": 0.640478253364563, "kl": 1.7535886261612177, "learning_rate": 7.133694920819958e-06, "loss": -0.0031, "num_tokens": 19146428.0, "reward": 0.19062501192092896, "reward_std": 0.26402896642684937, "rewards/rollout_reward_func/mean": 0.19062501192092896, "rewards/rollout_reward_func/std": 0.299848735332489, "sampling/importance_sampling_ratio/max": 1.1727609634399414, "sampling/importance_sampling_ratio/mean": 0.9558688998222351, "sampling/importance_sampling_ratio/min": 0.47249501943588257, "sampling/sampling_logp_difference/max": 0.8597159385681152, "sampling/sampling_logp_difference/mean": 0.030206676572561264, "step": 403, "step_time": 15.689371256990853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1279.0, "completions/max_terminated_length": 1279.0, "completions/mean_length": 334.59375, "completions/mean_terminated_length": 334.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11071347212418914, "epoch": 0.00808, "frac_reward_zero_std": 0.0, "grad_norm": 0.35206758975982666, "kl": 3.4519122997298837, "learning_rate": 7.12045306277795e-06, "loss": 0.0197, "num_tokens": 19197024.0, "reward": 0.109375, "reward_std": 0.22760793566703796, "rewards/rollout_reward_func/mean": 0.109375, "rewards/rollout_reward_func/std": 0.25318536162376404, "sampling/importance_sampling_ratio/max": 1.042145848274231, "sampling/importance_sampling_ratio/mean": 0.9514590501785278, "sampling/importance_sampling_ratio/min": 0.36785468459129333, "sampling/sampling_logp_difference/max": 0.8698587417602539, "sampling/sampling_logp_difference/mean": 0.028724996373057365, "step": 404, "step_time": 14.482286661997932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 245.65625, "completions/mean_terminated_length": 245.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08805217337794602, "epoch": 0.0081, "frac_reward_zero_std": 0.0, "grad_norm": 0.40022775530815125, "kl": 1.6165465554222465, "learning_rate": 7.107199722871614e-06, "loss": 0.0045, "num_tokens": 19243944.0, "reward": 0.1718750149011612, "reward_std": 0.22474589943885803, "rewards/rollout_reward_func/mean": 0.1718750149011612, "rewards/rollout_reward_func/std": 0.2593221664428711, "sampling/importance_sampling_ratio/max": 1.1969735622406006, "sampling/importance_sampling_ratio/mean": 0.9777218699455261, "sampling/importance_sampling_ratio/min": 0.5422399640083313, "sampling/sampling_logp_difference/max": 0.6191368103027344, "sampling/sampling_logp_difference/mean": 0.02099968120455742, "step": 405, "step_time": 14.089379328001087 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 377.3125, "completions/mean_terminated_length": 377.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1537801343947649, "epoch": 0.00812, "frac_reward_zero_std": 0.0, "grad_norm": 1.1925426721572876, "kl": 1.0030884621664882, "learning_rate": 7.093935075921465e-06, "loss": 0.0223, "num_tokens": 19296182.0, "reward": 0.13124999403953552, "reward_std": 0.2624821662902832, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.26449891924858093, "sampling/importance_sampling_ratio/max": 1.7060832977294922, "sampling/importance_sampling_ratio/mean": 1.0222843885421753, "sampling/importance_sampling_ratio/min": 0.41060295701026917, "sampling/sampling_logp_difference/max": 0.8901290893554688, "sampling/sampling_logp_difference/mean": 0.028036678209900856, "step": 406, "step_time": 14.982801234000362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 346.125, "completions/mean_terminated_length": 346.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15429664740804583, "epoch": 0.00814, "frac_reward_zero_std": 0.0, "grad_norm": 0.6841412782669067, "kl": 4.043543580919504, "learning_rate": 7.0806592968971624e-06, "loss": 0.0036, "num_tokens": 19346963.0, "reward": 0.05312500149011612, "reward_std": 0.327179878950119, "rewards/rollout_reward_func/mean": 0.05312500149011612, "rewards/rollout_reward_func/std": 0.3232739567756653, "sampling/importance_sampling_ratio/max": 1.125723958015442, "sampling/importance_sampling_ratio/mean": 0.8866658210754395, "sampling/importance_sampling_ratio/min": 0.331482470035553, "sampling/sampling_logp_difference/max": 1.1110063791275024, "sampling/sampling_logp_difference/mean": 0.058651555329561234, "step": 407, "step_time": 13.792359065995697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 322.40625, "completions/mean_terminated_length": 322.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10329376417212188, "epoch": 0.00816, "frac_reward_zero_std": 0.0, "grad_norm": 0.3678850829601288, "kl": 1.0000593746080995, "learning_rate": 7.067372560915205e-06, "loss": -0.0029, "num_tokens": 19395786.0, "reward": 0.140625, "reward_std": 0.30515003204345703, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.32513952255249023, "sampling/importance_sampling_ratio/max": 1.1284064054489136, "sampling/importance_sampling_ratio/mean": 0.9777091145515442, "sampling/importance_sampling_ratio/min": 0.5085418224334717, "sampling/sampling_logp_difference/max": 0.7398715019226074, "sampling/sampling_logp_difference/mean": 0.020889082923531532, "step": 408, "step_time": 14.311509453007602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 345.4375, "completions/mean_terminated_length": 345.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13301548757590353, "epoch": 0.00818, "frac_reward_zero_std": 0.0, "grad_norm": 0.5866270661354065, "kl": 0.8022051025182009, "learning_rate": 7.054075043236623e-06, "loss": 0.023, "num_tokens": 19446495.0, "reward": 0.10000000149011612, "reward_std": 0.3167133331298828, "rewards/rollout_reward_func/mean": 0.10000000149011612, "rewards/rollout_reward_func/std": 0.31004682183265686, "sampling/importance_sampling_ratio/max": 1.6211740970611572, "sampling/importance_sampling_ratio/mean": 1.013321042060852, "sampling/importance_sampling_ratio/min": 0.27548709511756897, "sampling/sampling_logp_difference/max": 1.2953026294708252, "sampling/sampling_logp_difference/mean": 0.032840367406606674, "step": 409, "step_time": 14.055405272989447 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 202.125, "completions/mean_terminated_length": 202.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10016851988621056, "epoch": 0.0082, "frac_reward_zero_std": 0.0, "grad_norm": 1.0338375568389893, "kl": 14.038616200909019, "learning_rate": 7.040766919264664e-06, "loss": 0.0001, "num_tokens": 19492464.0, "reward": 0.13750000298023224, "reward_std": 0.25083404779434204, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.259962797164917, "sampling/importance_sampling_ratio/max": 1.0505954027175903, "sampling/importance_sampling_ratio/mean": 0.9186977744102478, "sampling/importance_sampling_ratio/min": 0.4188453257083893, "sampling/sampling_logp_difference/max": 0.8703169822692871, "sampling/sampling_logp_difference/mean": 0.0411669984459877, "step": 410, "step_time": 13.506177545004903 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 347.53125, "completions/mean_terminated_length": 347.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12328414164949208, "epoch": 0.00822, "frac_reward_zero_std": 0.0, "grad_norm": 0.7381348609924316, "kl": 0.7071538716554642, "learning_rate": 7.027448364542479e-06, "loss": 0.0682, "num_tokens": 19543312.0, "reward": 0.07500000298023224, "reward_std": 0.23889513313770294, "rewards/rollout_reward_func/mean": 0.07500000298023224, "rewards/rollout_reward_func/std": 0.2711980640888214, "sampling/importance_sampling_ratio/max": 2.762969732284546, "sampling/importance_sampling_ratio/mean": 1.102088212966919, "sampling/importance_sampling_ratio/min": 0.46683767437934875, "sampling/sampling_logp_difference/max": 1.0088982582092285, "sampling/sampling_logp_difference/mean": 0.031747862696647644, "step": 411, "step_time": 14.839296371992532 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 313.21875, "completions/mean_terminated_length": 313.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13559229765087366, "epoch": 0.00824, "frac_reward_zero_std": 0.0, "grad_norm": 0.7994020581245422, "kl": 0.7602308066561818, "learning_rate": 7.01411955475081e-06, "loss": -0.0324, "num_tokens": 19593588.0, "reward": 0.15312500298023224, "reward_std": 0.32439061999320984, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.3182455599308014, "sampling/importance_sampling_ratio/max": 1.7935521602630615, "sampling/importance_sampling_ratio/mean": 1.0237417221069336, "sampling/importance_sampling_ratio/min": 0.35778751969337463, "sampling/sampling_logp_difference/max": 0.9666063785552979, "sampling/sampling_logp_difference/mean": 0.026221146807074547, "step": 412, "step_time": 14.387878210993222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 249.625, "completions/mean_terminated_length": 249.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10502685757819563, "epoch": 0.00826, "frac_reward_zero_std": 0.0, "grad_norm": 0.9654234647750854, "kl": 0.5174788311123848, "learning_rate": 7.000780665705665e-06, "loss": 0.0182, "num_tokens": 19640036.0, "reward": 0.15625, "reward_std": 0.28471311926841736, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.274669885635376, "sampling/importance_sampling_ratio/max": 1.3893886804580688, "sampling/importance_sampling_ratio/mean": 1.0172815322875977, "sampling/importance_sampling_ratio/min": 0.6051846742630005, "sampling/sampling_logp_difference/max": 0.3996235132217407, "sampling/sampling_logp_difference/mean": 0.016607580706477165, "step": 413, "step_time": 13.704181185996276 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009548611124046147, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 367.53125, "completions/mean_terminated_length": 367.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17163535091094673, "epoch": 0.00828, "frac_reward_zero_std": 0.0, "grad_norm": 1.2819037437438965, "kl": 0.8563981968909502, "learning_rate": 6.987431873356011e-06, "loss": -0.0542, "num_tokens": 19691222.0, "reward": 0.125, "reward_std": 0.2897608280181885, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.29402655363082886, "sampling/importance_sampling_ratio/max": 1.524096965789795, "sampling/importance_sampling_ratio/mean": 1.0006206035614014, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4932947158813477, "sampling/sampling_logp_difference/mean": 0.059883855283260345, "step": 414, "step_time": 15.301795853010844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1346.0, "completions/max_terminated_length": 1346.0, "completions/mean_length": 286.84375, "completions/mean_terminated_length": 286.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12269514752551913, "epoch": 0.0083, "frac_reward_zero_std": 0.0, "grad_norm": 0.5434938669204712, "kl": 0.5470169568434358, "learning_rate": 6.97407335378144e-06, "loss": 0.0127, "num_tokens": 19739625.0, "reward": 0.17812499403953552, "reward_std": 0.2674747109413147, "rewards/rollout_reward_func/mean": 0.17812499403953552, "rewards/rollout_reward_func/std": 0.2696884274482727, "sampling/importance_sampling_ratio/max": 1.329732060432434, "sampling/importance_sampling_ratio/mean": 1.0201640129089355, "sampling/importance_sampling_ratio/min": 0.6959369778633118, "sampling/sampling_logp_difference/max": 0.36252617835998535, "sampling/sampling_logp_difference/mean": 0.01931045576930046, "step": 415, "step_time": 14.609455374997196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1256.0, "completions/max_terminated_length": 1256.0, "completions/mean_length": 346.375, "completions/mean_terminated_length": 346.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08687153237406164, "epoch": 0.00832, "frac_reward_zero_std": 0.0, "grad_norm": 0.22295334935188293, "kl": 4.4898955803364515, "learning_rate": 6.960705283189857e-06, "loss": -0.0124, "num_tokens": 19790960.0, "reward": 0.16562500596046448, "reward_std": 0.20551438629627228, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.22662940621376038, "sampling/importance_sampling_ratio/max": 1.4189013242721558, "sampling/importance_sampling_ratio/mean": 0.9999182224273682, "sampling/importance_sampling_ratio/min": 0.6626611948013306, "sampling/sampling_logp_difference/max": 0.49414825439453125, "sampling/sampling_logp_difference/mean": 0.019964750856161118, "step": 416, "step_time": 14.449697591000586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 289.90625, "completions/mean_terminated_length": 289.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11180410906672478, "epoch": 0.00834, "frac_reward_zero_std": 0.0, "grad_norm": 1.3945441246032715, "kl": 12.069236384704709, "learning_rate": 6.94732783791515e-06, "loss": 0.0021, "num_tokens": 19840336.0, "reward": 0.12812501192092896, "reward_std": 0.2714652121067047, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.2819681763648987, "sampling/importance_sampling_ratio/max": 1.2487975358963013, "sampling/importance_sampling_ratio/mean": 0.9520187377929688, "sampling/importance_sampling_ratio/min": 0.2246093899011612, "sampling/sampling_logp_difference/max": 1.4933090209960938, "sampling/sampling_logp_difference/mean": 0.04957908019423485, "step": 417, "step_time": 14.075093522002135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 235.125, "completions/mean_terminated_length": 235.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1130560606252402, "epoch": 0.00836, "frac_reward_zero_std": 0.0, "grad_norm": 0.4593493938446045, "kl": 0.505438469350338, "learning_rate": 6.933941194414866e-06, "loss": 0.0045, "num_tokens": 19887081.0, "reward": 0.16250000894069672, "reward_std": 0.23076705634593964, "rewards/rollout_reward_func/mean": 0.16250000894069672, "rewards/rollout_reward_func/std": 0.23520752787590027, "sampling/importance_sampling_ratio/max": 1.2769275903701782, "sampling/importance_sampling_ratio/mean": 1.0174736976623535, "sampling/importance_sampling_ratio/min": 0.8936075568199158, "sampling/sampling_logp_difference/max": 0.21135014295578003, "sampling/sampling_logp_difference/mean": 0.009146693162620068, "step": 418, "step_time": 13.494175825999264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 251.71875, "completions/mean_terminated_length": 251.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08047635736875236, "epoch": 0.00838, "frac_reward_zero_std": 0.0, "grad_norm": 0.16560080647468567, "kl": 0.7471909206360579, "learning_rate": 6.920545529267882e-06, "loss": -0.0082, "num_tokens": 19933953.0, "reward": 0.2593750059604645, "reward_std": 0.20916879177093506, "rewards/rollout_reward_func/mean": 0.2593750059604645, "rewards/rollout_reward_func/std": 0.20613083243370056, "sampling/importance_sampling_ratio/max": 1.1084511280059814, "sampling/importance_sampling_ratio/mean": 1.0014915466308594, "sampling/importance_sampling_ratio/min": 0.4811517298221588, "sampling/sampling_logp_difference/max": 0.7371110916137695, "sampling/sampling_logp_difference/mean": 0.013211017474532127, "step": 419, "step_time": 13.503268573000241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1279.0, "completions/max_terminated_length": 1279.0, "completions/mean_length": 341.21875, "completions/mean_terminated_length": 341.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1366253310116008, "epoch": 0.0084, "frac_reward_zero_std": 0.0, "grad_norm": 0.40150827169418335, "kl": 0.4799592662602663, "learning_rate": 6.907141019172076e-06, "loss": -0.0012, "num_tokens": 19985689.0, "reward": 0.08125000447034836, "reward_std": 0.3179197311401367, "rewards/rollout_reward_func/mean": 0.08125000447034836, "rewards/rollout_reward_func/std": 0.333541601896286, "sampling/importance_sampling_ratio/max": 1.3258826732635498, "sampling/importance_sampling_ratio/mean": 1.0094228982925415, "sampling/importance_sampling_ratio/min": 0.5923804044723511, "sampling/sampling_logp_difference/max": 0.5236616134643555, "sampling/sampling_logp_difference/mean": 0.015085127204656601, "step": 420, "step_time": 14.850978655009385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 418.65625, "completions/mean_terminated_length": 418.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17021672462578863, "epoch": 0.00842, "frac_reward_zero_std": 0.0, "grad_norm": 0.5400846004486084, "kl": 0.4778284151107073, "learning_rate": 6.893727840941997e-06, "loss": 0.0135, "num_tokens": 20038735.0, "reward": 0.13437500596046448, "reward_std": 0.3212149143218994, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.3137796223163605, "sampling/importance_sampling_ratio/max": 1.602730631828308, "sampling/importance_sampling_ratio/mean": 1.0097603797912598, "sampling/importance_sampling_ratio/min": 0.5248827338218689, "sampling/sampling_logp_difference/max": 0.6505059003829956, "sampling/sampling_logp_difference/mean": 0.03200382739305496, "step": 421, "step_time": 14.528829053011577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 325.0625, "completions/mean_terminated_length": 325.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1527189896441996, "epoch": 0.00844, "frac_reward_zero_std": 0.0, "grad_norm": 0.9930731654167175, "kl": 5.588549569249153, "learning_rate": 6.880306171506535e-06, "loss": 0.0052, "num_tokens": 20088565.0, "reward": 0.09062500298023224, "reward_std": 0.29032403230667114, "rewards/rollout_reward_func/mean": 0.09062500298023224, "rewards/rollout_reward_func/std": 0.28889039158821106, "sampling/importance_sampling_ratio/max": 1.143249273300171, "sampling/importance_sampling_ratio/mean": 0.9256747961044312, "sampling/importance_sampling_ratio/min": 0.2028808742761612, "sampling/sampling_logp_difference/max": 1.6058518886566162, "sampling/sampling_logp_difference/mean": 0.05056462436914444, "step": 422, "step_time": 14.671720288995857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0015625000232830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015625000232830644, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 282.5625, "completions/mean_terminated_length": 282.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11718123243190348, "epoch": 0.00846, "frac_reward_zero_std": 0.0, "grad_norm": 0.2411380112171173, "kl": 2.069013139232993, "learning_rate": 6.866876187906582e-06, "loss": -0.0202, "num_tokens": 20135712.0, "reward": 0.18125000596046448, "reward_std": 0.27896445989608765, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.2740820646286011, "sampling/importance_sampling_ratio/max": 1.1962265968322754, "sampling/importance_sampling_ratio/mean": 0.9331724643707275, "sampling/importance_sampling_ratio/min": 6.134850139005721e-08, "sampling/sampling_logp_difference/max": 11.773531913757324, "sampling/sampling_logp_difference/mean": 0.16086164116859436, "step": 423, "step_time": 14.26622227000189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 288.53125, "completions/mean_terminated_length": 288.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13521528081037104, "epoch": 0.00848, "frac_reward_zero_std": 0.0, "grad_norm": 0.40748679637908936, "kl": 0.5258482899516821, "learning_rate": 6.8534380672927e-06, "loss": 0.0066, "num_tokens": 20183680.0, "reward": 0.11562500149011612, "reward_std": 0.2485811412334442, "rewards/rollout_reward_func/mean": 0.11562500149011612, "rewards/rollout_reward_func/std": 0.27487167716026306, "sampling/importance_sampling_ratio/max": 1.1514688730239868, "sampling/importance_sampling_ratio/mean": 1.0067434310913086, "sampling/importance_sampling_ratio/min": 0.8072936534881592, "sampling/sampling_logp_difference/max": 0.25671863555908203, "sampling/sampling_logp_difference/mean": 0.01581449620425701, "step": 424, "step_time": 13.697371198988549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 251.125, "completions/mean_terminated_length": 251.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12037546024657786, "epoch": 0.0085, "frac_reward_zero_std": 0.0, "grad_norm": 0.368671178817749, "kl": 0.573884753510356, "learning_rate": 6.839991986922785e-06, "loss": -0.028, "num_tokens": 20230412.0, "reward": 0.25, "reward_std": 0.2163912057876587, "rewards/rollout_reward_func/mean": 0.25, "rewards/rollout_reward_func/std": 0.24362848699092865, "sampling/importance_sampling_ratio/max": 1.273010492324829, "sampling/importance_sampling_ratio/mean": 1.025741457939148, "sampling/importance_sampling_ratio/min": 0.8016863465309143, "sampling/sampling_logp_difference/max": 0.22845029830932617, "sampling/sampling_logp_difference/mean": 0.011175092309713364, "step": 425, "step_time": 13.787661604001187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 304.09375, "completions/mean_terminated_length": 304.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1464216373860836, "epoch": 0.00852, "frac_reward_zero_std": 0.0, "grad_norm": 0.3733735680580139, "kl": 1.0764448633417487, "learning_rate": 6.826538124159727e-06, "loss": 0.0121, "num_tokens": 20279657.0, "reward": 0.10000000894069672, "reward_std": 0.31551554799079895, "rewards/rollout_reward_func/mean": 0.10000000894069672, "rewards/rollout_reward_func/std": 0.3110854923725128, "sampling/importance_sampling_ratio/max": 1.1425822973251343, "sampling/importance_sampling_ratio/mean": 1.0070700645446777, "sampling/importance_sampling_ratio/min": 0.7348750829696655, "sampling/sampling_logp_difference/max": 0.31998229026794434, "sampling/sampling_logp_difference/mean": 0.016386989504098892, "step": 426, "step_time": 14.105470290010999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 309.28125, "completions/mean_terminated_length": 309.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15733708534389734, "epoch": 0.00854, "frac_reward_zero_std": 0.0, "grad_norm": 0.758491575717926, "kl": 2.877655335702002, "learning_rate": 6.813076656469068e-06, "loss": -0.025, "num_tokens": 20329222.0, "reward": 0.10312500596046448, "reward_std": 0.2890503406524658, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.2956506013870239, "sampling/importance_sampling_ratio/max": 2.2842013835906982, "sampling/importance_sampling_ratio/mean": 1.0326950550079346, "sampling/importance_sampling_ratio/min": 0.4976659417152405, "sampling/sampling_logp_difference/max": 0.8022280931472778, "sampling/sampling_logp_difference/mean": 0.03512818366289139, "step": 427, "step_time": 15.055517036995298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1625.0, "completions/max_terminated_length": 1625.0, "completions/mean_length": 283.03125, "completions/mean_terminated_length": 283.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16803645621985197, "epoch": 0.00856, "frac_reward_zero_std": 0.0, "grad_norm": 0.363064706325531, "kl": 1.2409300617873669, "learning_rate": 6.799607761416671e-06, "loss": 0.0197, "num_tokens": 20377053.0, "reward": 0.078125, "reward_std": 0.29842114448547363, "rewards/rollout_reward_func/mean": 0.078125, "rewards/rollout_reward_func/std": 0.2893088161945343, "sampling/importance_sampling_ratio/max": 1.7186404466629028, "sampling/importance_sampling_ratio/mean": 1.008721113204956, "sampling/importance_sampling_ratio/min": 0.481313019990921, "sampling/sampling_logp_difference/max": 0.7312560081481934, "sampling/sampling_logp_difference/mean": 0.03531728684902191, "step": 428, "step_time": 16.175247158000275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 232.3125, "completions/mean_terminated_length": 232.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14415076281875372, "epoch": 0.00858, "frac_reward_zero_std": 0.0, "grad_norm": 0.6018484830856323, "kl": 1.8336940947920084, "learning_rate": 6.786131616666364e-06, "loss": -0.0072, "num_tokens": 20424257.0, "reward": 0.11249999701976776, "reward_std": 0.2774713635444641, "rewards/rollout_reward_func/mean": 0.11249999701976776, "rewards/rollout_reward_func/std": 0.2959402799606323, "sampling/importance_sampling_ratio/max": 1.149275541305542, "sampling/importance_sampling_ratio/mean": 0.9230738282203674, "sampling/importance_sampling_ratio/min": 0.25366970896720886, "sampling/sampling_logp_difference/max": 1.371656894683838, "sampling/sampling_logp_difference/mean": 0.05447674170136452, "step": 429, "step_time": 13.733626876000926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 307.34375, "completions/mean_terminated_length": 307.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1404179222881794, "epoch": 0.0086, "frac_reward_zero_std": 0.0, "grad_norm": 0.3765690326690674, "kl": 1.4189691003412008, "learning_rate": 6.772648399977606e-06, "loss": 0.0078, "num_tokens": 20473701.0, "reward": 0.13437500596046448, "reward_std": 0.23887106776237488, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.23774392902851105, "sampling/importance_sampling_ratio/max": 1.1480406522750854, "sampling/importance_sampling_ratio/mean": 0.9856574535369873, "sampling/importance_sampling_ratio/min": 0.38865911960601807, "sampling/sampling_logp_difference/max": 0.7312240600585938, "sampling/sampling_logp_difference/mean": 0.03253442794084549, "step": 430, "step_time": 13.70560430999467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1652.0, "completions/max_terminated_length": 1652.0, "completions/mean_length": 387.84375, "completions/mean_terminated_length": 387.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14935174956917763, "epoch": 0.00862, "frac_reward_zero_std": 0.0, "grad_norm": 0.7537128329277039, "kl": 0.8704253248870373, "learning_rate": 6.759158289203141e-06, "loss": 0.0167, "num_tokens": 20526294.0, "reward": 0.06562500447034836, "reward_std": 0.3550657331943512, "rewards/rollout_reward_func/mean": 0.06562500447034836, "rewards/rollout_reward_func/std": 0.35433658957481384, "sampling/importance_sampling_ratio/max": 1.3244190216064453, "sampling/importance_sampling_ratio/mean": 0.9813144207000732, "sampling/importance_sampling_ratio/min": 0.4291090965270996, "sampling/sampling_logp_difference/max": 0.8459229469299316, "sampling/sampling_logp_difference/mean": 0.02751581370830536, "step": 431, "step_time": 17.401548103000096 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 296.9375, "completions/mean_terminated_length": 296.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12938870210200548, "epoch": 0.00864, "frac_reward_zero_std": 0.0, "grad_norm": 1.5670355558395386, "kl": 0.4908400820568204, "learning_rate": 6.745661462286648e-06, "loss": 0.0499, "num_tokens": 20574960.0, "reward": 0.171875, "reward_std": 0.2797117233276367, "rewards/rollout_reward_func/mean": 0.171875, "rewards/rollout_reward_func/std": 0.30078861117362976, "sampling/importance_sampling_ratio/max": 2.4607937335968018, "sampling/importance_sampling_ratio/mean": 1.0640056133270264, "sampling/importance_sampling_ratio/min": 0.8570393323898315, "sampling/sampling_logp_difference/max": 0.8912742137908936, "sampling/sampling_logp_difference/mean": 0.026109613478183746, "step": 432, "step_time": 13.68581465799798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 413.5625, "completions/mean_terminated_length": 413.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.20409519877284765, "epoch": 0.00866, "frac_reward_zero_std": 0.0, "grad_norm": 0.4792039096355438, "kl": 0.8960625566542149, "learning_rate": 6.7321580972603996e-06, "loss": 0.0095, "num_tokens": 20629344.0, "reward": -0.04062499850988388, "reward_std": 0.34608906507492065, "rewards/rollout_reward_func/mean": -0.04062499850988388, "rewards/rollout_reward_func/std": 0.33394935727119446, "sampling/importance_sampling_ratio/max": 1.5436376333236694, "sampling/importance_sampling_ratio/mean": 1.0153212547302246, "sampling/importance_sampling_ratio/min": 0.5763399600982666, "sampling/sampling_logp_difference/max": 0.9858932495117188, "sampling/sampling_logp_difference/mean": 0.040325235575437546, "step": 433, "step_time": 14.585508741001831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1310.0, "completions/max_terminated_length": 1310.0, "completions/mean_length": 401.9375, "completions/mean_terminated_length": 401.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17278419947251678, "epoch": 0.00868, "frac_reward_zero_std": 0.0, "grad_norm": 0.4452258050441742, "kl": 2.190219271928072, "learning_rate": 6.718648372242909e-06, "loss": 0.0261, "num_tokens": 20681401.0, "reward": 0.07187500596046448, "reward_std": 0.2857586741447449, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.29318538308143616, "sampling/importance_sampling_ratio/max": 1.1701617240905762, "sampling/importance_sampling_ratio/mean": 0.9264224767684937, "sampling/importance_sampling_ratio/min": 0.32365643978118896, "sampling/sampling_logp_difference/max": 1.135070562362671, "sampling/sampling_logp_difference/mean": 0.05327718332409859, "step": 434, "step_time": 15.169155637006043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 254.65625, "completions/mean_terminated_length": 254.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14134816569276154, "epoch": 0.0087, "frac_reward_zero_std": 0.0, "grad_norm": 0.5080227851867676, "kl": 1.0836692629382014, "learning_rate": 6.705132465436579e-06, "loss": -0.0068, "num_tokens": 20728672.0, "reward": 0.125, "reward_std": 0.32617777585983276, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.32328954339027405, "sampling/importance_sampling_ratio/max": 1.4307204484939575, "sampling/importance_sampling_ratio/mean": 1.0308847427368164, "sampling/importance_sampling_ratio/min": 0.69562166929245, "sampling/sampling_logp_difference/max": 0.3682374954223633, "sampling/sampling_logp_difference/mean": 0.02187609300017357, "step": 435, "step_time": 13.184785362995171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 298.75, "completions/mean_terminated_length": 298.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15965187433175743, "epoch": 0.00872, "frac_reward_zero_std": 0.0, "grad_norm": 0.420966774225235, "kl": 0.43749012891203165, "learning_rate": 6.691610555125361e-06, "loss": 0.0407, "num_tokens": 20777673.0, "reward": 0.08124999701976776, "reward_std": 0.22323143482208252, "rewards/rollout_reward_func/mean": 0.08124999701976776, "rewards/rollout_reward_func/std": 0.238864928483963, "sampling/importance_sampling_ratio/max": 1.4306247234344482, "sampling/importance_sampling_ratio/mean": 1.0210705995559692, "sampling/importance_sampling_ratio/min": 0.5837316513061523, "sampling/sampling_logp_difference/max": 0.44653749465942383, "sampling/sampling_logp_difference/mean": 0.01951594464480877, "step": 436, "step_time": 14.109348219000822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1535.0, "completions/max_terminated_length": 1535.0, "completions/mean_length": 394.15625, "completions/mean_terminated_length": 394.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1306122199166566, "epoch": 0.00874, "frac_reward_zero_std": 0.0, "grad_norm": 0.4693588316440582, "kl": 0.4813559828326106, "learning_rate": 6.678082819672389e-06, "loss": -0.0445, "num_tokens": 20829801.0, "reward": 0.19062501192092896, "reward_std": 0.27464067935943604, "rewards/rollout_reward_func/mean": 0.19062501192092896, "rewards/rollout_reward_func/std": 0.2966037690639496, "sampling/importance_sampling_ratio/max": 1.4500583410263062, "sampling/importance_sampling_ratio/mean": 1.0611703395843506, "sampling/importance_sampling_ratio/min": 0.6510156989097595, "sampling/sampling_logp_difference/max": 0.43720245361328125, "sampling/sampling_logp_difference/mean": 0.020463164895772934, "step": 437, "step_time": 15.698222672996053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 398.78125, "completions/mean_terminated_length": 398.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13529891427606344, "epoch": 0.00876, "frac_reward_zero_std": 0.0, "grad_norm": 0.7886952757835388, "kl": 0.4798082048073411, "learning_rate": 6.664549437517642e-06, "loss": -0.0094, "num_tokens": 20882860.0, "reward": 0.15937498211860657, "reward_std": 0.29271191358566284, "rewards/rollout_reward_func/mean": 0.15937498211860657, "rewards/rollout_reward_func/std": 0.30781158804893494, "sampling/importance_sampling_ratio/max": 1.2775452136993408, "sampling/importance_sampling_ratio/mean": 0.9999099969863892, "sampling/importance_sampling_ratio/min": 0.5254403352737427, "sampling/sampling_logp_difference/max": 0.661545991897583, "sampling/sampling_logp_difference/mean": 0.03489738702774048, "step": 438, "step_time": 13.792009779008367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 350.71875, "completions/mean_terminated_length": 350.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1345423818565905, "epoch": 0.00878, "frac_reward_zero_std": 0.0, "grad_norm": 0.549470841884613, "kl": 1.8810350894927979, "learning_rate": 6.651010587175577e-06, "loss": -0.0001, "num_tokens": 20934922.0, "reward": 0.06562499701976776, "reward_std": 0.17739354074001312, "rewards/rollout_reward_func/mean": 0.06562499701976776, "rewards/rollout_reward_func/std": 0.20257914066314697, "sampling/importance_sampling_ratio/max": 1.1544569730758667, "sampling/importance_sampling_ratio/mean": 0.9757360219955444, "sampling/importance_sampling_ratio/min": 0.36012163758277893, "sampling/sampling_logp_difference/max": 0.726752758026123, "sampling/sampling_logp_difference/mean": 0.029930703341960907, "step": 439, "step_time": 14.104368999003782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 331.90625, "completions/mean_terminated_length": 331.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1145544801838696, "epoch": 0.0088, "frac_reward_zero_std": 0.0, "grad_norm": 0.842621922492981, "kl": 0.6403545383363962, "learning_rate": 6.637466447232784e-06, "loss": 0.002, "num_tokens": 20985284.0, "reward": 0.13750000298023224, "reward_std": 0.247517928481102, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.2697071433067322, "sampling/importance_sampling_ratio/max": 1.6123915910720825, "sampling/importance_sampling_ratio/mean": 0.9961374998092651, "sampling/importance_sampling_ratio/min": 0.43526315689086914, "sampling/sampling_logp_difference/max": 0.8520708084106445, "sampling/sampling_logp_difference/mean": 0.030260805040597916, "step": 440, "step_time": 15.452269685993087 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 322.96875, "completions/mean_terminated_length": 322.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10110700083896518, "epoch": 0.00882, "frac_reward_zero_std": 0.0, "grad_norm": 0.3781487047672272, "kl": 0.9445002563297749, "learning_rate": 6.623917196345622e-06, "loss": 0.0052, "num_tokens": 21035214.0, "reward": 0.13124999403953552, "reward_std": 0.24623459577560425, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.25455525517463684, "sampling/importance_sampling_ratio/max": 1.1469049453735352, "sampling/importance_sampling_ratio/mean": 0.9876381754875183, "sampling/importance_sampling_ratio/min": 0.5931151509284973, "sampling/sampling_logp_difference/max": 0.48769283294677734, "sampling/sampling_logp_difference/mean": 0.02040206640958786, "step": 441, "step_time": 13.907406623002316 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 226.5625, "completions/mean_terminated_length": 226.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1229006047360599, "epoch": 0.00884, "frac_reward_zero_std": 0.0, "grad_norm": 0.4210430383682251, "kl": 0.4455987233668566, "learning_rate": 6.610363013237871e-06, "loss": -0.0195, "num_tokens": 21081726.0, "reward": 0.17812500894069672, "reward_std": 0.26330891251564026, "rewards/rollout_reward_func/mean": 0.17812500894069672, "rewards/rollout_reward_func/std": 0.2790934145450592, "sampling/importance_sampling_ratio/max": 1.2410709857940674, "sampling/importance_sampling_ratio/mean": 0.9879369139671326, "sampling/importance_sampling_ratio/min": 0.41511136293411255, "sampling/sampling_logp_difference/max": 0.5001621246337891, "sampling/sampling_logp_difference/mean": 0.0223191250115633, "step": 442, "step_time": 13.774339738003619 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1661.0, "completions/max_terminated_length": 1661.0, "completions/mean_length": 449.5625, "completions/mean_terminated_length": 449.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11386880092322826, "epoch": 0.00886, "frac_reward_zero_std": 0.0, "grad_norm": 0.5926356911659241, "kl": 0.4486855361610651, "learning_rate": 6.59680407669837e-06, "loss": 0.0725, "num_tokens": 21134992.0, "reward": 0.07187499105930328, "reward_std": 0.2757096588611603, "rewards/rollout_reward_func/mean": 0.07187499105930328, "rewards/rollout_reward_func/std": 0.29428356885910034, "sampling/importance_sampling_ratio/max": 1.75272536277771, "sampling/importance_sampling_ratio/mean": 1.0108791589736938, "sampling/importance_sampling_ratio/min": 0.7351943850517273, "sampling/sampling_logp_difference/max": 0.6199746131896973, "sampling/sampling_logp_difference/mean": 0.025225404649972916, "step": 443, "step_time": 16.46292193899717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1679.0, "completions/max_terminated_length": 1679.0, "completions/mean_length": 309.34375, "completions/mean_terminated_length": 309.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14201355166733265, "epoch": 0.00888, "frac_reward_zero_std": 0.0, "grad_norm": 0.6748592853546143, "kl": 0.5013363352045417, "learning_rate": 6.583240565578657e-06, "loss": -0.0548, "num_tokens": 21184106.0, "reward": 0.15312500298023224, "reward_std": 0.28818202018737793, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.28847137093544006, "sampling/importance_sampling_ratio/max": 2.090906858444214, "sampling/importance_sampling_ratio/mean": 1.0202646255493164, "sampling/importance_sampling_ratio/min": 0.610682487487793, "sampling/sampling_logp_difference/max": 0.4931640625, "sampling/sampling_logp_difference/mean": 0.03131510689854622, "step": 444, "step_time": 16.034328651003307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1418.0, "completions/max_terminated_length": 1418.0, "completions/mean_length": 443.09375, "completions/mean_terminated_length": 443.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11805749265477061, "epoch": 0.0089, "frac_reward_zero_std": 0.0, "grad_norm": 0.43796107172966003, "kl": 0.4323974484577775, "learning_rate": 6.569672658790611e-06, "loss": 0.0333, "num_tokens": 21237987.0, "reward": 0.07187500596046448, "reward_std": 0.24778161942958832, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.26668137311935425, "sampling/importance_sampling_ratio/max": 1.5024018287658691, "sampling/importance_sampling_ratio/mean": 0.980978786945343, "sampling/importance_sampling_ratio/min": 0.2597346603870392, "sampling/sampling_logp_difference/max": 1.3480968475341797, "sampling/sampling_logp_difference/mean": 0.035059306770563126, "step": 445, "step_time": 15.504307967996283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 285.96875, "completions/mean_terminated_length": 285.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07805271004326642, "epoch": 0.00892, "frac_reward_zero_std": 0.0, "grad_norm": 0.1796276867389679, "kl": 0.5042393635958433, "learning_rate": 6.556100535304097e-06, "loss": -0.0158, "num_tokens": 21287225.0, "reward": 0.25312501192092896, "reward_std": 0.1969682276248932, "rewards/rollout_reward_func/mean": 0.25312501192092896, "rewards/rollout_reward_func/std": 0.19835512340068817, "sampling/importance_sampling_ratio/max": 1.1705052852630615, "sampling/importance_sampling_ratio/mean": 1.009132981300354, "sampling/importance_sampling_ratio/min": 0.6122534871101379, "sampling/sampling_logp_difference/max": 0.49552488327026367, "sampling/sampling_logp_difference/mean": 0.013599516823887825, "step": 446, "step_time": 13.717971595000563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 286.40625, "completions/mean_terminated_length": 286.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08962459862232208, "epoch": 0.00894, "frac_reward_zero_std": 0.0, "grad_norm": 0.29549750685691833, "kl": 0.5679004918783903, "learning_rate": 6.542524374144598e-06, "loss": 0.0447, "num_tokens": 21335008.0, "reward": 0.171875, "reward_std": 0.17154526710510254, "rewards/rollout_reward_func/mean": 0.171875, "rewards/rollout_reward_func/std": 0.172709658741951, "sampling/importance_sampling_ratio/max": 1.2146750688552856, "sampling/importance_sampling_ratio/mean": 1.0263803005218506, "sampling/importance_sampling_ratio/min": 0.4899781048297882, "sampling/sampling_logp_difference/max": 0.7191647887229919, "sampling/sampling_logp_difference/mean": 0.01820582151412964, "step": 447, "step_time": 13.73036283399415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 236.5625, "completions/mean_terminated_length": 236.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0870105482172221, "epoch": 0.00896, "frac_reward_zero_std": 0.0, "grad_norm": 0.31101617217063904, "kl": 1.4035230157896876, "learning_rate": 6.528944354390855e-06, "loss": 0.018, "num_tokens": 21380136.0, "reward": 0.13437500596046448, "reward_std": 0.28787630796432495, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.3022596538066864, "sampling/importance_sampling_ratio/max": 1.1726744174957275, "sampling/importance_sampling_ratio/mean": 0.9815227389335632, "sampling/importance_sampling_ratio/min": 0.4117162823677063, "sampling/sampling_logp_difference/max": 0.9052615165710449, "sampling/sampling_logp_difference/mean": 0.03235447034239769, "step": 448, "step_time": 14.943456817996775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 261.125, "completions/mean_terminated_length": 261.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07001803815364838, "epoch": 0.00898, "frac_reward_zero_std": 0.0, "grad_norm": 0.4085979163646698, "kl": 0.6465711845085025, "learning_rate": 6.515360655172512e-06, "loss": 0.0069, "num_tokens": 21427236.0, "reward": 0.1937500238418579, "reward_std": 0.2303827404975891, "rewards/rollout_reward_func/mean": 0.1937500238418579, "rewards/rollout_reward_func/std": 0.2381887286901474, "sampling/importance_sampling_ratio/max": 1.1588634252548218, "sampling/importance_sampling_ratio/mean": 1.0028917789459229, "sampling/importance_sampling_ratio/min": 0.6772589683532715, "sampling/sampling_logp_difference/max": 0.4960951805114746, "sampling/sampling_logp_difference/mean": 0.01374916173517704, "step": 449, "step_time": 13.714474597014487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 259.3125, "completions/mean_terminated_length": 259.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06805372750386596, "epoch": 0.009, "frac_reward_zero_std": 0.0, "grad_norm": 1.1079260110855103, "kl": 1.0743695944547653, "learning_rate": 6.501773455667742e-06, "loss": -0.008, "num_tokens": 21474436.0, "reward": 0.15312500298023224, "reward_std": 0.29160189628601074, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.2929101884365082, "sampling/importance_sampling_ratio/max": 1.571265697479248, "sampling/importance_sampling_ratio/mean": 1.0009307861328125, "sampling/importance_sampling_ratio/min": 0.5395470857620239, "sampling/sampling_logp_difference/max": 0.6170334815979004, "sampling/sampling_logp_difference/mean": 0.02768099121749401, "step": 450, "step_time": 13.316212720004842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 293.8125, "completions/mean_terminated_length": 293.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.056102871894836426, "epoch": 0.00902, "frac_reward_zero_std": 0.0, "grad_norm": 0.431458055973053, "kl": 0.818661886267364, "learning_rate": 6.488182935100893e-06, "loss": -0.0033, "num_tokens": 21524534.0, "reward": 0.13750000298023224, "reward_std": 0.2883473336696625, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.2836967706680298, "sampling/importance_sampling_ratio/max": 1.1051424741744995, "sampling/importance_sampling_ratio/mean": 0.9833378195762634, "sampling/importance_sampling_ratio/min": 0.3569657802581787, "sampling/sampling_logp_difference/max": 1.034766674041748, "sampling/sampling_logp_difference/mean": 0.01918959990143776, "step": 451, "step_time": 13.834141077997629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 308.625, "completions/mean_terminated_length": 308.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06727051513735205, "epoch": 0.00904, "frac_reward_zero_std": 0.0, "grad_norm": 0.3526538908481598, "kl": 0.5032529206946492, "learning_rate": 6.474589272740116e-06, "loss": 0.0107, "num_tokens": 21573208.0, "reward": 0.18125000596046448, "reward_std": 0.22513651847839355, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.25072476267814636, "sampling/importance_sampling_ratio/max": 1.1488423347473145, "sampling/importance_sampling_ratio/mean": 0.9865505695343018, "sampling/importance_sampling_ratio/min": 0.08150265365839005, "sampling/sampling_logp_difference/max": 2.513140916824341, "sampling/sampling_logp_difference/mean": 0.03291352838277817, "step": 452, "step_time": 13.679186102999665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 311.5625, "completions/mean_terminated_length": 311.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09915350913070142, "epoch": 0.00906, "frac_reward_zero_std": 0.0, "grad_norm": 0.7170552611351013, "kl": 0.8355281669646502, "learning_rate": 6.46099264789501e-06, "loss": 0.0243, "num_tokens": 21623251.0, "reward": 0.10312500596046448, "reward_std": 0.3088546693325043, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.30950987339019775, "sampling/importance_sampling_ratio/max": 1.4925819635391235, "sampling/importance_sampling_ratio/mean": 0.9968535900115967, "sampling/importance_sampling_ratio/min": 0.24106720089912415, "sampling/sampling_logp_difference/max": 1.422654151916504, "sampling/sampling_logp_difference/mean": 0.02935473620891571, "step": 453, "step_time": 14.697387711988995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 256.1875, "completions/mean_terminated_length": 256.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06497138971462846, "epoch": 0.00908, "frac_reward_zero_std": 0.0, "grad_norm": 0.10624273121356964, "kl": 0.41613150760531425, "learning_rate": 6.447393239914247e-06, "loss": -0.0129, "num_tokens": 21670167.0, "reward": 0.3031249940395355, "reward_std": 0.1933746486902237, "rewards/rollout_reward_func/mean": 0.3031249940395355, "rewards/rollout_reward_func/std": 0.19424688816070557, "sampling/importance_sampling_ratio/max": 1.0512436628341675, "sampling/importance_sampling_ratio/mean": 0.9785129427909851, "sampling/importance_sampling_ratio/min": 1.5189785926850163e-06, "sampling/sampling_logp_difference/max": 13.321922302246094, "sampling/sampling_logp_difference/mean": 0.12300033867359161, "step": 454, "step_time": 14.079137954999169 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 279.34375, "completions/mean_terminated_length": 279.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0582974684657529, "epoch": 0.0091, "frac_reward_zero_std": 0.0, "grad_norm": 0.5461732745170593, "kl": 0.5847330149263144, "learning_rate": 6.433791228183211e-06, "loss": 0.0356, "num_tokens": 21719252.0, "reward": 0.20937500894069672, "reward_std": 0.18301406502723694, "rewards/rollout_reward_func/mean": 0.20937500894069672, "rewards/rollout_reward_func/std": 0.18898604810237885, "sampling/importance_sampling_ratio/max": 1.0804007053375244, "sampling/importance_sampling_ratio/mean": 0.9747315645217896, "sampling/importance_sampling_ratio/min": 0.4818727970123291, "sampling/sampling_logp_difference/max": 0.7940740585327148, "sampling/sampling_logp_difference/mean": 0.020564531907439232, "step": 455, "step_time": 15.257837680997909 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 213.78125, "completions/mean_terminated_length": 213.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0803898696321994, "epoch": 0.00912, "frac_reward_zero_std": 0.0, "grad_norm": 4.93668794631958, "kl": 32.274774947203696, "learning_rate": 6.42018679212163e-06, "loss": 0.0251, "num_tokens": 21764759.0, "reward": 0.125, "reward_std": 0.31290867924690247, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.3121207356452942, "sampling/importance_sampling_ratio/max": 1.1063593626022339, "sampling/importance_sampling_ratio/mean": 0.9186422824859619, "sampling/importance_sampling_ratio/min": 0.28730034828186035, "sampling/sampling_logp_difference/max": 1.265537977218628, "sampling/sampling_logp_difference/mean": 0.05060885474085808, "step": 456, "step_time": 13.978842099008034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 312.84375, "completions/mean_terminated_length": 312.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08540154120419174, "epoch": 0.00914, "frac_reward_zero_std": 0.0, "grad_norm": 0.49509528279304504, "kl": 1.5144995264708996, "learning_rate": 6.406580111181216e-06, "loss": 0.0011, "num_tokens": 21813702.0, "reward": 0.109375, "reward_std": 0.2739243507385254, "rewards/rollout_reward_func/mean": 0.109375, "rewards/rollout_reward_func/std": 0.28438884019851685, "sampling/importance_sampling_ratio/max": 1.211756944656372, "sampling/importance_sampling_ratio/mean": 0.9710383415222168, "sampling/importance_sampling_ratio/min": 0.27200406789779663, "sampling/sampling_logp_difference/max": 1.3056309223175049, "sampling/sampling_logp_difference/mean": 0.032044775784015656, "step": 457, "step_time": 14.13409402999605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 367.46875, "completions/mean_terminated_length": 367.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09478158387355506, "epoch": 0.00916, "frac_reward_zero_std": 0.0, "grad_norm": 0.5716108679771423, "kl": 0.5463854055851698, "learning_rate": 6.392971364843287e-06, "loss": -0.0201, "num_tokens": 21865868.0, "reward": 0.10000000149011612, "reward_std": 0.26034075021743774, "rewards/rollout_reward_func/mean": 0.10000000149011612, "rewards/rollout_reward_func/std": 0.2782433331012726, "sampling/importance_sampling_ratio/max": 1.2003169059753418, "sampling/importance_sampling_ratio/mean": 0.9975398778915405, "sampling/importance_sampling_ratio/min": 0.7223106026649475, "sampling/sampling_logp_difference/max": 0.3409423828125, "sampling/sampling_logp_difference/mean": 0.014895254746079445, "step": 458, "step_time": 15.282519084001251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 279.0, "completions/mean_terminated_length": 279.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09701639483682811, "epoch": 0.00918, "frac_reward_zero_std": 0.0, "grad_norm": 0.6320081949234009, "kl": 2.810023056343198, "learning_rate": 6.379360732616405e-06, "loss": 0.0076, "num_tokens": 21913449.0, "reward": 0.125, "reward_std": 0.2912595272064209, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.3026762306690216, "sampling/importance_sampling_ratio/max": 1.0939016342163086, "sampling/importance_sampling_ratio/mean": 0.9552700519561768, "sampling/importance_sampling_ratio/min": 0.6018095016479492, "sampling/sampling_logp_difference/max": 0.602604866027832, "sampling/sampling_logp_difference/mean": 0.03045308031141758, "step": 459, "step_time": 14.298475446004886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 226.875, "completions/mean_terminated_length": 226.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07735799648799002, "epoch": 0.0092, "frac_reward_zero_std": 0.0, "grad_norm": 0.2605865001678467, "kl": 0.42357591912150383, "learning_rate": 6.365748394034013e-06, "loss": 0.032, "num_tokens": 21958692.0, "reward": 0.125, "reward_std": 0.2677494287490845, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.2700059711933136, "sampling/importance_sampling_ratio/max": 1.0607808828353882, "sampling/importance_sampling_ratio/mean": 0.9893472194671631, "sampling/importance_sampling_ratio/min": 0.7202231884002686, "sampling/sampling_logp_difference/max": 0.3671119213104248, "sampling/sampling_logp_difference/mean": 0.012495272792875767, "step": 460, "step_time": 14.539111489997595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 258.03125, "completions/mean_terminated_length": 258.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08347574062645435, "epoch": 0.00922, "frac_reward_zero_std": 0.0, "grad_norm": 0.413011372089386, "kl": 4.160022780299187, "learning_rate": 6.352134528652057e-06, "loss": -0.0022, "num_tokens": 22006094.0, "reward": 0.16562500596046448, "reward_std": 0.24106410145759583, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.24836155772209167, "sampling/importance_sampling_ratio/max": 1.1710691452026367, "sampling/importance_sampling_ratio/mean": 0.9740516543388367, "sampling/importance_sampling_ratio/min": 0.3702920377254486, "sampling/sampling_logp_difference/max": 0.997157096862793, "sampling/sampling_logp_difference/mean": 0.030289623886346817, "step": 461, "step_time": 13.509597322004993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2097.0, "completions/max_terminated_length": 2097.0, "completions/mean_length": 367.03125, "completions/mean_terminated_length": 367.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10541097354143858, "epoch": 0.00924, "frac_reward_zero_std": 0.0, "grad_norm": 0.4419015645980835, "kl": 0.534148583188653, "learning_rate": 6.3385193160466255e-06, "loss": -0.0189, "num_tokens": 22057979.0, "reward": 0.16249999403953552, "reward_std": 0.29082560539245605, "rewards/rollout_reward_func/mean": 0.16249999403953552, "rewards/rollout_reward_func/std": 0.31289491057395935, "sampling/importance_sampling_ratio/max": 1.6064050197601318, "sampling/importance_sampling_ratio/mean": 1.0286471843719482, "sampling/importance_sampling_ratio/min": 0.6646354794502258, "sampling/sampling_logp_difference/max": 0.4085143804550171, "sampling/sampling_logp_difference/mean": 0.017412036657333374, "step": 462, "step_time": 20.321474063995993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 297.53125, "completions/mean_terminated_length": 297.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08621953357942402, "epoch": 0.00926, "frac_reward_zero_std": 0.0, "grad_norm": 0.7218598127365112, "kl": 0.4880874324589968, "learning_rate": 6.324902935811576e-06, "loss": 0.0147, "num_tokens": 22106287.0, "reward": 0.20937500894069672, "reward_std": 0.23751245439052582, "rewards/rollout_reward_func/mean": 0.20937500894069672, "rewards/rollout_reward_func/std": 0.2644036114215851, "sampling/importance_sampling_ratio/max": 1.70063316822052, "sampling/importance_sampling_ratio/mean": 1.041657567024231, "sampling/importance_sampling_ratio/min": 0.5106067657470703, "sampling/sampling_logp_difference/max": 0.6817488670349121, "sampling/sampling_logp_difference/mean": 0.025984160602092743, "step": 463, "step_time": 15.191101176984375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 367.25, "completions/mean_terminated_length": 367.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10585637344047427, "epoch": 0.00928, "frac_reward_zero_std": 0.0, "grad_norm": 0.4990706741809845, "kl": 0.6761839333921671, "learning_rate": 6.311285567556168e-06, "loss": -0.0198, "num_tokens": 22158889.0, "reward": 0.14687499403953552, "reward_std": 0.23706328868865967, "rewards/rollout_reward_func/mean": 0.14687499403953552, "rewards/rollout_reward_func/std": 0.28282490372657776, "sampling/importance_sampling_ratio/max": 1.6485018730163574, "sampling/importance_sampling_ratio/mean": 1.0043047666549683, "sampling/importance_sampling_ratio/min": 0.4822704792022705, "sampling/sampling_logp_difference/max": 0.7425212860107422, "sampling/sampling_logp_difference/mean": 0.025158125907182693, "step": 464, "step_time": 14.263024812007643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 223.59375, "completions/mean_terminated_length": 223.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.130990631878376, "epoch": 0.0093, "frac_reward_zero_std": 0.0, "grad_norm": 0.5704765319824219, "kl": 0.6067672278732061, "learning_rate": 6.297667390902694e-06, "loss": -0.0103, "num_tokens": 22205827.0, "reward": 0.12812499701976776, "reward_std": 0.25553497672080994, "rewards/rollout_reward_func/mean": 0.12812499701976776, "rewards/rollout_reward_func/std": 0.2593221664428711, "sampling/importance_sampling_ratio/max": 1.4358272552490234, "sampling/importance_sampling_ratio/mean": 0.9768781661987305, "sampling/importance_sampling_ratio/min": 0.2660187780857086, "sampling/sampling_logp_difference/max": 1.3241612911224365, "sampling/sampling_logp_difference/mean": 0.039804328233003616, "step": 465, "step_time": 12.686231181014591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1328.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 296.25, "completions/mean_terminated_length": 296.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08893824578262866, "epoch": 0.00932, "frac_reward_zero_std": 0.0, "grad_norm": 0.6183410882949829, "kl": 0.4659380167722702, "learning_rate": 6.284048585484113e-06, "loss": 0.0335, "num_tokens": 22254331.0, "reward": 0.14999999105930328, "reward_std": 0.30509546399116516, "rewards/rollout_reward_func/mean": 0.14999999105930328, "rewards/rollout_reward_func/std": 0.31418097019195557, "sampling/importance_sampling_ratio/max": 2.117435932159424, "sampling/importance_sampling_ratio/mean": 1.0331077575683594, "sampling/importance_sampling_ratio/min": 0.6443914771080017, "sampling/sampling_logp_difference/max": 0.6509475708007812, "sampling/sampling_logp_difference/mean": 0.02490672841668129, "step": 466, "step_time": 14.511594454008446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 270.8125, "completions/mean_terminated_length": 270.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08615504950284958, "epoch": 0.00934, "frac_reward_zero_std": 0.0, "grad_norm": 0.3088890314102173, "kl": 0.898677084594965, "learning_rate": 6.270429330941671e-06, "loss": 0.0184, "num_tokens": 22302616.0, "reward": 0.12187500298023224, "reward_std": 0.3061155676841736, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.31799206137657166, "sampling/importance_sampling_ratio/max": 1.16474187374115, "sampling/importance_sampling_ratio/mean": 0.9870100021362305, "sampling/importance_sampling_ratio/min": 0.4239427447319031, "sampling/sampling_logp_difference/max": 0.8650827407836914, "sampling/sampling_logp_difference/mean": 0.01924203149974346, "step": 467, "step_time": 13.942429371985781 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 294.65625, "completions/mean_terminated_length": 294.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15629862085916102, "epoch": 0.00936, "frac_reward_zero_std": 0.0, "grad_norm": 0.6960627436637878, "kl": 0.5188144519925117, "learning_rate": 6.2568098069225436e-06, "loss": 0.0317, "num_tokens": 22352415.0, "reward": 0.06562500447034836, "reward_std": 0.27103671431541443, "rewards/rollout_reward_func/mean": 0.06562500447034836, "rewards/rollout_reward_func/std": 0.28916943073272705, "sampling/importance_sampling_ratio/max": 1.3407522439956665, "sampling/importance_sampling_ratio/mean": 0.9665672183036804, "sampling/importance_sampling_ratio/min": 1.4134072898741579e-06, "sampling/sampling_logp_difference/max": 13.398346900939941, "sampling/sampling_logp_difference/mean": 0.12836892902851105, "step": 468, "step_time": 14.081260395014397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 288.0625, "completions/mean_terminated_length": 288.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11308033112436533, "epoch": 0.00938, "frac_reward_zero_std": 0.0, "grad_norm": 0.31189295649528503, "kl": 0.7886871946975589, "learning_rate": 6.243190193077457e-06, "loss": -0.0111, "num_tokens": 22401089.0, "reward": 0.08750000596046448, "reward_std": 0.28391826152801514, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.3097865879535675, "sampling/importance_sampling_ratio/max": 1.1804509162902832, "sampling/importance_sampling_ratio/mean": 0.9830670952796936, "sampling/importance_sampling_ratio/min": 0.5072558522224426, "sampling/sampling_logp_difference/max": 0.6835724115371704, "sampling/sampling_logp_difference/mean": 0.034990377724170685, "step": 469, "step_time": 12.479999118997512 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1409.0, "completions/max_terminated_length": 1409.0, "completions/mean_length": 305.34375, "completions/mean_terminated_length": 305.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11487723095342517, "epoch": 0.0094, "frac_reward_zero_std": 0.0, "grad_norm": 0.38952741026878357, "kl": 0.8220664793625474, "learning_rate": 6.2295706690583325e-06, "loss": 0.0028, "num_tokens": 22450523.0, "reward": 0.14687500894069672, "reward_std": 0.2907605767250061, "rewards/rollout_reward_func/mean": 0.14687500894069672, "rewards/rollout_reward_func/std": 0.3110693097114563, "sampling/importance_sampling_ratio/max": 1.109169602394104, "sampling/importance_sampling_ratio/mean": 0.9442484974861145, "sampling/importance_sampling_ratio/min": 0.4108275771141052, "sampling/sampling_logp_difference/max": 1.0467491149902344, "sampling/sampling_logp_difference/mean": 0.037864118814468384, "step": 470, "step_time": 15.11709236400202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 865.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 189.59375, "completions/mean_terminated_length": 189.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10339253954589367, "epoch": 0.00942, "frac_reward_zero_std": 0.0, "grad_norm": 0.26081669330596924, "kl": 0.9546050066128373, "learning_rate": 6.215951414515888e-06, "loss": -0.0082, "num_tokens": 22494721.0, "reward": 0.21250000596046448, "reward_std": 0.2627588212490082, "rewards/rollout_reward_func/mean": 0.21250000596046448, "rewards/rollout_reward_func/std": 0.2648797929286957, "sampling/importance_sampling_ratio/max": 1.2533378601074219, "sampling/importance_sampling_ratio/mean": 1.0003079175949097, "sampling/importance_sampling_ratio/min": 0.5495671033859253, "sampling/sampling_logp_difference/max": 0.5986495018005371, "sampling/sampling_logp_difference/mean": 0.019177792593836784, "step": 471, "step_time": 12.538303994006128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 278.96875, "completions/mean_terminated_length": 278.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12891492201015353, "epoch": 0.00944, "frac_reward_zero_std": 0.0, "grad_norm": 0.35514408349990845, "kl": 0.47408119309693575, "learning_rate": 6.202332609097308e-06, "loss": 0.0319, "num_tokens": 22542974.0, "reward": 0.109375, "reward_std": 0.22824794054031372, "rewards/rollout_reward_func/mean": 0.109375, "rewards/rollout_reward_func/std": 0.23604178428649902, "sampling/importance_sampling_ratio/max": 1.4746664762496948, "sampling/importance_sampling_ratio/mean": 1.0056840181350708, "sampling/importance_sampling_ratio/min": 0.641943633556366, "sampling/sampling_logp_difference/max": 0.4433351457118988, "sampling/sampling_logp_difference/mean": 0.021190840750932693, "step": 472, "step_time": 13.751259927008505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1261.0, "completions/max_terminated_length": 1261.0, "completions/mean_length": 312.0, "completions/mean_terminated_length": 312.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09316121065057814, "epoch": 0.00946, "frac_reward_zero_std": 0.0, "grad_norm": 0.1598655730485916, "kl": 0.49872606433928013, "learning_rate": 6.188714432443833e-06, "loss": -0.0006, "num_tokens": 22591978.0, "reward": 0.15937501192092896, "reward_std": 0.322401225566864, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.322149395942688, "sampling/importance_sampling_ratio/max": 1.1310676336288452, "sampling/importance_sampling_ratio/mean": 1.0264832973480225, "sampling/importance_sampling_ratio/min": 0.9865149855613708, "sampling/sampling_logp_difference/max": 0.09442192316055298, "sampling/sampling_logp_difference/mean": 0.00781965535134077, "step": 473, "step_time": 14.6192798179909 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 332.75, "completions/mean_terminated_length": 332.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11225810437463224, "epoch": 0.00948, "frac_reward_zero_std": 0.0, "grad_norm": 81.3935317993164, "kl": 167.9002489876002, "learning_rate": 6.175097064188427e-06, "loss": 0.4019, "num_tokens": 22642856.0, "reward": 0.10312499850988388, "reward_std": 0.26467040181159973, "rewards/rollout_reward_func/mean": 0.10312499850988388, "rewards/rollout_reward_func/std": 0.2583875060081482, "sampling/importance_sampling_ratio/max": 1.2955431938171387, "sampling/importance_sampling_ratio/mean": 0.9812687039375305, "sampling/importance_sampling_ratio/min": 0.1397647112607956, "sampling/sampling_logp_difference/max": 1.9708433151245117, "sampling/sampling_logp_difference/mean": 0.03745491802692413, "step": 474, "step_time": 13.823586595004599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 244.25, "completions/mean_terminated_length": 244.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1338141541928053, "epoch": 0.0095, "frac_reward_zero_std": 0.0, "grad_norm": 0.8915480375289917, "kl": 0.45360810123384, "learning_rate": 6.161480683953376e-06, "loss": 0.019, "num_tokens": 22690662.0, "reward": 0.13124999403953552, "reward_std": 0.2803419232368469, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.2923072874546051, "sampling/importance_sampling_ratio/max": 1.3357813358306885, "sampling/importance_sampling_ratio/mean": 0.9679811000823975, "sampling/importance_sampling_ratio/min": 0.4040350019931793, "sampling/sampling_logp_difference/max": 0.9273200035095215, "sampling/sampling_logp_difference/mean": 0.033993229269981384, "step": 475, "step_time": 13.81360604499423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1418.0, "completions/max_terminated_length": 1418.0, "completions/mean_length": 366.34375, "completions/mean_terminated_length": 366.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11958582885563374, "epoch": 0.00952, "frac_reward_zero_std": 0.0, "grad_norm": 0.7378484606742859, "kl": 1.7981013674288988, "learning_rate": 6.147865471347945e-06, "loss": -0.0018, "num_tokens": 22741869.0, "reward": 0.05312500149011612, "reward_std": 0.3137362003326416, "rewards/rollout_reward_func/mean": 0.05312500149011612, "rewards/rollout_reward_func/std": 0.3182455599308014, "sampling/importance_sampling_ratio/max": 2.092175006866455, "sampling/importance_sampling_ratio/mean": 1.0336854457855225, "sampling/importance_sampling_ratio/min": 0.3476461172103882, "sampling/sampling_logp_difference/max": 0.9608125686645508, "sampling/sampling_logp_difference/mean": 0.04898162558674812, "step": 476, "step_time": 15.346272068996768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 310.5625, "completions/mean_terminated_length": 314.4193420410156, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17292615119367838, "epoch": 0.00954, "frac_reward_zero_std": 0.0, "grad_norm": 0.5525920987129211, "kl": 0.4799490300938487, "learning_rate": 6.134251605965988e-06, "loss": -0.0344, "num_tokens": 22791789.0, "reward": 0.11874999850988388, "reward_std": 0.29361528158187866, "rewards/rollout_reward_func/mean": 0.11874999850988388, "rewards/rollout_reward_func/std": 0.29009175300598145, "sampling/importance_sampling_ratio/max": 1.3418431282043457, "sampling/importance_sampling_ratio/mean": 1.019049882888794, "sampling/importance_sampling_ratio/min": 0.1797299087047577, "sampling/sampling_logp_difference/max": 1.720088005065918, "sampling/sampling_logp_difference/mean": 0.038032494485378265, "step": 477, "step_time": 16.015479310997762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 424.1875, "completions/mean_terminated_length": 424.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08953057788312435, "epoch": 0.00956, "frac_reward_zero_std": 0.0, "grad_norm": 0.27096325159072876, "kl": 1.5799953881651163, "learning_rate": 6.1206392673835955e-06, "loss": -0.0143, "num_tokens": 22846721.0, "reward": 0.10000000149011612, "reward_std": 0.34574511647224426, "rewards/rollout_reward_func/mean": 0.10000000149011612, "rewards/rollout_reward_func/std": 0.3340851664543152, "sampling/importance_sampling_ratio/max": 1.1977567672729492, "sampling/importance_sampling_ratio/mean": 0.9247232675552368, "sampling/importance_sampling_ratio/min": 0.37965989112854004, "sampling/sampling_logp_difference/max": 0.9684882164001465, "sampling/sampling_logp_difference/mean": 0.0402502715587616, "step": 478, "step_time": 13.623328393008705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 400.59375, "completions/mean_terminated_length": 400.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09218995156697929, "epoch": 0.00958, "frac_reward_zero_std": 0.0, "grad_norm": 0.30598193407058716, "kl": 0.44704100489616394, "learning_rate": 6.1070286351567154e-06, "loss": 0.0197, "num_tokens": 22899748.0, "reward": 0.20937499403953552, "reward_std": 0.22137677669525146, "rewards/rollout_reward_func/mean": 0.20937499403953552, "rewards/rollout_reward_func/std": 0.22910678386688232, "sampling/importance_sampling_ratio/max": 1.2437137365341187, "sampling/importance_sampling_ratio/mean": 1.0290451049804688, "sampling/importance_sampling_ratio/min": 0.559307873249054, "sampling/sampling_logp_difference/max": 0.6596033573150635, "sampling/sampling_logp_difference/mean": 0.01587507128715515, "step": 479, "step_time": 14.803998386007152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010937500046566129, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010937500046566129, "completions/clipped_ratio": 0.0, "completions/max_length": 1854.0, "completions/max_terminated_length": 1854.0, "completions/mean_length": 369.75, "completions/mean_terminated_length": 369.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1282380442135036, "epoch": 0.0096, "frac_reward_zero_std": 0.0, "grad_norm": 0.8365519046783447, "kl": 1.4138909857720137, "learning_rate": 6.093419888818785e-06, "loss": 0.012, "num_tokens": 22950260.0, "reward": 0.03750000149011612, "reward_std": 0.33389192819595337, "rewards/rollout_reward_func/mean": 0.03750000149011612, "rewards/rollout_reward_func/std": 0.3328760266304016, "sampling/importance_sampling_ratio/max": 1.533352255821228, "sampling/importance_sampling_ratio/mean": 0.9801427125930786, "sampling/importance_sampling_ratio/min": 0.21602557599544525, "sampling/sampling_logp_difference/max": 1.28640878200531, "sampling/sampling_logp_difference/mean": 0.03835825249552727, "step": 480, "step_time": 16.621338822995313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 318.15625, "completions/mean_terminated_length": 318.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10990736330859363, "epoch": 0.00962, "frac_reward_zero_std": 0.0, "grad_norm": 0.36486878991127014, "kl": 0.49721127562224865, "learning_rate": 6.0798132078783714e-06, "loss": 0.0056, "num_tokens": 22999153.0, "reward": 0.23125000298023224, "reward_std": 0.21344618499279022, "rewards/rollout_reward_func/mean": 0.23125000298023224, "rewards/rollout_reward_func/std": 0.22206872701644897, "sampling/importance_sampling_ratio/max": 1.3971961736679077, "sampling/importance_sampling_ratio/mean": 1.0326569080352783, "sampling/importance_sampling_ratio/min": 0.8874096274375916, "sampling/sampling_logp_difference/max": 0.32297325134277344, "sampling/sampling_logp_difference/mean": 0.013650760054588318, "step": 481, "step_time": 14.038137431016366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 294.96875, "completions/mean_terminated_length": 294.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08086838247254491, "epoch": 0.00964, "frac_reward_zero_std": 0.0, "grad_norm": 0.2985641658306122, "kl": 1.3382384795695543, "learning_rate": 6.0662087718167915e-06, "loss": -0.0009, "num_tokens": 23047328.0, "reward": 0.13437500596046448, "reward_std": 0.28573501110076904, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.296875536441803, "sampling/importance_sampling_ratio/max": 1.155083417892456, "sampling/importance_sampling_ratio/mean": 0.9787537455558777, "sampling/importance_sampling_ratio/min": 0.4308132827281952, "sampling/sampling_logp_difference/max": 0.841937780380249, "sampling/sampling_logp_difference/mean": 0.025336986407637596, "step": 482, "step_time": 13.876530399007606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 359.3125, "completions/mean_terminated_length": 359.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12213777378201485, "epoch": 0.00966, "frac_reward_zero_std": 0.0, "grad_norm": 0.5516947507858276, "kl": 0.5931388307362795, "learning_rate": 6.0526067600857556e-06, "loss": -0.0158, "num_tokens": 23098786.0, "reward": 0.09062500298023224, "reward_std": 0.3531293272972107, "rewards/rollout_reward_func/mean": 0.09062500298023224, "rewards/rollout_reward_func/std": 0.3476737141609192, "sampling/importance_sampling_ratio/max": 1.4324049949645996, "sampling/importance_sampling_ratio/mean": 1.0263164043426514, "sampling/importance_sampling_ratio/min": 0.4962688386440277, "sampling/sampling_logp_difference/max": 0.9586195945739746, "sampling/sampling_logp_difference/mean": 0.023221395909786224, "step": 483, "step_time": 13.787378486991656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1688.0, "completions/max_terminated_length": 1688.0, "completions/mean_length": 403.34375, "completions/mean_terminated_length": 403.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10713796713389456, "epoch": 0.00968, "frac_reward_zero_std": 0.0, "grad_norm": 0.755982518196106, "kl": 2.627741226926446, "learning_rate": 6.039007352104992e-06, "loss": -0.0185, "num_tokens": 23151023.0, "reward": 0.13750000298023224, "reward_std": 0.36858731508255005, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.378238707780838, "sampling/importance_sampling_ratio/max": 1.676841139793396, "sampling/importance_sampling_ratio/mean": 1.0208988189697266, "sampling/importance_sampling_ratio/min": 0.4194191098213196, "sampling/sampling_logp_difference/max": 0.8689160346984863, "sampling/sampling_logp_difference/mean": 0.030806582421064377, "step": 484, "step_time": 16.56055976999778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 259.0, "completions/mean_terminated_length": 259.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09165397519245744, "epoch": 0.0097, "frac_reward_zero_std": 0.0, "grad_norm": 0.6476885080337524, "kl": 1.0840071067214012, "learning_rate": 6.025410727259885e-06, "loss": -0.0074, "num_tokens": 23199164.0, "reward": 0.17499999701976776, "reward_std": 0.2850794792175293, "rewards/rollout_reward_func/mean": 0.17499999701976776, "rewards/rollout_reward_func/std": 0.3069096803665161, "sampling/importance_sampling_ratio/max": 1.335154414176941, "sampling/importance_sampling_ratio/mean": 1.0241200923919678, "sampling/importance_sampling_ratio/min": 0.6101618409156799, "sampling/sampling_logp_difference/max": 0.49410438537597656, "sampling/sampling_logp_difference/mean": 0.014388521201908588, "step": 485, "step_time": 13.99984866300656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 394.40625, "completions/mean_terminated_length": 394.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0934990905225277, "epoch": 0.00972, "frac_reward_zero_std": 0.0, "grad_norm": 1.0598607063293457, "kl": 5.514862045645714, "learning_rate": 6.01181706489911e-06, "loss": 0.0285, "num_tokens": 23252986.0, "reward": 0.13437500596046448, "reward_std": 0.30439308285713196, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.30544453859329224, "sampling/importance_sampling_ratio/max": 1.8719347715377808, "sampling/importance_sampling_ratio/mean": 1.066078782081604, "sampling/importance_sampling_ratio/min": 0.8098617792129517, "sampling/sampling_logp_difference/max": 0.6235577464103699, "sampling/sampling_logp_difference/mean": 0.017591726034879684, "step": 486, "step_time": 13.78450261600301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 317.3125, "completions/mean_terminated_length": 317.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.093321418389678, "epoch": 0.00974, "frac_reward_zero_std": 0.0, "grad_norm": 0.3244117498397827, "kl": 0.5404838491231203, "learning_rate": 5.99822654433226e-06, "loss": 0.0367, "num_tokens": 23303403.0, "reward": 0.06562499701976776, "reward_std": 0.29501214623451233, "rewards/rollout_reward_func/mean": 0.06562499701976776, "rewards/rollout_reward_func/std": 0.2858031988143921, "sampling/importance_sampling_ratio/max": 1.61736261844635, "sampling/importance_sampling_ratio/mean": 1.0231678485870361, "sampling/importance_sampling_ratio/min": 0.40794116258621216, "sampling/sampling_logp_difference/max": 0.9024873971939087, "sampling/sampling_logp_difference/mean": 0.024090150371193886, "step": 487, "step_time": 14.728143533997354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 320.28125, "completions/mean_terminated_length": 320.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0723965554498136, "epoch": 0.00976, "frac_reward_zero_std": 0.0, "grad_norm": 1.1086945533752441, "kl": 0.5061975196003914, "learning_rate": 5.984639344827491e-06, "loss": 0.018, "num_tokens": 23353883.0, "reward": 0.16562500596046448, "reward_std": 0.24999596178531647, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.273105651140213, "sampling/importance_sampling_ratio/max": 1.6372661590576172, "sampling/importance_sampling_ratio/mean": 0.9932971000671387, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1101740598678589, "sampling/sampling_logp_difference/mean": 0.02685726433992386, "step": 488, "step_time": 13.967214939995756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 196.625, "completions/mean_terminated_length": 196.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06873580568935722, "epoch": 0.00978, "frac_reward_zero_std": 0.0, "grad_norm": 0.46438470482826233, "kl": 0.5748436264693737, "learning_rate": 5.971055645609147e-06, "loss": 0.0117, "num_tokens": 23399327.0, "reward": 0.18125000596046448, "reward_std": 0.22242116928100586, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.23886491358280182, "sampling/importance_sampling_ratio/max": 1.2301033735275269, "sampling/importance_sampling_ratio/mean": 0.997927725315094, "sampling/importance_sampling_ratio/min": 0.6367825865745544, "sampling/sampling_logp_difference/max": 0.46105217933654785, "sampling/sampling_logp_difference/mean": 0.016075383871793747, "step": 489, "step_time": 11.068696117996296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 197.90625, "completions/mean_terminated_length": 197.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08178244938608259, "epoch": 0.0098, "frac_reward_zero_std": 0.0, "grad_norm": 0.19371172785758972, "kl": 0.5731030106544495, "learning_rate": 5.957475625855404e-06, "loss": -0.0268, "num_tokens": 23444588.0, "reward": 0.11875000596046448, "reward_std": 0.2779819965362549, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.2740820348262787, "sampling/importance_sampling_ratio/max": 1.2093716859817505, "sampling/importance_sampling_ratio/mean": 1.0212191343307495, "sampling/importance_sampling_ratio/min": 0.9589686393737793, "sampling/sampling_logp_difference/max": 0.1902504563331604, "sampling/sampling_logp_difference/mean": 0.007413627579808235, "step": 490, "step_time": 13.838095408002118 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 291.1875, "completions/mean_terminated_length": 291.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09410754288546741, "epoch": 0.00982, "frac_reward_zero_std": 0.0, "grad_norm": 0.7472526431083679, "kl": 2.4004106298089027, "learning_rate": 5.943899464695905e-06, "loss": 0.0298, "num_tokens": 23493498.0, "reward": 0.10625000298023224, "reward_std": 0.2561027407646179, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.2601953446865082, "sampling/importance_sampling_ratio/max": 1.3085718154907227, "sampling/importance_sampling_ratio/mean": 0.9660689830780029, "sampling/importance_sampling_ratio/min": 0.4208488464355469, "sampling/sampling_logp_difference/max": 0.8742709159851074, "sampling/sampling_logp_difference/mean": 0.03556545823812485, "step": 491, "step_time": 14.64625643799809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 282.34375, "completions/mean_terminated_length": 282.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07253416115418077, "epoch": 0.00984, "frac_reward_zero_std": 0.0, "grad_norm": 0.4030205309391022, "kl": 1.903130080550909, "learning_rate": 5.930327341209392e-06, "loss": -0.0164, "num_tokens": 23542702.0, "reward": 0.265625, "reward_std": 0.21520861983299255, "rewards/rollout_reward_func/mean": 0.265625, "rewards/rollout_reward_func/std": 0.23638319969177246, "sampling/importance_sampling_ratio/max": 1.4576568603515625, "sampling/importance_sampling_ratio/mean": 1.0139055252075195, "sampling/importance_sampling_ratio/min": 0.6289598941802979, "sampling/sampling_logp_difference/max": 0.4741523265838623, "sampling/sampling_logp_difference/mean": 0.012105092406272888, "step": 492, "step_time": 12.272093624000263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1760.0, "completions/max_terminated_length": 1760.0, "completions/mean_length": 298.25, "completions/mean_terminated_length": 298.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09702993463724852, "epoch": 0.00986, "frac_reward_zero_std": 0.0, "grad_norm": 0.23152422904968262, "kl": 2.2016284596174955, "learning_rate": 5.916759434421345e-06, "loss": 0.0005, "num_tokens": 23591836.0, "reward": 0.11250000447034836, "reward_std": 0.3312201499938965, "rewards/rollout_reward_func/mean": 0.11250000447034836, "rewards/rollout_reward_func/std": 0.31902065873146057, "sampling/importance_sampling_ratio/max": 1.2300058603286743, "sampling/importance_sampling_ratio/mean": 0.9687206149101257, "sampling/importance_sampling_ratio/min": 0.39896759390830994, "sampling/sampling_logp_difference/max": 0.9294466972351074, "sampling/sampling_logp_difference/mean": 0.03413885086774826, "step": 493, "step_time": 16.671504221994837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 137.375, "completions/mean_terminated_length": 137.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05147601943463087, "epoch": 0.00988, "frac_reward_zero_std": 0.0, "grad_norm": 0.11500600725412369, "kl": 4.148211754858494, "learning_rate": 5.903195923301632e-06, "loss": 0.0173, "num_tokens": 23634178.0, "reward": 0.15937501192092896, "reward_std": 0.2627425193786621, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.25885525345802307, "sampling/importance_sampling_ratio/max": 1.4155139923095703, "sampling/importance_sampling_ratio/mean": 1.0315349102020264, "sampling/importance_sampling_ratio/min": 1.0013829469680786, "sampling/sampling_logp_difference/max": 0.33298254013061523, "sampling/sampling_logp_difference/mean": 0.00965790543705225, "step": 494, "step_time": 10.754944588996295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 296.6875, "completions/mean_terminated_length": 296.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08653213270008564, "epoch": 0.0099, "frac_reward_zero_std": 0.0, "grad_norm": 0.6059703230857849, "kl": 0.9344570655375719, "learning_rate": 5.88963698676213e-06, "loss": 0.0187, "num_tokens": 23683070.0, "reward": 0.11249999701976776, "reward_std": 0.28624165058135986, "rewards/rollout_reward_func/mean": 0.11249999701976776, "rewards/rollout_reward_func/std": 0.29154759645462036, "sampling/importance_sampling_ratio/max": 1.187873363494873, "sampling/importance_sampling_ratio/mean": 0.9499494433403015, "sampling/importance_sampling_ratio/min": 0.44329679012298584, "sampling/sampling_logp_difference/max": 0.8175995349884033, "sampling/sampling_logp_difference/mean": 0.030723705887794495, "step": 495, "step_time": 14.057592963996285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 263.8125, "completions/mean_terminated_length": 263.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0616281321272254, "epoch": 0.00992, "frac_reward_zero_std": 0.0, "grad_norm": 0.3454331159591675, "kl": 0.5093859154731035, "learning_rate": 5.87608280365438e-06, "loss": 0.0323, "num_tokens": 23729667.0, "reward": 0.15937501192092896, "reward_std": 0.2295302450656891, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.26743629574775696, "sampling/importance_sampling_ratio/max": 1.489280343055725, "sampling/importance_sampling_ratio/mean": 1.0526363849639893, "sampling/importance_sampling_ratio/min": 1.0021913051605225, "sampling/sampling_logp_difference/max": 0.39767080545425415, "sampling/sampling_logp_difference/mean": 0.01433565653860569, "step": 496, "step_time": 13.713001731986878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1886.0, "completions/max_terminated_length": 1886.0, "completions/mean_length": 333.09375, "completions/mean_terminated_length": 333.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10725827515125275, "epoch": 0.00994, "frac_reward_zero_std": 0.0, "grad_norm": 0.48821374773979187, "kl": 1.0927138440310955, "learning_rate": 5.862533552767218e-06, "loss": 0.0417, "num_tokens": 23780493.0, "reward": 0.06875000149011612, "reward_std": 0.26906293630599976, "rewards/rollout_reward_func/mean": 0.06875000149011612, "rewards/rollout_reward_func/std": 0.2833412289619446, "sampling/importance_sampling_ratio/max": 1.6097707748413086, "sampling/importance_sampling_ratio/mean": 1.0302157402038574, "sampling/importance_sampling_ratio/min": 0.6934543251991272, "sampling/sampling_logp_difference/max": 0.4760169982910156, "sampling/sampling_logp_difference/mean": 0.014149912633001804, "step": 497, "step_time": 16.523987749002117 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 883.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 237.21875, "completions/mean_terminated_length": 237.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07845151040237397, "epoch": 0.00996, "frac_reward_zero_std": 0.0, "grad_norm": 0.1861019730567932, "kl": 0.5722859278321266, "learning_rate": 5.848989412824425e-06, "loss": -0.021, "num_tokens": 23827891.0, "reward": 0.22187501192092896, "reward_std": 0.23901958763599396, "rewards/rollout_reward_func/mean": 0.22187501192092896, "rewards/rollout_reward_func/std": 0.2732532322406769, "sampling/importance_sampling_ratio/max": 1.3072633743286133, "sampling/importance_sampling_ratio/mean": 0.9969213008880615, "sampling/importance_sampling_ratio/min": 0.45116063952445984, "sampling/sampling_logp_difference/max": 0.8045740127563477, "sampling/sampling_logp_difference/mean": 0.022288663312792778, "step": 498, "step_time": 12.609626912995736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1841.0, "completions/max_terminated_length": 1841.0, "completions/mean_length": 279.0, "completions/mean_terminated_length": 279.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07184241118375212, "epoch": 0.00998, "frac_reward_zero_std": 0.0, "grad_norm": 0.1938573122024536, "kl": 0.48895006347447634, "learning_rate": 5.8354505624823585e-06, "loss": 0.0388, "num_tokens": 23875882.0, "reward": 0.12187501043081284, "reward_std": 0.3484642505645752, "rewards/rollout_reward_func/mean": 0.12187501043081284, "rewards/rollout_reward_func/std": 0.33957692980766296, "sampling/importance_sampling_ratio/max": 1.3230111598968506, "sampling/importance_sampling_ratio/mean": 0.9996103048324585, "sampling/importance_sampling_ratio/min": 0.29251569509506226, "sampling/sampling_logp_difference/max": 1.232386589050293, "sampling/sampling_logp_difference/mean": 0.022290877997875214, "step": 499, "step_time": 16.810782469005062 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 313.125, "completions/mean_terminated_length": 313.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10245605581440032, "epoch": 0.01, "frac_reward_zero_std": 0.0, "grad_norm": 0.351238489151001, "kl": 0.7380426004528999, "learning_rate": 5.821917180327612e-06, "loss": -0.0, "num_tokens": 23926249.0, "reward": 0.10000000894069672, "reward_std": 0.2928588390350342, "rewards/rollout_reward_func/mean": 0.10000000894069672, "rewards/rollout_reward_func/std": 0.2983827590942383, "sampling/importance_sampling_ratio/max": 1.359262228012085, "sampling/importance_sampling_ratio/mean": 1.0089294910430908, "sampling/importance_sampling_ratio/min": 0.6334314346313477, "sampling/sampling_logp_difference/max": 0.45659303665161133, "sampling/sampling_logp_difference/mean": 0.015445586293935776, "step": 500, "step_time": 13.614300954010105 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 312.40625, "completions/mean_terminated_length": 312.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06921436754055321, "epoch": 0.01002, "frac_reward_zero_std": 0.0, "grad_norm": 0.5836217999458313, "kl": 1.9639893509447575, "learning_rate": 5.808389444874641e-06, "loss": 0.0194, "num_tokens": 23975581.0, "reward": 0.16250000894069672, "reward_std": 0.26119521260261536, "rewards/rollout_reward_func/mean": 0.16250000894069672, "rewards/rollout_reward_func/std": 0.2697071433067322, "sampling/importance_sampling_ratio/max": 1.4580302238464355, "sampling/importance_sampling_ratio/mean": 1.0379921197891235, "sampling/importance_sampling_ratio/min": 0.9601997137069702, "sampling/sampling_logp_difference/max": 0.3770427703857422, "sampling/sampling_logp_difference/mean": 0.011733639054000378, "step": 501, "step_time": 13.927478268000414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 226.875, "completions/mean_terminated_length": 226.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05494062916841358, "epoch": 0.01004, "frac_reward_zero_std": 0.0, "grad_norm": 1.2452198266983032, "kl": 0.8351330403238535, "learning_rate": 5.794867534563422e-06, "loss": 0.0259, "num_tokens": 24020937.0, "reward": 0.21250000596046448, "reward_std": 0.212186798453331, "rewards/rollout_reward_func/mean": 0.21250000596046448, "rewards/rollout_reward_func/std": 0.2196037471294403, "sampling/importance_sampling_ratio/max": 1.7712397575378418, "sampling/importance_sampling_ratio/mean": 1.012845516204834, "sampling/importance_sampling_ratio/min": 0.6100878715515137, "sampling/sampling_logp_difference/max": 0.5651187896728516, "sampling/sampling_logp_difference/mean": 0.020326213911175728, "step": 502, "step_time": 13.746397457987769 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 218.40625, "completions/mean_terminated_length": 218.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09551530191674829, "epoch": 0.01006, "frac_reward_zero_std": 0.0, "grad_norm": 0.7224723696708679, "kl": 3.001770196482539, "learning_rate": 5.781351627757093e-06, "loss": 0.0085, "num_tokens": 24067831.0, "reward": 0.15937499701976776, "reward_std": 0.22978875041007996, "rewards/rollout_reward_func/mean": 0.15937499701976776, "rewards/rollout_reward_func/std": 0.22840170562267303, "sampling/importance_sampling_ratio/max": 1.1280871629714966, "sampling/importance_sampling_ratio/mean": 1.0022313594818115, "sampling/importance_sampling_ratio/min": 0.6614378094673157, "sampling/sampling_logp_difference/max": 0.4173934757709503, "sampling/sampling_logp_difference/mean": 0.011569207534193993, "step": 503, "step_time": 13.41809406500397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 263.25, "completions/mean_terminated_length": 263.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0759890649933368, "epoch": 0.01008, "frac_reward_zero_std": 0.0, "grad_norm": 0.6227708458900452, "kl": 0.4967890549451113, "learning_rate": 5.767841902739602e-06, "loss": 0.0477, "num_tokens": 24114685.0, "reward": 0.08437500149011612, "reward_std": 0.25884634256362915, "rewards/rollout_reward_func/mean": 0.08437500149011612, "rewards/rollout_reward_func/std": 0.27013662457466125, "sampling/importance_sampling_ratio/max": 1.3917882442474365, "sampling/importance_sampling_ratio/mean": 1.0151293277740479, "sampling/importance_sampling_ratio/min": 0.7631263732910156, "sampling/sampling_logp_difference/max": 0.32296305894851685, "sampling/sampling_logp_difference/mean": 0.01035163551568985, "step": 504, "step_time": 14.126724130001094 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 388.96875, "completions/mean_terminated_length": 388.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12094950675964355, "epoch": 0.0101, "frac_reward_zero_std": 0.0, "grad_norm": 0.5646456480026245, "kl": 0.47913349512964487, "learning_rate": 5.754338537713353e-06, "loss": 0.007, "num_tokens": 24167584.0, "reward": 0.140625, "reward_std": 0.3076016902923584, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.31093963980674744, "sampling/importance_sampling_ratio/max": 1.242651104927063, "sampling/importance_sampling_ratio/mean": 0.9967185258865356, "sampling/importance_sampling_ratio/min": 0.602006196975708, "sampling/sampling_logp_difference/max": 0.5112996101379395, "sampling/sampling_logp_difference/mean": 0.01836981624364853, "step": 505, "step_time": 14.839693530004297 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 199.3125, "completions/mean_terminated_length": 199.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.053907602676190436, "epoch": 0.01012, "frac_reward_zero_std": 0.0, "grad_norm": 0.10055962204933167, "kl": 1.4545675870031118, "learning_rate": 5.740841710796861e-06, "loss": 0.0219, "num_tokens": 24212558.0, "reward": 0.15937501192092896, "reward_std": 0.2084997445344925, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.22698497772216797, "sampling/importance_sampling_ratio/max": 1.2046581506729126, "sampling/importance_sampling_ratio/mean": 0.9976238012313843, "sampling/importance_sampling_ratio/min": 0.5112394690513611, "sampling/sampling_logp_difference/max": 0.7585868835449219, "sampling/sampling_logp_difference/mean": 0.012229438871145248, "step": 506, "step_time": 12.165966427994135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 293.875, "completions/mean_terminated_length": 293.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08386114647146314, "epoch": 0.01014, "frac_reward_zero_std": 0.0, "grad_norm": 0.7101455330848694, "kl": 1.0825207084417343, "learning_rate": 5.727351600022396e-06, "loss": 0.0109, "num_tokens": 24260963.0, "reward": 0.13125000894069672, "reward_std": 0.24486398696899414, "rewards/rollout_reward_func/mean": 0.13125000894069672, "rewards/rollout_reward_func/std": 0.25832900404930115, "sampling/importance_sampling_ratio/max": 1.1160215139389038, "sampling/importance_sampling_ratio/mean": 0.948466420173645, "sampling/importance_sampling_ratio/min": 0.4241492748260498, "sampling/sampling_logp_difference/max": 0.8665814399719238, "sampling/sampling_logp_difference/mean": 0.026261016726493835, "step": 507, "step_time": 14.566642087007494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 334.53125, "completions/mean_terminated_length": 334.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16824141202960163, "epoch": 0.01016, "frac_reward_zero_std": 0.0, "grad_norm": 1.0565555095672607, "kl": 0.5156540973111987, "learning_rate": 5.7138683833336385e-06, "loss": 0.0347, "num_tokens": 24312110.0, "reward": 0.08437500149011612, "reward_std": 0.2515203058719635, "rewards/rollout_reward_func/mean": 0.08437500149011612, "rewards/rollout_reward_func/std": 0.27013659477233887, "sampling/importance_sampling_ratio/max": 1.8821616172790527, "sampling/importance_sampling_ratio/mean": 1.0060491561889648, "sampling/importance_sampling_ratio/min": 0.2910092771053314, "sampling/sampling_logp_difference/max": 1.2405784130096436, "sampling/sampling_logp_difference/mean": 0.03972402215003967, "step": 508, "step_time": 14.894589790008467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 302.96875, "completions/mean_terminated_length": 302.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09951365017332137, "epoch": 0.01018, "frac_reward_zero_std": 0.0, "grad_norm": 0.5911460518836975, "kl": 0.45311727467924356, "learning_rate": 5.70039223858333e-06, "loss": -0.0182, "num_tokens": 24362715.0, "reward": 0.15000000596046448, "reward_std": 0.2704285979270935, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.26518404483795166, "sampling/importance_sampling_ratio/max": 1.405975341796875, "sampling/importance_sampling_ratio/mean": 1.0048363208770752, "sampling/importance_sampling_ratio/min": 0.47949329018592834, "sampling/sampling_logp_difference/max": 0.7350685596466064, "sampling/sampling_logp_difference/mean": 0.022556636482477188, "step": 509, "step_time": 13.846522058011033 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 331.21875, "completions/mean_terminated_length": 331.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07974978780839592, "epoch": 0.0102, "frac_reward_zero_std": 0.0, "grad_norm": 0.3472816050052643, "kl": 0.5347178820520639, "learning_rate": 5.686923343530932e-06, "loss": 0.0112, "num_tokens": 24413483.0, "reward": 0.14999999105930328, "reward_std": 0.25629621744155884, "rewards/rollout_reward_func/mean": 0.14999999105930328, "rewards/rollout_reward_func/std": 0.27591490745544434, "sampling/importance_sampling_ratio/max": 1.342223882675171, "sampling/importance_sampling_ratio/mean": 0.994388222694397, "sampling/importance_sampling_ratio/min": 0.5121868252754211, "sampling/sampling_logp_difference/max": 0.6689876914024353, "sampling/sampling_logp_difference/mean": 0.020650846883654594, "step": 510, "step_time": 14.301617482011352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 317.34375, "completions/mean_terminated_length": 317.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11001799255609512, "epoch": 0.01022, "frac_reward_zero_std": 0.0, "grad_norm": 0.479348748922348, "kl": 0.5839377585798502, "learning_rate": 5.673461875840275e-06, "loss": -0.0252, "num_tokens": 24463517.0, "reward": 0.19687500596046448, "reward_std": 0.2791457772254944, "rewards/rollout_reward_func/mean": 0.19687500596046448, "rewards/rollout_reward_func/std": 0.272957980632782, "sampling/importance_sampling_ratio/max": 1.1671077013015747, "sampling/importance_sampling_ratio/mean": 0.9992051124572754, "sampling/importance_sampling_ratio/min": 0.6480128169059753, "sampling/sampling_logp_difference/max": 0.433870792388916, "sampling/sampling_logp_difference/mean": 0.015219608321785927, "step": 511, "step_time": 13.706512741013285 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 423.53125, "completions/mean_terminated_length": 423.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10014648525975645, "epoch": 0.01024, "frac_reward_zero_std": 0.0, "grad_norm": 0.506767213344574, "kl": 0.6200972069054842, "learning_rate": 5.6600080130772166e-06, "loss": 0.0283, "num_tokens": 24517486.0, "reward": 0.08124999701976776, "reward_std": 0.2947782874107361, "rewards/rollout_reward_func/mean": 0.08124999701976776, "rewards/rollout_reward_func/std": 0.30100637674331665, "sampling/importance_sampling_ratio/max": 1.142641544342041, "sampling/importance_sampling_ratio/mean": 0.9848167896270752, "sampling/importance_sampling_ratio/min": 0.3990115225315094, "sampling/sampling_logp_difference/max": 0.9262475967407227, "sampling/sampling_logp_difference/mean": 0.019398696720600128, "step": 512, "step_time": 14.68615049300206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 333.5, "completions/mean_terminated_length": 333.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05758908519055694, "epoch": 0.01026, "frac_reward_zero_std": 0.0, "grad_norm": 0.18121474981307983, "kl": 0.6889986898750067, "learning_rate": 5.646561932707302e-06, "loss": -0.001, "num_tokens": 24567165.0, "reward": 0.21875, "reward_std": 0.2655576169490814, "rewards/rollout_reward_func/mean": 0.21875, "rewards/rollout_reward_func/std": 0.27759045362472534, "sampling/importance_sampling_ratio/max": 1.1420023441314697, "sampling/importance_sampling_ratio/mean": 1.0038806200027466, "sampling/importance_sampling_ratio/min": 0.6094009280204773, "sampling/sampling_logp_difference/max": 0.49633026123046875, "sampling/sampling_logp_difference/mean": 0.0099113117903471, "step": 513, "step_time": 14.650081065992708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 257.625, "completions/mean_terminated_length": 257.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06284393661189824, "epoch": 0.01028, "frac_reward_zero_std": 0.0, "grad_norm": 0.6173299551010132, "kl": 0.5435734670609236, "learning_rate": 5.633123812093419e-06, "loss": 0.0211, "num_tokens": 24614365.0, "reward": 0.13437500596046448, "reward_std": 0.2839360237121582, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.296875536441803, "sampling/importance_sampling_ratio/max": 1.2634027004241943, "sampling/importance_sampling_ratio/mean": 1.03254234790802, "sampling/importance_sampling_ratio/min": 1.001372218132019, "sampling/sampling_logp_difference/max": 0.23208868503570557, "sampling/sampling_logp_difference/mean": 0.009521976113319397, "step": 514, "step_time": 13.583202819001599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 234.0, "completions/mean_terminated_length": 234.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0763992303982377, "epoch": 0.0103, "frac_reward_zero_std": 0.0, "grad_norm": 0.3011552393436432, "kl": 0.5513538299128413, "learning_rate": 5.619693828493467e-06, "loss": 0.0411, "num_tokens": 24659878.0, "reward": 0.140625, "reward_std": 0.23432418704032898, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.23535749316215515, "sampling/importance_sampling_ratio/max": 1.2759181261062622, "sampling/importance_sampling_ratio/mean": 1.0159283876419067, "sampling/importance_sampling_ratio/min": 0.7268650531768799, "sampling/sampling_logp_difference/max": 0.3190317153930664, "sampling/sampling_logp_difference/mean": 0.01013573445379734, "step": 515, "step_time": 13.989564292001887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 234.875, "completions/mean_terminated_length": 241.9354705810547, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1784049030393362, "epoch": 0.01032, "frac_reward_zero_std": 0.0, "grad_norm": 1.316250205039978, "kl": 3.0037334840744734, "learning_rate": 5.606272159058005e-06, "loss": -0.0057, "num_tokens": 24706722.0, "reward": 0.20625001192092896, "reward_std": 0.2699100375175476, "rewards/rollout_reward_func/mean": 0.20625001192092896, "rewards/rollout_reward_func/std": 0.26993128657341003, "sampling/importance_sampling_ratio/max": 1.2415910959243774, "sampling/importance_sampling_ratio/mean": 0.9861487150192261, "sampling/importance_sampling_ratio/min": 0.2906503677368164, "sampling/sampling_logp_difference/max": 1.3639168739318848, "sampling/sampling_logp_difference/mean": 0.025325950235128403, "step": 516, "step_time": 13.85695915100223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 222.09375, "completions/mean_terminated_length": 222.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.054965540184639394, "epoch": 0.01034, "frac_reward_zero_std": 0.0, "grad_norm": 0.5393273234367371, "kl": 4.722749901935458, "learning_rate": 5.5928589808279266e-06, "loss": 0.0274, "num_tokens": 24751588.0, "reward": 0.11875000596046448, "reward_std": 0.26516440510749817, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.26692694425582886, "sampling/importance_sampling_ratio/max": 1.0647106170654297, "sampling/importance_sampling_ratio/mean": 0.957587718963623, "sampling/importance_sampling_ratio/min": 0.37035319209098816, "sampling/sampling_logp_difference/max": 0.9933185577392578, "sampling/sampling_logp_difference/mean": 0.025261713191866875, "step": 517, "step_time": 13.63833519200125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 251.84375, "completions/mean_terminated_length": 251.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06588097719941288, "epoch": 0.01036, "frac_reward_zero_std": 0.0, "grad_norm": 0.23595473170280457, "kl": 3.9518870878964663, "learning_rate": 5.5794544707321184e-06, "loss": 0.0187, "num_tokens": 24798949.0, "reward": 0.18125000596046448, "reward_std": 0.23798570036888123, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.23886491358280182, "sampling/importance_sampling_ratio/max": 1.1820229291915894, "sampling/importance_sampling_ratio/mean": 0.9795506596565247, "sampling/importance_sampling_ratio/min": 0.4919656217098236, "sampling/sampling_logp_difference/max": 0.7094058990478516, "sampling/sampling_logp_difference/mean": 0.02265031635761261, "step": 518, "step_time": 14.105185108004662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 847.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 192.5625, "completions/mean_terminated_length": 192.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05648258049041033, "epoch": 0.01038, "frac_reward_zero_std": 0.0, "grad_norm": 0.051364194601774216, "kl": 0.5090538281947374, "learning_rate": 5.566058805585135e-06, "loss": -0.0071, "num_tokens": 24843845.0, "reward": 0.20624999701976776, "reward_std": 0.22715778648853302, "rewards/rollout_reward_func/mean": 0.20624999701976776, "rewards/rollout_reward_func/std": 0.22991932928562164, "sampling/importance_sampling_ratio/max": 1.1852649450302124, "sampling/importance_sampling_ratio/mean": 1.0038050413131714, "sampling/importance_sampling_ratio/min": 0.593702495098114, "sampling/sampling_logp_difference/max": 0.5254641771316528, "sampling/sampling_logp_difference/mean": 0.010847882367670536, "step": 519, "step_time": 12.356510243997036 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 292.1875, "completions/mean_terminated_length": 292.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08772244513966143, "epoch": 0.0104, "frac_reward_zero_std": 0.0, "grad_norm": 0.4563632011413574, "kl": 0.6335414741188288, "learning_rate": 5.55267216208485e-06, "loss": -0.0275, "num_tokens": 24893448.0, "reward": 0.18125000596046448, "reward_std": 0.27268868684768677, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.27171793580055237, "sampling/importance_sampling_ratio/max": 1.2063995599746704, "sampling/importance_sampling_ratio/mean": 0.9430111646652222, "sampling/importance_sampling_ratio/min": 0.45181089639663696, "sampling/sampling_logp_difference/max": 0.794600248336792, "sampling/sampling_logp_difference/mean": 0.03367552533745766, "step": 520, "step_time": 13.680047078996722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 288.9375, "completions/mean_terminated_length": 288.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07737960992380977, "epoch": 0.01042, "frac_reward_zero_std": 0.0, "grad_norm": 0.7561383843421936, "kl": 4.80039684753865, "learning_rate": 5.539294716810144e-06, "loss": 0.0003, "num_tokens": 24942330.0, "reward": 0.203125, "reward_std": 0.22095540165901184, "rewards/rollout_reward_func/mean": 0.203125, "rewards/rollout_reward_func/std": 0.22502240538597107, "sampling/importance_sampling_ratio/max": 1.0932520627975464, "sampling/importance_sampling_ratio/mean": 0.9717237949371338, "sampling/importance_sampling_ratio/min": 0.37662220001220703, "sampling/sampling_logp_difference/max": 0.9765251874923706, "sampling/sampling_logp_difference/mean": 0.02297203429043293, "step": 521, "step_time": 14.127686774998438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 269.71875, "completions/mean_terminated_length": 269.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.04637193738017231, "epoch": 0.01044, "frac_reward_zero_std": 0.0, "grad_norm": 0.04275986924767494, "kl": 0.5766235999763012, "learning_rate": 5.525926646218561e-06, "loss": 0.0262, "num_tokens": 24988585.0, "reward": 0.2562500238418579, "reward_std": 0.1906895935535431, "rewards/rollout_reward_func/mean": 0.2562500238418579, "rewards/rollout_reward_func/std": 0.20150642096996307, "sampling/importance_sampling_ratio/max": 1.1200188398361206, "sampling/importance_sampling_ratio/mean": 1.0180848836898804, "sampling/importance_sampling_ratio/min": 1.0010865926742554, "sampling/sampling_logp_difference/max": 0.1013554185628891, "sampling/sampling_logp_difference/mean": 0.005421079695224762, "step": 522, "step_time": 14.020241602993337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 367.90625, "completions/mean_terminated_length": 367.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07472604350186884, "epoch": 0.01046, "frac_reward_zero_std": 0.0, "grad_norm": 0.4021729528903961, "kl": 0.5013142861425877, "learning_rate": 5.512568126643991e-06, "loss": 0.0262, "num_tokens": 25040757.0, "reward": 0.12187500298023224, "reward_std": 0.2610330581665039, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.2696884572505951, "sampling/importance_sampling_ratio/max": 1.1430732011795044, "sampling/importance_sampling_ratio/mean": 0.9920177459716797, "sampling/importance_sampling_ratio/min": 0.6534340977668762, "sampling/sampling_logp_difference/max": 0.4453604221343994, "sampling/sampling_logp_difference/mean": 0.011126468889415264, "step": 523, "step_time": 14.426032741990639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 297.96875, "completions/mean_terminated_length": 297.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07262235973030329, "epoch": 0.01048, "frac_reward_zero_std": 0.0, "grad_norm": 0.13354156911373138, "kl": 0.4244502820074558, "learning_rate": 5.499219334294335e-06, "loss": 0.041, "num_tokens": 25088644.0, "reward": 0.12812501192092896, "reward_std": 0.2583432197570801, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.26668137311935425, "sampling/importance_sampling_ratio/max": 1.1560724973678589, "sampling/importance_sampling_ratio/mean": 1.0279359817504883, "sampling/importance_sampling_ratio/min": 1.0022841691970825, "sampling/sampling_logp_difference/max": 0.14505121111869812, "sampling/sampling_logp_difference/mean": 0.00845939852297306, "step": 524, "step_time": 15.452798395002901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 311.4375, "completions/mean_terminated_length": 311.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06970260431990027, "epoch": 0.0105, "frac_reward_zero_std": 0.0, "grad_norm": 0.19668123126029968, "kl": 0.5733298780396581, "learning_rate": 5.485880445249192e-06, "loss": 0.0365, "num_tokens": 25137807.0, "reward": 0.12812501192092896, "reward_std": 0.29132288694381714, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.29097652435302734, "sampling/importance_sampling_ratio/max": 1.0426888465881348, "sampling/importance_sampling_ratio/mean": 1.0095645189285278, "sampling/importance_sampling_ratio/min": 0.9669487476348877, "sampling/sampling_logp_difference/max": 0.03639712929725647, "sampling/sampling_logp_difference/mean": 0.0038251662626862526, "step": 525, "step_time": 14.160736496003665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 441.96875, "completions/mean_terminated_length": 441.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11090936465188861, "epoch": 0.01052, "frac_reward_zero_std": 0.0, "grad_norm": 0.4545576870441437, "kl": 1.7043701047077775, "learning_rate": 5.472551635457521e-06, "loss": 0.0079, "num_tokens": 25191871.0, "reward": 0.0625000074505806, "reward_std": 0.27312415838241577, "rewards/rollout_reward_func/mean": 0.0625000074505806, "rewards/rollout_reward_func/std": 0.2991924583911896, "sampling/importance_sampling_ratio/max": 1.133989930152893, "sampling/importance_sampling_ratio/mean": 0.9704577326774597, "sampling/importance_sampling_ratio/min": 0.3316822350025177, "sampling/sampling_logp_difference/max": 1.1036384105682373, "sampling/sampling_logp_difference/mean": 0.02963452786207199, "step": 526, "step_time": 15.457800717005739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 192.15625, "completions/mean_terminated_length": 192.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11261358601041138, "epoch": 0.01054, "frac_reward_zero_std": 0.0, "grad_norm": 0.1590682864189148, "kl": 0.8699741382151842, "learning_rate": 5.459233080735336e-06, "loss": -0.0049, "num_tokens": 25236767.0, "reward": 0.12812501192092896, "reward_std": 0.3529778718948364, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.34476205706596375, "sampling/importance_sampling_ratio/max": 1.1835789680480957, "sampling/importance_sampling_ratio/mean": 1.0031601190567017, "sampling/importance_sampling_ratio/min": 0.5088897943496704, "sampling/sampling_logp_difference/max": 0.6816649436950684, "sampling/sampling_logp_difference/mean": 0.01961863972246647, "step": 527, "step_time": 14.582808170995122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 313.78125, "completions/mean_terminated_length": 313.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07135850004851818, "epoch": 0.01056, "frac_reward_zero_std": 0.0, "grad_norm": 0.27777063846588135, "kl": 4.6806051805615425, "learning_rate": 5.4459249567633776e-06, "loss": 0.0168, "num_tokens": 25286036.0, "reward": 0.18125000596046448, "reward_std": 0.281713604927063, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.3052629828453064, "sampling/importance_sampling_ratio/max": 1.2897746562957764, "sampling/importance_sampling_ratio/mean": 0.9981662631034851, "sampling/importance_sampling_ratio/min": 0.37114790081977844, "sampling/sampling_logp_difference/max": 0.9912137985229492, "sampling/sampling_logp_difference/mean": 0.02011878602206707, "step": 528, "step_time": 14.840291281001555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 249.75, "completions/mean_terminated_length": 249.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10197544447146356, "epoch": 0.01058, "frac_reward_zero_std": 0.0, "grad_norm": 0.12979839742183685, "kl": 0.4512835508212447, "learning_rate": 5.432627439084797e-06, "loss": 0.0039, "num_tokens": 25333290.0, "reward": 0.19374999403953552, "reward_std": 0.2309408336877823, "rewards/rollout_reward_func/mean": 0.19374999403953552, "rewards/rollout_reward_func/std": 0.23409055173397064, "sampling/importance_sampling_ratio/max": 1.2621006965637207, "sampling/importance_sampling_ratio/mean": 1.0186176300048828, "sampling/importance_sampling_ratio/min": 0.6457576155662537, "sampling/sampling_logp_difference/max": 0.43745458126068115, "sampling/sampling_logp_difference/mean": 0.01331893540918827, "step": 529, "step_time": 14.132924642995931 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 293.84375, "completions/mean_terminated_length": 293.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09837014391086996, "epoch": 0.0106, "frac_reward_zero_std": 0.0, "grad_norm": 0.3110145330429077, "kl": 0.5833736192435026, "learning_rate": 5.419340703102839e-06, "loss": 0.0267, "num_tokens": 25381451.0, "reward": 0.1875, "reward_std": 0.17670613527297974, "rewards/rollout_reward_func/mean": 0.1875, "rewards/rollout_reward_func/std": 0.18094731867313385, "sampling/importance_sampling_ratio/max": 1.1598007678985596, "sampling/importance_sampling_ratio/mean": 1.020216941833496, "sampling/importance_sampling_ratio/min": 0.8421584367752075, "sampling/sampling_logp_difference/max": 0.1795351505279541, "sampling/sampling_logp_difference/mean": 0.010587228462100029, "step": 530, "step_time": 14.878094225990935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 318.03125, "completions/mean_terminated_length": 318.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11072496883571148, "epoch": 0.01062, "frac_reward_zero_std": 0.0, "grad_norm": 0.33359912037849426, "kl": 0.46540657617151737, "learning_rate": 5.406064924078536e-06, "loss": 0.0094, "num_tokens": 25431025.0, "reward": 0.20624999701976776, "reward_std": 0.23426553606987, "rewards/rollout_reward_func/mean": 0.20624999701976776, "rewards/rollout_reward_func/std": 0.25895261764526367, "sampling/importance_sampling_ratio/max": 1.230563998222351, "sampling/importance_sampling_ratio/mean": 1.005523443222046, "sampling/importance_sampling_ratio/min": 0.5624719858169556, "sampling/sampling_logp_difference/max": 0.5753799080848694, "sampling/sampling_logp_difference/mean": 0.0168559942394495, "step": 531, "step_time": 13.827621480999369 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 210.84375, "completions/mean_terminated_length": 210.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09423909639008343, "epoch": 0.01064, "frac_reward_zero_std": 0.0, "grad_norm": 0.1916363537311554, "kl": 1.1828741785138845, "learning_rate": 5.392800277128386e-06, "loss": 0.0082, "num_tokens": 25475646.0, "reward": 0.20625001192092896, "reward_std": 0.23701566457748413, "rewards/rollout_reward_func/mean": 0.20625001192092896, "rewards/rollout_reward_func/std": 0.25518810749053955, "sampling/importance_sampling_ratio/max": 1.180074691772461, "sampling/importance_sampling_ratio/mean": 1.0021467208862305, "sampling/importance_sampling_ratio/min": 0.3309255838394165, "sampling/sampling_logp_difference/max": 1.1103668212890625, "sampling/sampling_logp_difference/mean": 0.01844189688563347, "step": 532, "step_time": 13.840607297988754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 317.75, "completions/mean_terminated_length": 317.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12665933393873274, "epoch": 0.01066, "frac_reward_zero_std": 0.0, "grad_norm": 0.5830035209655762, "kl": 1.8490807805210352, "learning_rate": 5.3795469372220525e-06, "loss": 0.0016, "num_tokens": 25525441.0, "reward": 0.10000000149011612, "reward_std": 0.26336991786956787, "rewards/rollout_reward_func/mean": 0.10000000149011612, "rewards/rollout_reward_func/std": 0.27708157896995544, "sampling/importance_sampling_ratio/max": 1.1722313165664673, "sampling/importance_sampling_ratio/mean": 0.9448956251144409, "sampling/importance_sampling_ratio/min": 0.3899911046028137, "sampling/sampling_logp_difference/max": 0.9928412437438965, "sampling/sampling_logp_difference/mean": 0.04011809080839157, "step": 533, "step_time": 14.215620773004048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1814.0, "completions/max_terminated_length": 1814.0, "completions/mean_length": 358.78125, "completions/mean_terminated_length": 358.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10716785956174135, "epoch": 0.01068, "frac_reward_zero_std": 0.0, "grad_norm": 0.3809496760368347, "kl": 1.852969678118825, "learning_rate": 5.366305079180043e-06, "loss": 0.0076, "num_tokens": 25576999.0, "reward": 0.10625000298023224, "reward_std": 0.3138108551502228, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.3078934848308563, "sampling/importance_sampling_ratio/max": 1.1353957653045654, "sampling/importance_sampling_ratio/mean": 0.9891811013221741, "sampling/importance_sampling_ratio/min": 0.33337289094924927, "sampling/sampling_logp_difference/max": 1.11220121383667, "sampling/sampling_logp_difference/mean": 0.02329554408788681, "step": 534, "step_time": 16.43757207800445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1409.0, "completions/max_terminated_length": 1409.0, "completions/mean_length": 341.875, "completions/mean_terminated_length": 341.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12387669598683715, "epoch": 0.0107, "frac_reward_zero_std": 0.0, "grad_norm": 1.1252131462097168, "kl": 0.7120672203600407, "learning_rate": 5.35307487767142e-06, "loss": -0.0138, "num_tokens": 25628264.0, "reward": 0.16562500596046448, "reward_std": 0.2523258328437805, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.253503680229187, "sampling/importance_sampling_ratio/max": 1.4957503080368042, "sampling/importance_sampling_ratio/mean": 1.001678228378296, "sampling/importance_sampling_ratio/min": 0.37796750664711, "sampling/sampling_logp_difference/max": 0.9729140996932983, "sampling/sampling_logp_difference/mean": 0.031367868185043335, "step": 535, "step_time": 16.01456483199945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 354.8125, "completions/mean_terminated_length": 354.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08778148959390819, "epoch": 0.01072, "frac_reward_zero_std": 0.0, "grad_norm": 0.35500484704971313, "kl": 1.0265771076083183, "learning_rate": 5.339856507211485e-06, "loss": -0.0006, "num_tokens": 25680414.0, "reward": 0.12812501192092896, "reward_std": 0.2692607641220093, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.31544578075408936, "sampling/importance_sampling_ratio/max": 1.2724424600601196, "sampling/importance_sampling_ratio/mean": 0.9851064682006836, "sampling/importance_sampling_ratio/min": 0.4670916497707367, "sampling/sampling_logp_difference/max": 0.7700140476226807, "sampling/sampling_logp_difference/mean": 0.019698407500982285, "step": 536, "step_time": 14.168460356988362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 180.1875, "completions/mean_terminated_length": 180.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10840532183647156, "epoch": 0.01074, "frac_reward_zero_std": 0.0, "grad_norm": 0.6040395498275757, "kl": 0.9689740780740976, "learning_rate": 5.326650142159479e-06, "loss": -0.0001, "num_tokens": 25724148.0, "reward": 0.18125000596046448, "reward_std": 0.2667011618614197, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.2822004556655884, "sampling/importance_sampling_ratio/max": 1.2379992008209229, "sampling/importance_sampling_ratio/mean": 0.9636602401733398, "sampling/importance_sampling_ratio/min": 0.42379122972488403, "sampling/sampling_logp_difference/max": 0.8628458976745605, "sampling/sampling_logp_difference/mean": 0.04293285682797432, "step": 537, "step_time": 11.027795375004644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 381.375, "completions/mean_terminated_length": 381.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10570096410810947, "epoch": 0.01076, "frac_reward_zero_std": 0.0, "grad_norm": 0.5110510587692261, "kl": 0.8100769575685263, "learning_rate": 5.313455956716286e-06, "loss": 0.0018, "num_tokens": 25776678.0, "reward": 0.10625000298023224, "reward_std": 0.31383007764816284, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.3262074291706085, "sampling/importance_sampling_ratio/max": 1.457227349281311, "sampling/importance_sampling_ratio/mean": 1.0157634019851685, "sampling/importance_sampling_ratio/min": 0.5826101303100586, "sampling/sampling_logp_difference/max": 0.6068991422653198, "sampling/sampling_logp_difference/mean": 0.020729873329401016, "step": 538, "step_time": 14.231675604005432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 247.09375, "completions/mean_terminated_length": 247.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11004820675589144, "epoch": 0.01078, "frac_reward_zero_std": 0.0, "grad_norm": 0.38873061537742615, "kl": 2.195252390578389, "learning_rate": 5.3002741249221305e-06, "loss": 0.0006, "num_tokens": 25823495.0, "reward": 0.13437500596046448, "reward_std": 0.2516390085220337, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.273105651140213, "sampling/importance_sampling_ratio/max": 1.3594988584518433, "sampling/importance_sampling_ratio/mean": 1.0063003301620483, "sampling/importance_sampling_ratio/min": 0.6118748188018799, "sampling/sampling_logp_difference/max": 0.49126625061035156, "sampling/sampling_logp_difference/mean": 0.02208131179213524, "step": 539, "step_time": 14.528189710013976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 288.9375, "completions/mean_terminated_length": 288.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1089557665400207, "epoch": 0.0108, "frac_reward_zero_std": 0.0, "grad_norm": 0.39840757846832275, "kl": 0.6094961892813444, "learning_rate": 5.287104820654283e-06, "loss": -0.0233, "num_tokens": 25872440.0, "reward": 0.17500001192092896, "reward_std": 0.2679120898246765, "rewards/rollout_reward_func/mean": 0.17500001192092896, "rewards/rollout_reward_func/std": 0.2663976848125458, "sampling/importance_sampling_ratio/max": 1.103212833404541, "sampling/importance_sampling_ratio/mean": 0.9636791944503784, "sampling/importance_sampling_ratio/min": 0.34413033723831177, "sampling/sampling_logp_difference/max": 1.0668116807937622, "sampling/sampling_logp_difference/mean": 0.028545156121253967, "step": 540, "step_time": 13.77037130800818 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014062500093132257, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 240.71875, "completions/mean_terminated_length": 240.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12379762181080878, "epoch": 0.01082, "frac_reward_zero_std": 0.0, "grad_norm": 0.5939421057701111, "kl": 1.8593947384506464, "learning_rate": 5.273948217624771e-06, "loss": 0.0136, "num_tokens": 25918919.0, "reward": 0.14375001192092896, "reward_std": 0.30153024196624756, "rewards/rollout_reward_func/mean": 0.14375001192092896, "rewards/rollout_reward_func/std": 0.30684396624565125, "sampling/importance_sampling_ratio/max": 1.162719964981079, "sampling/importance_sampling_ratio/mean": 1.0054457187652588, "sampling/importance_sampling_ratio/min": 0.5405802726745605, "sampling/sampling_logp_difference/max": 0.6194214820861816, "sampling/sampling_logp_difference/mean": 0.019232764840126038, "step": 541, "step_time": 14.614917141992919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1751.0, "completions/max_terminated_length": 1751.0, "completions/mean_length": 406.53125, "completions/mean_terminated_length": 406.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09619038086384535, "epoch": 0.01084, "frac_reward_zero_std": 0.0, "grad_norm": 0.6205554008483887, "kl": 2.158434689976275, "learning_rate": 5.260804489378083e-06, "loss": 0.0534, "num_tokens": 25971137.0, "reward": 0.16249999403953552, "reward_std": 0.2331787645816803, "rewards/rollout_reward_func/mean": 0.16249999403953552, "rewards/rollout_reward_func/std": 0.2720887064933777, "sampling/importance_sampling_ratio/max": 1.7430676221847534, "sampling/importance_sampling_ratio/mean": 0.9994027614593506, "sampling/importance_sampling_ratio/min": 0.5470163822174072, "sampling/sampling_logp_difference/max": 0.6033272743225098, "sampling/sampling_logp_difference/mean": 0.025745656341314316, "step": 542, "step_time": 16.509508327999356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 324.1875, "completions/mean_terminated_length": 324.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08350000786595047, "epoch": 0.01086, "frac_reward_zero_std": 0.0, "grad_norm": 0.5532575845718384, "kl": 0.4593951841816306, "learning_rate": 5.2476738092888805e-06, "loss": 0.0223, "num_tokens": 26020881.0, "reward": 0.19062499701976776, "reward_std": 0.17287307977676392, "rewards/rollout_reward_func/mean": 0.19062499701976776, "rewards/rollout_reward_func/std": 0.18898604810237885, "sampling/importance_sampling_ratio/max": 1.3512121438980103, "sampling/importance_sampling_ratio/mean": 1.0219175815582275, "sampling/importance_sampling_ratio/min": 0.47901737689971924, "sampling/sampling_logp_difference/max": 0.7376141548156738, "sampling/sampling_logp_difference/mean": 0.019521908834576607, "step": 543, "step_time": 15.743927910007187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 829.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 190.46875, "completions/mean_terminated_length": 190.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08367170998826623, "epoch": 0.01088, "frac_reward_zero_std": 0.0, "grad_norm": 0.4713335335254669, "kl": 1.2733533587306738, "learning_rate": 5.234556350559705e-06, "loss": 0.0103, "num_tokens": 26065122.0, "reward": 0.16562500596046448, "reward_std": 0.24978244304656982, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.24836157262325287, "sampling/importance_sampling_ratio/max": 1.2022309303283691, "sampling/importance_sampling_ratio/mean": 0.9745546579360962, "sampling/importance_sampling_ratio/min": 0.42354297637939453, "sampling/sampling_logp_difference/max": 0.8589072227478027, "sampling/sampling_logp_difference/mean": 0.026046138256788254, "step": 544, "step_time": 12.289840720997745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 306.8125, "completions/mean_terminated_length": 306.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1218776332680136, "epoch": 0.0109, "frac_reward_zero_std": 0.0, "grad_norm": 0.3211233913898468, "kl": 0.7766364198178053, "learning_rate": 5.221452286218712e-06, "loss": 0.0581, "num_tokens": 26114108.0, "reward": 0.15312500298023224, "reward_std": 0.2659749686717987, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.28847137093544006, "sampling/importance_sampling_ratio/max": 1.2793316841125488, "sampling/importance_sampling_ratio/mean": 1.0091865062713623, "sampling/importance_sampling_ratio/min": 0.7346323728561401, "sampling/sampling_logp_difference/max": 0.3074789047241211, "sampling/sampling_logp_difference/mean": 0.015553845092654228, "step": 545, "step_time": 13.948093424001854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 199.65625, "completions/mean_terminated_length": 199.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07918636174872518, "epoch": 0.01092, "frac_reward_zero_std": 0.0, "grad_norm": 0.16954247653484344, "kl": 1.2392978481948376, "learning_rate": 5.2083617891173625e-06, "loss": 0.0222, "num_tokens": 26156931.0, "reward": 0.13750000298023224, "reward_std": 0.253360390663147, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.2836968004703522, "sampling/importance_sampling_ratio/max": 1.0548532009124756, "sampling/importance_sampling_ratio/mean": 0.9679164290428162, "sampling/importance_sampling_ratio/min": 0.2485768049955368, "sampling/sampling_logp_difference/max": 1.021240472793579, "sampling/sampling_logp_difference/mean": 0.028486866503953934, "step": 546, "step_time": 13.759487651001109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 329.8125, "completions/mean_terminated_length": 329.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09283183561637998, "epoch": 0.01094, "frac_reward_zero_std": 0.0, "grad_norm": 0.7277811169624329, "kl": 1.6710301917046309, "learning_rate": 5.195285031928168e-06, "loss": -0.0077, "num_tokens": 26208279.0, "reward": 0.13124999403953552, "reward_std": 0.2936222553253174, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.30207616090774536, "sampling/importance_sampling_ratio/max": 1.2844338417053223, "sampling/importance_sampling_ratio/mean": 0.9997717142105103, "sampling/importance_sampling_ratio/min": 0.5417020320892334, "sampling/sampling_logp_difference/max": 0.6130666732788086, "sampling/sampling_logp_difference/mean": 0.019087065011262894, "step": 547, "step_time": 14.144798193006864 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 351.71875, "completions/mean_terminated_length": 351.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10360681707970798, "epoch": 0.01096, "frac_reward_zero_std": 0.0, "grad_norm": 0.4881179928779602, "kl": 0.6402804302051663, "learning_rate": 5.182222187142388e-06, "loss": 0.0111, "num_tokens": 26259612.0, "reward": 0.11875000596046448, "reward_std": 0.29095950722694397, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.2966887056827545, "sampling/importance_sampling_ratio/max": 1.1855536699295044, "sampling/importance_sampling_ratio/mean": 0.9812241792678833, "sampling/importance_sampling_ratio/min": 0.5850773453712463, "sampling/sampling_logp_difference/max": 0.49263668060302734, "sampling/sampling_logp_difference/mean": 0.023109866306185722, "step": 548, "step_time": 14.321241880003072 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1270.0, "completions/max_terminated_length": 1270.0, "completions/mean_length": 320.28125, "completions/mean_terminated_length": 320.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08320063492283225, "epoch": 0.01098, "frac_reward_zero_std": 0.0, "grad_norm": 0.6496937274932861, "kl": 0.6824236158281565, "learning_rate": 5.169173427067784e-06, "loss": 0.0207, "num_tokens": 26309628.0, "reward": 0.05625000596046448, "reward_std": 0.31031349301338196, "rewards/rollout_reward_func/mean": 0.05625000596046448, "rewards/rollout_reward_func/std": 0.31102070212364197, "sampling/importance_sampling_ratio/max": 1.250675916671753, "sampling/importance_sampling_ratio/mean": 1.015751600265503, "sampling/importance_sampling_ratio/min": 0.5721473693847656, "sampling/sampling_logp_difference/max": 0.5921010971069336, "sampling/sampling_logp_difference/mean": 0.01744556985795498, "step": 549, "step_time": 14.666095252003288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 245.8125, "completions/mean_terminated_length": 245.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08984046243131161, "epoch": 0.011, "frac_reward_zero_std": 0.0, "grad_norm": 0.479480117559433, "kl": 0.989590891636908, "learning_rate": 5.156138923826317e-06, "loss": 0.0315, "num_tokens": 26356200.0, "reward": 0.14374999701976776, "reward_std": 0.26611191034317017, "rewards/rollout_reward_func/mean": 0.14374999701976776, "rewards/rollout_reward_func/std": 0.28277143836021423, "sampling/importance_sampling_ratio/max": 1.9065372943878174, "sampling/importance_sampling_ratio/mean": 1.0189778804779053, "sampling/importance_sampling_ratio/min": 0.4295181930065155, "sampling/sampling_logp_difference/max": 0.845142126083374, "sampling/sampling_logp_difference/mean": 0.029787644743919373, "step": 550, "step_time": 15.571851693999633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 226.59375, "completions/mean_terminated_length": 226.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07330916961655021, "epoch": 0.01102, "frac_reward_zero_std": 0.0, "grad_norm": 0.3747907876968384, "kl": 0.4363550962880254, "learning_rate": 5.143118849351898e-06, "loss": 0.0301, "num_tokens": 26401617.0, "reward": 0.203125, "reward_std": 0.22346696257591248, "rewards/rollout_reward_func/mean": 0.203125, "rewards/rollout_reward_func/std": 0.2442689836025238, "sampling/importance_sampling_ratio/max": 1.3670493364334106, "sampling/importance_sampling_ratio/mean": 1.0206831693649292, "sampling/importance_sampling_ratio/min": 0.8954575061798096, "sampling/sampling_logp_difference/max": 0.30051177740097046, "sampling/sampling_logp_difference/mean": 0.008726735599339008, "step": 551, "step_time": 13.635208449995844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 308.0, "completions/mean_terminated_length": 308.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08390237018465996, "epoch": 0.01104, "frac_reward_zero_std": 0.0, "grad_norm": 0.22219938039779663, "kl": 1.759651156142354, "learning_rate": 5.1301133753881115e-06, "loss": -0.016, "num_tokens": 26452232.0, "reward": 0.16875000298023224, "reward_std": 0.28096136450767517, "rewards/rollout_reward_func/mean": 0.16875000298023224, "rewards/rollout_reward_func/std": 0.2966887354850769, "sampling/importance_sampling_ratio/max": 1.553214192390442, "sampling/importance_sampling_ratio/mean": 1.011305332183838, "sampling/importance_sampling_ratio/min": 0.48691222071647644, "sampling/sampling_logp_difference/max": 0.7228652238845825, "sampling/sampling_logp_difference/mean": 0.018441304564476013, "step": 552, "step_time": 14.351740612004505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 207.15625, "completions/mean_terminated_length": 207.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.059094053925946355, "epoch": 0.01106, "frac_reward_zero_std": 0.0, "grad_norm": 0.2892518639564514, "kl": 0.5681295925751328, "learning_rate": 5.1171226734859505e-06, "loss": -0.0043, "num_tokens": 26497115.0, "reward": 0.29374998807907104, "reward_std": 0.16272574663162231, "rewards/rollout_reward_func/mean": 0.29374998807907104, "rewards/rollout_reward_func/std": 0.16448844969272614, "sampling/importance_sampling_ratio/max": 1.1706786155700684, "sampling/importance_sampling_ratio/mean": 1.0108988285064697, "sampling/importance_sampling_ratio/min": 0.9108057022094727, "sampling/sampling_logp_difference/max": 0.14632755517959595, "sampling/sampling_logp_difference/mean": 0.00587718840688467, "step": 553, "step_time": 13.656579547008732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 265.21875, "completions/mean_terminated_length": 265.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07630625704769045, "epoch": 0.01108, "frac_reward_zero_std": 0.0, "grad_norm": 0.1708008199930191, "kl": 0.5405040550976992, "learning_rate": 5.1041469150015535e-06, "loss": -0.0102, "num_tokens": 26545246.0, "reward": 0.16250000894069672, "reward_std": 0.2888694405555725, "rewards/rollout_reward_func/mean": 0.16250000894069672, "rewards/rollout_reward_func/std": 0.28026485443115234, "sampling/importance_sampling_ratio/max": 1.157016396522522, "sampling/importance_sampling_ratio/mean": 1.0271869897842407, "sampling/importance_sampling_ratio/min": 0.9306083917617798, "sampling/sampling_logp_difference/max": 0.1459246575832367, "sampling/sampling_logp_difference/mean": 0.009509475901722908, "step": 554, "step_time": 13.440617740998277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1270.0, "completions/max_terminated_length": 1270.0, "completions/mean_length": 442.125, "completions/mean_terminated_length": 442.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10956091992557049, "epoch": 0.0111, "frac_reward_zero_std": 0.0, "grad_norm": 0.33454272150993347, "kl": 0.5959991049021482, "learning_rate": 5.091186271093949e-06, "loss": 0.0289, "num_tokens": 26600125.0, "reward": 0.08749999850988388, "reward_std": 0.3212750256061554, "rewards/rollout_reward_func/mean": 0.08749999850988388, "rewards/rollout_reward_func/std": 0.32403701543807983, "sampling/importance_sampling_ratio/max": 1.5242173671722412, "sampling/importance_sampling_ratio/mean": 1.0250290632247925, "sampling/importance_sampling_ratio/min": 0.632452666759491, "sampling/sampling_logp_difference/max": 0.46878695487976074, "sampling/sampling_logp_difference/mean": 0.02074025571346283, "step": 555, "step_time": 14.949542791000567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 272.0625, "completions/mean_terminated_length": 272.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09998213243670762, "epoch": 0.01112, "frac_reward_zero_std": 0.0, "grad_norm": 0.3877198100090027, "kl": 2.402788480743766, "learning_rate": 5.078240912722787e-06, "loss": -0.004, "num_tokens": 26648583.0, "reward": 0.14374999701976776, "reward_std": 0.28011852502822876, "rewards/rollout_reward_func/mean": 0.14374999701976776, "rewards/rollout_reward_func/std": 0.2793281078338623, "sampling/importance_sampling_ratio/max": 2.0203967094421387, "sampling/importance_sampling_ratio/mean": 1.0313109159469604, "sampling/importance_sampling_ratio/min": 0.5398167967796326, "sampling/sampling_logp_difference/max": 0.6929817199707031, "sampling/sampling_logp_difference/mean": 0.019959697499871254, "step": 556, "step_time": 12.937772077002592 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 748.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 244.40625, "completions/mean_terminated_length": 244.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08175115287303925, "epoch": 0.01114, "frac_reward_zero_std": 0.0, "grad_norm": 0.6050269603729248, "kl": 1.9579047337174416, "learning_rate": 5.065311010646101e-06, "loss": 0.0046, "num_tokens": 26696238.0, "reward": 0.18125000596046448, "reward_std": 0.25053033232688904, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.25832900404930115, "sampling/importance_sampling_ratio/max": 1.3722347021102905, "sampling/importance_sampling_ratio/mean": 0.9807574152946472, "sampling/importance_sampling_ratio/min": 0.25858181715011597, "sampling/sampling_logp_difference/max": 1.3617372512817383, "sampling/sampling_logp_difference/mean": 0.02763240784406662, "step": 557, "step_time": 11.835938647003786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 315.53125, "completions/mean_terminated_length": 315.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07069044932723045, "epoch": 0.01116, "frac_reward_zero_std": 0.0, "grad_norm": 0.14424900710582733, "kl": 0.9895272646099329, "learning_rate": 5.052396735418037e-06, "loss": 0.0313, "num_tokens": 26744710.0, "reward": 0.12812499701976776, "reward_std": 0.21901488304138184, "rewards/rollout_reward_func/mean": 0.12812499701976776, "rewards/rollout_reward_func/std": 0.23033547401428223, "sampling/importance_sampling_ratio/max": 1.1589608192443848, "sampling/importance_sampling_ratio/mean": 1.0072548389434814, "sampling/importance_sampling_ratio/min": 0.5413704514503479, "sampling/sampling_logp_difference/max": 0.6137218475341797, "sampling/sampling_logp_difference/mean": 0.011626935563981533, "step": 558, "step_time": 14.18041605599501 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 229.09375, "completions/mean_terminated_length": 229.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05686701531521976, "epoch": 0.01118, "frac_reward_zero_std": 0.0, "grad_norm": 0.1419660747051239, "kl": 1.7706294301897287, "learning_rate": 5.039498257386617e-06, "loss": 0.0016, "num_tokens": 26790399.0, "reward": 0.17500001192092896, "reward_std": 0.25625234842300415, "rewards/rollout_reward_func/mean": 0.17500001192092896, "rewards/rollout_reward_func/std": 0.26396480202674866, "sampling/importance_sampling_ratio/max": 1.0455738306045532, "sampling/importance_sampling_ratio/mean": 0.9853618144989014, "sampling/importance_sampling_ratio/min": 0.5399896502494812, "sampling/sampling_logp_difference/max": 0.6162009239196777, "sampling/sampling_logp_difference/mean": 0.013324538245797157, "step": 559, "step_time": 13.669971586004976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 262.9375, "completions/mean_terminated_length": 262.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06001900543924421, "epoch": 0.0112, "frac_reward_zero_std": 0.0, "grad_norm": 0.197825625538826, "kl": 0.6405659634619951, "learning_rate": 5.026615746691483e-06, "loss": -0.0039, "num_tokens": 26839071.0, "reward": 0.16562499105930328, "reward_std": 0.2116931676864624, "rewards/rollout_reward_func/mean": 0.16562499105930328, "rewards/rollout_reward_func/std": 0.2522279918193817, "sampling/importance_sampling_ratio/max": 1.122352957725525, "sampling/importance_sampling_ratio/mean": 0.9894745945930481, "sampling/importance_sampling_ratio/min": 0.3405934274196625, "sampling/sampling_logp_difference/max": 1.102550983428955, "sampling/sampling_logp_difference/mean": 0.01649653911590576, "step": 560, "step_time": 14.088705501009827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 261.28125, "completions/mean_terminated_length": 261.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09093166422098875, "epoch": 0.01122, "frac_reward_zero_std": 0.0, "grad_norm": 0.38612833619117737, "kl": 2.5453225849196315, "learning_rate": 5.013749373261662e-06, "loss": 0.0054, "num_tokens": 26886639.0, "reward": 0.15937501192092896, "reward_std": 0.29503223299980164, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.2849554121494293, "sampling/importance_sampling_ratio/max": 1.6489204168319702, "sampling/importance_sampling_ratio/mean": 1.002807855606079, "sampling/importance_sampling_ratio/min": 0.5589994788169861, "sampling/sampling_logp_difference/max": 0.5816259384155273, "sampling/sampling_logp_difference/mean": 0.027633186429739, "step": 561, "step_time": 13.736742021996179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 224.0625, "completions/mean_terminated_length": 224.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07244700938463211, "epoch": 0.01124, "frac_reward_zero_std": 0.0, "grad_norm": 0.3538002073764801, "kl": 3.0915384646505117, "learning_rate": 5.0008993068133165e-06, "loss": -0.0018, "num_tokens": 26932291.0, "reward": 0.13437500596046448, "reward_std": 0.27964717149734497, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.27192193269729614, "sampling/importance_sampling_ratio/max": 1.4273080825805664, "sampling/importance_sampling_ratio/mean": 1.0157655477523804, "sampling/importance_sampling_ratio/min": 0.5374866127967834, "sampling/sampling_logp_difference/max": 0.6208438873291016, "sampling/sampling_logp_difference/mean": 0.020958883687853813, "step": 562, "step_time": 13.407146139998076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 265.625, "completions/mean_terminated_length": 265.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0749654769897461, "epoch": 0.01126, "frac_reward_zero_std": 0.0, "grad_norm": 0.4821628928184509, "kl": 1.8661275329068303, "learning_rate": 4.98806571684751e-06, "loss": -0.002, "num_tokens": 26979698.0, "reward": 0.171875, "reward_std": 0.23117947578430176, "rewards/rollout_reward_func/mean": 0.171875, "rewards/rollout_reward_func/std": 0.2465691864490509, "sampling/importance_sampling_ratio/max": 1.3529397249221802, "sampling/importance_sampling_ratio/mean": 0.9857765436172485, "sampling/importance_sampling_ratio/min": 0.5256960391998291, "sampling/sampling_logp_difference/max": 0.7221956253051758, "sampling/sampling_logp_difference/mean": 0.020825015380978584, "step": 563, "step_time": 14.23259875700387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 191.0, "completions/mean_terminated_length": 191.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05929407337680459, "epoch": 0.01128, "frac_reward_zero_std": 0.0, "grad_norm": 0.04124113544821739, "kl": 0.4992482643574476, "learning_rate": 4.975248772647969e-06, "loss": 0.0173, "num_tokens": 27024296.0, "reward": 0.19062501192092896, "reward_std": 0.16818132996559143, "rewards/rollout_reward_func/mean": 0.19062501192092896, "rewards/rollout_reward_func/std": 0.17106756567955017, "sampling/importance_sampling_ratio/max": 1.185698390007019, "sampling/importance_sampling_ratio/mean": 1.0057322978973389, "sampling/importance_sampling_ratio/min": 0.38833433389663696, "sampling/sampling_logp_difference/max": 0.9507296085357666, "sampling/sampling_logp_difference/mean": 0.017591970041394234, "step": 564, "step_time": 11.385529615003179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1297.0, "completions/max_terminated_length": 1297.0, "completions/mean_length": 340.65625, "completions/mean_terminated_length": 340.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10361435799859464, "epoch": 0.0113, "frac_reward_zero_std": 0.0, "grad_norm": 0.640923261642456, "kl": 0.6026933677494526, "learning_rate": 4.962448643278852e-06, "loss": 0.0207, "num_tokens": 27074230.0, "reward": 0.08125000447034836, "reward_std": 0.28874218463897705, "rewards/rollout_reward_func/mean": 0.08125000447034836, "rewards/rollout_reward_func/std": 0.30420440435409546, "sampling/importance_sampling_ratio/max": 1.0700539350509644, "sampling/importance_sampling_ratio/mean": 0.9557797908782959, "sampling/importance_sampling_ratio/min": 0.36251741647720337, "sampling/sampling_logp_difference/max": 1.0146952867507935, "sampling/sampling_logp_difference/mean": 0.03068929724395275, "step": 565, "step_time": 14.450023896002676 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 856.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 312.5625, "completions/mean_terminated_length": 312.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09400858799926937, "epoch": 0.01132, "frac_reward_zero_std": 0.0, "grad_norm": 0.4851783215999603, "kl": 0.9812254142016172, "learning_rate": 4.949665497582519e-06, "loss": 0.0136, "num_tokens": 27123972.0, "reward": 0.05625000223517418, "reward_std": 0.2852572500705719, "rewards/rollout_reward_func/mean": 0.05625000223517418, "rewards/rollout_reward_func/std": 0.2895352244377136, "sampling/importance_sampling_ratio/max": 1.2759301662445068, "sampling/importance_sampling_ratio/mean": 0.947152853012085, "sampling/importance_sampling_ratio/min": 0.2940555512905121, "sampling/sampling_logp_difference/max": 1.2240643501281738, "sampling/sampling_logp_difference/mean": 0.03521501645445824, "step": 566, "step_time": 12.521130752997124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1971.0, "completions/max_terminated_length": 1971.0, "completions/mean_length": 336.5625, "completions/mean_terminated_length": 336.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08921226859092712, "epoch": 0.01134, "frac_reward_zero_std": 0.0, "grad_norm": 0.5720261335372925, "kl": 0.5360112655907869, "learning_rate": 4.936899504177303e-06, "loss": 0.0113, "num_tokens": 27174134.0, "reward": 0.203125, "reward_std": 0.2003004252910614, "rewards/rollout_reward_func/mean": 0.203125, "rewards/rollout_reward_func/std": 0.21773670613765717, "sampling/importance_sampling_ratio/max": 1.2172385454177856, "sampling/importance_sampling_ratio/mean": 1.0126104354858398, "sampling/importance_sampling_ratio/min": 0.6066129803657532, "sampling/sampling_logp_difference/max": 0.6829080581665039, "sampling/sampling_logp_difference/mean": 0.015888279303908348, "step": 567, "step_time": 17.021615186993586 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 306.09375, "completions/mean_terminated_length": 306.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09240428335033357, "epoch": 0.01136, "frac_reward_zero_std": 0.0, "grad_norm": 0.257470041513443, "kl": 0.89353615231812, "learning_rate": 4.9241508314552856e-06, "loss": -0.0014, "num_tokens": 27223069.0, "reward": 0.11875000596046448, "reward_std": 0.2620834708213806, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.29009175300598145, "sampling/importance_sampling_ratio/max": 1.3744760751724243, "sampling/importance_sampling_ratio/mean": 0.9719030857086182, "sampling/importance_sampling_ratio/min": 0.393740177154541, "sampling/sampling_logp_difference/max": 0.9345450401306152, "sampling/sampling_logp_difference/mean": 0.03640005365014076, "step": 568, "step_time": 15.381358174003253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0015625000232830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015625000232830644, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.0, "completions/max_terminated_length": 1252.0, "completions/mean_length": 362.46875, "completions/mean_terminated_length": 362.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10017999215051532, "epoch": 0.01138, "frac_reward_zero_std": 0.0, "grad_norm": 0.6702986359596252, "kl": 4.4937596544623375, "learning_rate": 4.911419647580079e-06, "loss": 0.067, "num_tokens": 27274859.0, "reward": 0.15312501788139343, "reward_std": 0.3265305161476135, "rewards/rollout_reward_func/mean": 0.15312501788139343, "rewards/rollout_reward_func/std": 0.31723031401634216, "sampling/importance_sampling_ratio/max": 1.3082714080810547, "sampling/importance_sampling_ratio/mean": 1.0092217922210693, "sampling/importance_sampling_ratio/min": 0.37113216519355774, "sampling/sampling_logp_difference/max": 0.9913516044616699, "sampling/sampling_logp_difference/mean": 0.027245357632637024, "step": 569, "step_time": 14.860805381995306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 240.34375, "completions/mean_terminated_length": 240.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07677422557026148, "epoch": 0.0114, "frac_reward_zero_std": 0.0, "grad_norm": 0.4712778329849243, "kl": 0.5518181174993515, "learning_rate": 4.898706120484606e-06, "loss": 0.0169, "num_tokens": 27320740.0, "reward": 0.21875, "reward_std": 0.18295049667358398, "rewards/rollout_reward_func/mean": 0.21875, "rewards/rollout_reward_func/std": 0.20546956360340118, "sampling/importance_sampling_ratio/max": 1.1339651346206665, "sampling/importance_sampling_ratio/mean": 0.9891991019248962, "sampling/importance_sampling_ratio/min": 0.6312743425369263, "sampling/sampling_logp_difference/max": 0.47771549224853516, "sampling/sampling_logp_difference/mean": 0.016669053584337234, "step": 570, "step_time": 13.640493521994358 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 304.375, "completions/mean_terminated_length": 304.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07824724121019244, "epoch": 0.01142, "frac_reward_zero_std": 0.0, "grad_norm": 0.6772036552429199, "kl": 0.4617312354966998, "learning_rate": 4.886010417868881e-06, "loss": 0.0046, "num_tokens": 27370328.0, "reward": 0.20000000298023224, "reward_std": 0.2816488742828369, "rewards/rollout_reward_func/mean": 0.20000000298023224, "rewards/rollout_reward_func/std": 0.2873684763908386, "sampling/importance_sampling_ratio/max": 1.6440232992172241, "sampling/importance_sampling_ratio/mean": 1.040659785270691, "sampling/importance_sampling_ratio/min": 0.9928531050682068, "sampling/sampling_logp_difference/max": 0.49707913398742676, "sampling/sampling_logp_difference/mean": 0.010226868093013763, "step": 571, "step_time": 13.736469973999192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 168.125, "completions/mean_terminated_length": 168.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0764514955226332, "epoch": 0.01144, "frac_reward_zero_std": 0.0, "grad_norm": 0.26586949825286865, "kl": 0.5408900771290064, "learning_rate": 4.873332707197804e-06, "loss": 0.0047, "num_tokens": 27413695.0, "reward": 0.15625, "reward_std": 0.24364128708839417, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.2526472508907318, "sampling/importance_sampling_ratio/max": 1.0465599298477173, "sampling/importance_sampling_ratio/mean": 0.9949542284011841, "sampling/importance_sampling_ratio/min": 0.574020266532898, "sampling/sampling_logp_difference/max": 0.5607506036758423, "sampling/sampling_logp_difference/mean": 0.012796302326023579, "step": 572, "step_time": 11.656166949993349 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 214.0625, "completions/mean_terminated_length": 214.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08192973677068949, "epoch": 0.01146, "frac_reward_zero_std": 0.0, "grad_norm": 0.3834482431411743, "kl": 0.6535308919847012, "learning_rate": 4.860673155698953e-06, "loss": 0.0095, "num_tokens": 27458573.0, "reward": 0.19062499701976776, "reward_std": 0.2429349422454834, "rewards/rollout_reward_func/mean": 0.19062499701976776, "rewards/rollout_reward_func/std": 0.2557208240032196, "sampling/importance_sampling_ratio/max": 1.2306383848190308, "sampling/importance_sampling_ratio/mean": 1.0035734176635742, "sampling/importance_sampling_ratio/min": 0.693739652633667, "sampling/sampling_logp_difference/max": 0.4408717155456543, "sampling/sampling_logp_difference/mean": 0.016352498903870583, "step": 573, "step_time": 12.273821407001378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 331.65625, "completions/mean_terminated_length": 331.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07136766170151532, "epoch": 0.01148, "frac_reward_zero_std": 0.0, "grad_norm": 0.37756022810935974, "kl": 0.6144616175442934, "learning_rate": 4.848031930360365e-06, "loss": 0.0184, "num_tokens": 27508589.0, "reward": 0.19375000894069672, "reward_std": 0.24234655499458313, "rewards/rollout_reward_func/mean": 0.19375000894069672, "rewards/rollout_reward_func/std": 0.25895261764526367, "sampling/importance_sampling_ratio/max": 1.2656131982803345, "sampling/importance_sampling_ratio/mean": 1.0227818489074707, "sampling/importance_sampling_ratio/min": 0.6942477822303772, "sampling/sampling_logp_difference/max": 0.3724517822265625, "sampling/sampling_logp_difference/mean": 0.011980246752500534, "step": 574, "step_time": 14.20662127399919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 308.21875, "completions/mean_terminated_length": 308.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08675129036419094, "epoch": 0.0115, "frac_reward_zero_std": 0.0, "grad_norm": 0.31591179966926575, "kl": 1.5428480245172977, "learning_rate": 4.835409197928351e-06, "loss": 0.0433, "num_tokens": 27556336.0, "reward": 0.14687499403953552, "reward_std": 0.22089825570583344, "rewards/rollout_reward_func/mean": 0.14687499403953552, "rewards/rollout_reward_func/std": 0.22430448234081268, "sampling/importance_sampling_ratio/max": 1.1165138483047485, "sampling/importance_sampling_ratio/mean": 0.9665932655334473, "sampling/importance_sampling_ratio/min": 0.3191133439540863, "sampling/sampling_logp_difference/max": 1.1688568592071533, "sampling/sampling_logp_difference/mean": 0.027339309453964233, "step": 575, "step_time": 14.497742755007494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 291.78125, "completions/mean_terminated_length": 291.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11013942258432508, "epoch": 0.01152, "frac_reward_zero_std": 0.0, "grad_norm": 0.5329829454421997, "kl": 0.4910810887813568, "learning_rate": 4.822805124905282e-06, "loss": 0.0246, "num_tokens": 27604538.0, "reward": 0.15000000596046448, "reward_std": 0.29543864727020264, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.291824072599411, "sampling/importance_sampling_ratio/max": 1.1729626655578613, "sampling/importance_sampling_ratio/mean": 0.991260290145874, "sampling/importance_sampling_ratio/min": 0.4863624572753906, "sampling/sampling_logp_difference/max": 0.7422482967376709, "sampling/sampling_logp_difference/mean": 0.021072421222925186, "step": 576, "step_time": 14.23082785999577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 298.21875, "completions/mean_terminated_length": 298.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08375159022398293, "epoch": 0.01154, "frac_reward_zero_std": 0.0, "grad_norm": 0.5009937286376953, "kl": 0.5681555364280939, "learning_rate": 4.810219877547406e-06, "loss": 0.0354, "num_tokens": 27653456.0, "reward": 0.12812501192092896, "reward_std": 0.22053569555282593, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.23449929058551788, "sampling/importance_sampling_ratio/max": 1.4726465940475464, "sampling/importance_sampling_ratio/mean": 1.0129958391189575, "sampling/importance_sampling_ratio/min": 0.6829168200492859, "sampling/sampling_logp_difference/max": 0.38565099239349365, "sampling/sampling_logp_difference/mean": 0.016045846045017242, "step": 577, "step_time": 14.074157925999316 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1742.0, "completions/max_terminated_length": 1742.0, "completions/mean_length": 478.0625, "completions/mean_terminated_length": 478.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07909428817220032, "epoch": 0.01156, "frac_reward_zero_std": 0.0, "grad_norm": 0.18277910351753235, "kl": 1.3025858271867037, "learning_rate": 4.797653621862637e-06, "loss": 0.0255, "num_tokens": 27708730.0, "reward": 0.14687499403953552, "reward_std": 0.25935572385787964, "rewards/rollout_reward_func/mean": 0.14687499403953552, "rewards/rollout_reward_func/std": 0.2552473545074463, "sampling/importance_sampling_ratio/max": 1.176821231842041, "sampling/importance_sampling_ratio/mean": 1.0014572143554688, "sampling/importance_sampling_ratio/min": 0.33173614740371704, "sampling/sampling_logp_difference/max": 1.1034893989562988, "sampling/sampling_logp_difference/mean": 0.017961572855710983, "step": 578, "step_time": 16.754941024006257 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 341.875, "completions/mean_terminated_length": 341.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07705849991180003, "epoch": 0.01158, "frac_reward_zero_std": 0.0, "grad_norm": 0.24317607283592224, "kl": 0.5776197984814644, "learning_rate": 4.785106523608388e-06, "loss": 0.0234, "num_tokens": 27759478.0, "reward": 0.23125000298023224, "reward_std": 0.20768976211547852, "rewards/rollout_reward_func/mean": 0.23125000298023224, "rewards/rollout_reward_func/std": 0.21766725182533264, "sampling/importance_sampling_ratio/max": 1.1704715490341187, "sampling/importance_sampling_ratio/mean": 0.9909195899963379, "sampling/importance_sampling_ratio/min": 0.6105166673660278, "sampling/sampling_logp_difference/max": 0.49752140045166016, "sampling/sampling_logp_difference/mean": 0.019417883828282356, "step": 579, "step_time": 13.776539491998847 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 302.0, "completions/mean_terminated_length": 302.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07941461424343288, "epoch": 0.0116, "frac_reward_zero_std": 0.0, "grad_norm": 0.4825649857521057, "kl": 0.5399841032922268, "learning_rate": 4.7725787482893645e-06, "loss": 0.0136, "num_tokens": 27808963.0, "reward": 0.13750000298023224, "reward_std": 0.2230597287416458, "rewards/rollout_reward_func/mean": 0.13750000298023224, "rewards/rollout_reward_func/std": 0.22396716475486755, "sampling/importance_sampling_ratio/max": 1.191859245300293, "sampling/importance_sampling_ratio/mean": 0.986854076385498, "sampling/importance_sampling_ratio/min": 0.6111171245574951, "sampling/sampling_logp_difference/max": 0.5367319583892822, "sampling/sampling_logp_difference/mean": 0.017696207389235497, "step": 580, "step_time": 14.04417820499657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 355.625, "completions/mean_terminated_length": 355.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08065266557969153, "epoch": 0.01162, "frac_reward_zero_std": 0.0, "grad_norm": 0.4083823561668396, "kl": 0.4840092249214649, "learning_rate": 4.760070461155393e-06, "loss": -0.0005, "num_tokens": 27861231.0, "reward": 0.15625, "reward_std": 0.2754896283149719, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.26993128657341003, "sampling/importance_sampling_ratio/max": 1.181146264076233, "sampling/importance_sampling_ratio/mean": 1.0178052186965942, "sampling/importance_sampling_ratio/min": 0.7142857909202576, "sampling/sampling_logp_difference/max": 0.3850076198577881, "sampling/sampling_logp_difference/mean": 0.013740330934524536, "step": 581, "step_time": 15.477511633995164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 356.09375, "completions/mean_terminated_length": 356.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06809695856645703, "epoch": 0.01164, "frac_reward_zero_std": 0.0, "grad_norm": 0.17858631908893585, "kl": 0.5316841397434473, "learning_rate": 4.7475818271992335e-06, "loss": -0.0039, "num_tokens": 27912425.0, "reward": 0.19687500596046448, "reward_std": 0.20824968814849854, "rewards/rollout_reward_func/mean": 0.19687500596046448, "rewards/rollout_reward_func/std": 0.21625010669231415, "sampling/importance_sampling_ratio/max": 1.157712697982788, "sampling/importance_sampling_ratio/mean": 0.9854030013084412, "sampling/importance_sampling_ratio/min": 0.44422271847724915, "sampling/sampling_logp_difference/max": 0.8200733661651611, "sampling/sampling_logp_difference/mean": 0.01675717532634735, "step": 582, "step_time": 15.505662191004376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 243.125, "completions/mean_terminated_length": 243.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10159896209370345, "epoch": 0.01166, "frac_reward_zero_std": 0.0, "grad_norm": 0.5469534993171692, "kl": 1.0637384131550789, "learning_rate": 4.735113011154413e-06, "loss": 0.0084, "num_tokens": 27961093.0, "reward": 0.125, "reward_std": 0.2753280997276306, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.286243736743927, "sampling/importance_sampling_ratio/max": 1.6202366352081299, "sampling/importance_sampling_ratio/mean": 1.007470965385437, "sampling/importance_sampling_ratio/min": 0.45598286390304565, "sampling/sampling_logp_difference/max": 0.8522253036499023, "sampling/sampling_logp_difference/mean": 0.022661719471216202, "step": 583, "step_time": 14.525277522003307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.012276785913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012276785913854837, "completions/clipped_ratio": 0.03125, "completions/max_length": 1198.0, "completions/max_terminated_length": 1198.0, "completions/mean_length": 388.3125, "completions/mean_terminated_length": 388.8709411621094, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.15851563168689609, "epoch": 0.01168, "frac_reward_zero_std": 0.0, "grad_norm": 0.4639096260070801, "kl": 0.9762555919587612, "learning_rate": 4.722664177493042e-06, "loss": 0.0243, "num_tokens": 28013525.0, "reward": 0.09062500298023224, "reward_std": 0.24256117641925812, "rewards/rollout_reward_func/mean": 0.09062500298023224, "rewards/rollout_reward_func/std": 0.2763347327709198, "sampling/importance_sampling_ratio/max": 1.1669846773147583, "sampling/importance_sampling_ratio/mean": 0.9383014440536499, "sampling/importance_sampling_ratio/min": 0.3341253697872162, "sampling/sampling_logp_difference/max": 1.1082282066345215, "sampling/sampling_logp_difference/mean": 0.038192059844732285, "step": 584, "step_time": 14.908715107001626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1796.0, "completions/max_terminated_length": 1796.0, "completions/mean_length": 422.09375, "completions/mean_terminated_length": 422.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09179421071894467, "epoch": 0.0117, "frac_reward_zero_std": 0.0, "grad_norm": 0.33324021100997925, "kl": 0.471486727707088, "learning_rate": 4.710235490423655e-06, "loss": 0.0119, "num_tokens": 28068263.0, "reward": 0.21250000596046448, "reward_std": 0.2035098373889923, "rewards/rollout_reward_func/mean": 0.21250000596046448, "rewards/rollout_reward_func/std": 0.2254028618335724, "sampling/importance_sampling_ratio/max": 1.1764439344406128, "sampling/importance_sampling_ratio/mean": 1.0218913555145264, "sampling/importance_sampling_ratio/min": 0.7311403155326843, "sampling/sampling_logp_difference/max": 0.23213815689086914, "sampling/sampling_logp_difference/mean": 0.010113855823874474, "step": 585, "step_time": 16.711832368004252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 162.375, "completions/mean_terminated_length": 162.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08787870174273849, "epoch": 0.01172, "frac_reward_zero_std": 0.0, "grad_norm": 0.309064656496048, "kl": 1.2775088138878345, "learning_rate": 4.697827113889034e-06, "loss": 0.0098, "num_tokens": 28111333.0, "reward": 0.10625000298023224, "reward_std": 0.2488563507795334, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.24221757054328918, "sampling/importance_sampling_ratio/max": 1.2684205770492554, "sampling/importance_sampling_ratio/mean": 0.9925737380981445, "sampling/importance_sampling_ratio/min": 0.6221768260002136, "sampling/sampling_logp_difference/max": 0.4744985103607178, "sampling/sampling_logp_difference/mean": 0.018806813284754753, "step": 586, "step_time": 11.352031744991109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1297.0, "completions/max_terminated_length": 1297.0, "completions/mean_length": 258.59375, "completions/mean_terminated_length": 258.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10921953641809523, "epoch": 0.01174, "frac_reward_zero_std": 0.0, "grad_norm": 0.2604910135269165, "kl": 0.5118231289088726, "learning_rate": 4.685439211564055e-06, "loss": 0.043, "num_tokens": 28158925.0, "reward": 0.17812500894069672, "reward_std": 0.18126335740089417, "rewards/rollout_reward_func/mean": 0.17812500894069672, "rewards/rollout_reward_func/std": 0.21362368762493134, "sampling/importance_sampling_ratio/max": 1.3368725776672363, "sampling/importance_sampling_ratio/mean": 0.9954442977905273, "sampling/importance_sampling_ratio/min": 0.3649098575115204, "sampling/sampling_logp_difference/max": 0.8891539573669434, "sampling/sampling_logp_difference/mean": 0.025029798969626427, "step": 587, "step_time": 14.703920518004452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 256.96875, "completions/mean_terminated_length": 256.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08297864906489849, "epoch": 0.01176, "frac_reward_zero_std": 0.0, "grad_norm": 0.26785460114479065, "kl": 0.695111345499754, "learning_rate": 4.67307194685352e-06, "loss": 0.014, "num_tokens": 28206140.0, "reward": 0.1718750149011612, "reward_std": 0.2418946921825409, "rewards/rollout_reward_func/mean": 0.1718750149011612, "rewards/rollout_reward_func/std": 0.2654689848423004, "sampling/importance_sampling_ratio/max": 1.1241267919540405, "sampling/importance_sampling_ratio/mean": 1.019378423690796, "sampling/importance_sampling_ratio/min": 0.9142871499061584, "sampling/sampling_logp_difference/max": 0.23893451690673828, "sampling/sampling_logp_difference/mean": 0.011349081061780453, "step": 588, "step_time": 13.280588935005653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 856.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 321.375, "completions/mean_terminated_length": 321.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08638173039071262, "epoch": 0.01178, "frac_reward_zero_std": 0.0, "grad_norm": 0.48789530992507935, "kl": 0.6580924578011036, "learning_rate": 4.660725482890016e-06, "loss": -0.0038, "num_tokens": 28255985.0, "reward": 0.17500001192092896, "reward_std": 0.22492440044879913, "rewards/rollout_reward_func/mean": 0.17500001192092896, "rewards/rollout_reward_func/std": 0.22860023379325867, "sampling/importance_sampling_ratio/max": 1.1435644626617432, "sampling/importance_sampling_ratio/mean": 1.0036271810531616, "sampling/importance_sampling_ratio/min": 0.6211674809455872, "sampling/sampling_logp_difference/max": 0.4785555601119995, "sampling/sampling_logp_difference/mean": 0.01331283524632454, "step": 589, "step_time": 12.60101146800298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1625.0, "completions/max_terminated_length": 1625.0, "completions/mean_length": 281.5, "completions/mean_terminated_length": 281.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07433164515532553, "epoch": 0.0118, "frac_reward_zero_std": 0.0, "grad_norm": 0.07792889326810837, "kl": 0.4927640650421381, "learning_rate": 4.648399982531745e-06, "loss": 0.0025, "num_tokens": 28305084.0, "reward": 0.17812500894069672, "reward_std": 0.25016704201698303, "rewards/rollout_reward_func/mean": 0.17812500894069672, "rewards/rollout_reward_func/std": 0.26118138432502747, "sampling/importance_sampling_ratio/max": 1.328669548034668, "sampling/importance_sampling_ratio/mean": 1.0044856071472168, "sampling/importance_sampling_ratio/min": 0.2967644929885864, "sampling/sampling_logp_difference/max": 1.2290687561035156, "sampling/sampling_logp_difference/mean": 0.020142680034041405, "step": 590, "step_time": 16.205233172004228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 269.6875, "completions/mean_terminated_length": 269.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08584713097661734, "epoch": 0.01182, "frac_reward_zero_std": 0.0, "grad_norm": 0.42589035630226135, "kl": 0.6011733710765839, "learning_rate": 4.636095608360393e-06, "loss": 0.0205, "num_tokens": 28352329.0, "reward": 0.18125000596046448, "reward_std": 0.21288812160491943, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.22495518624782562, "sampling/importance_sampling_ratio/max": 1.1981984376907349, "sampling/importance_sampling_ratio/mean": 0.9893428087234497, "sampling/importance_sampling_ratio/min": 0.5757352709770203, "sampling/sampling_logp_difference/max": 0.5576269626617432, "sampling/sampling_logp_difference/mean": 0.017012055963277817, "step": 591, "step_time": 13.569479510002566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 275.03125, "completions/mean_terminated_length": 275.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10053591150790453, "epoch": 0.01184, "frac_reward_zero_std": 0.0, "grad_norm": 0.3124133348464966, "kl": 0.883921317756176, "learning_rate": 4.623812522678968e-06, "loss": 0.0137, "num_tokens": 28401122.0, "reward": 0.02500000223517418, "reward_std": 0.2979797422885895, "rewards/rollout_reward_func/mean": 0.02500000223517418, "rewards/rollout_reward_func/std": 0.2929273843765259, "sampling/importance_sampling_ratio/max": 1.498123049736023, "sampling/importance_sampling_ratio/mean": 1.0388071537017822, "sampling/importance_sampling_ratio/min": 0.6248644590377808, "sampling/sampling_logp_difference/max": 0.47009944915771484, "sampling/sampling_logp_difference/mean": 0.018959473818540573, "step": 592, "step_time": 14.550078506992577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 207.46875, "completions/mean_terminated_length": 207.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08350459625944495, "epoch": 0.01186, "frac_reward_zero_std": 0.0, "grad_norm": 0.242730513215065, "kl": 0.5100141037255526, "learning_rate": 4.611550887509677e-06, "loss": 0.0078, "num_tokens": 28446750.0, "reward": 0.11249999701976776, "reward_std": 0.2522539794445038, "rewards/rollout_reward_func/mean": 0.11249999701976776, "rewards/rollout_reward_func/std": 0.2524077594280243, "sampling/importance_sampling_ratio/max": 1.1461513042449951, "sampling/importance_sampling_ratio/mean": 1.0042377710342407, "sampling/importance_sampling_ratio/min": 0.5478670597076416, "sampling/sampling_logp_difference/max": 0.6626632213592529, "sampling/sampling_logp_difference/mean": 0.016303248703479767, "step": 593, "step_time": 13.094902576995082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 391.1875, "completions/mean_terminated_length": 391.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0765700547490269, "epoch": 0.01188, "frac_reward_zero_std": 0.0, "grad_norm": 0.3187299966812134, "kl": 0.8161630537360907, "learning_rate": 4.599310864591776e-06, "loss": 0.007, "num_tokens": 28500066.0, "reward": 0.17499999701976776, "reward_std": 0.2689557373523712, "rewards/rollout_reward_func/mean": 0.17499999701976776, "rewards/rollout_reward_func/std": 0.2929273843765259, "sampling/importance_sampling_ratio/max": 1.246660828590393, "sampling/importance_sampling_ratio/mean": 1.0245717763900757, "sampling/importance_sampling_ratio/min": 0.6086084842681885, "sampling/sampling_logp_difference/max": 0.4982161521911621, "sampling/sampling_logp_difference/mean": 0.012981612235307693, "step": 594, "step_time": 14.045090028997947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1535.0, "completions/max_terminated_length": 1535.0, "completions/mean_length": 258.03125, "completions/mean_terminated_length": 258.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07209600578062236, "epoch": 0.0119, "frac_reward_zero_std": 0.0, "grad_norm": 0.4259035885334015, "kl": 1.8937995973974466, "learning_rate": 4.587092615379442e-06, "loss": 0.0142, "num_tokens": 28547257.0, "reward": 0.10625000298023224, "reward_std": 0.28998905420303345, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.2884189486503601, "sampling/importance_sampling_ratio/max": 1.1364772319793701, "sampling/importance_sampling_ratio/mean": 0.9714799523353577, "sampling/importance_sampling_ratio/min": 0.3766588568687439, "sampling/sampling_logp_difference/max": 0.9764914512634277, "sampling/sampling_logp_difference/mean": 0.02594667673110962, "step": 595, "step_time": 15.556976045005285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1823.0, "completions/max_terminated_length": 1823.0, "completions/mean_length": 383.09375, "completions/mean_terminated_length": 383.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10121701215393841, "epoch": 0.01192, "frac_reward_zero_std": 0.0, "grad_norm": 0.8101791143417358, "kl": 3.6783230481669307, "learning_rate": 4.574896301039641e-06, "loss": -0.0039, "num_tokens": 28599517.0, "reward": 0.15625, "reward_std": 0.2800178527832031, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.0532044172286987, "sampling/importance_sampling_ratio/mean": 0.9352564811706543, "sampling/importance_sampling_ratio/min": 0.2570299506187439, "sampling/sampling_logp_difference/max": 1.3585758209228516, "sampling/sampling_logp_difference/mean": 0.03377406299114227, "step": 596, "step_time": 16.53126304000034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 352.84375, "completions/mean_terminated_length": 352.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06911354861222208, "epoch": 0.01194, "frac_reward_zero_std": 0.0, "grad_norm": 0.12188505381345749, "kl": 0.5949629750102758, "learning_rate": 4.562722082450004e-06, "loss": 0.0528, "num_tokens": 28650653.0, "reward": 0.125, "reward_std": 0.24257811903953552, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.24230079352855682, "sampling/importance_sampling_ratio/max": 1.2608275413513184, "sampling/importance_sampling_ratio/mean": 1.029550313949585, "sampling/importance_sampling_ratio/min": 1.0019042491912842, "sampling/sampling_logp_difference/max": 0.19233494997024536, "sampling/sampling_logp_difference/mean": 0.007466577924787998, "step": 597, "step_time": 14.081147328997758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 349.34375, "completions/mean_terminated_length": 349.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08603993186261505, "epoch": 0.01196, "frac_reward_zero_std": 0.0, "grad_norm": 0.2295958250761032, "kl": 0.5239422749727964, "learning_rate": 4.550570120196702e-06, "loss": 0.0209, "num_tokens": 28701420.0, "reward": 0.15312500298023224, "reward_std": 0.22056016325950623, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.24094487726688385, "sampling/importance_sampling_ratio/max": 1.1542288064956665, "sampling/importance_sampling_ratio/mean": 1.0001157522201538, "sampling/importance_sampling_ratio/min": 0.7202198505401611, "sampling/sampling_logp_difference/max": 0.3359353542327881, "sampling/sampling_logp_difference/mean": 0.012857344001531601, "step": 598, "step_time": 14.747123372002534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 204.65625, "completions/mean_terminated_length": 204.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07522305473685265, "epoch": 0.01198, "frac_reward_zero_std": 0.0, "grad_norm": 0.3407248854637146, "kl": 4.206764169037342, "learning_rate": 4.538440574572337e-06, "loss": 0.0131, "num_tokens": 28745537.0, "reward": 0.18437501788139343, "reward_std": 0.2092369794845581, "rewards/rollout_reward_func/mean": 0.18437501788139343, "rewards/rollout_reward_func/std": 0.21866069734096527, "sampling/importance_sampling_ratio/max": 1.238419532775879, "sampling/importance_sampling_ratio/mean": 0.9756972789764404, "sampling/importance_sampling_ratio/min": 0.4218251407146454, "sampling/sampling_logp_difference/max": 0.8631963729858398, "sampling/sampling_logp_difference/mean": 0.03427407890558243, "step": 599, "step_time": 13.077034867012117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 239.5, "completions/mean_terminated_length": 239.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0645762603962794, "epoch": 0.012, "frac_reward_zero_std": 0.0, "grad_norm": 0.27425071597099304, "kl": 1.5000935858115554, "learning_rate": 4.5263336055738115e-06, "loss": 0.0166, "num_tokens": 28791621.0, "reward": 0.15937499701976776, "reward_std": 0.2578248381614685, "rewards/rollout_reward_func/mean": 0.15937499701976776, "rewards/rollout_reward_func/std": 0.25885525345802307, "sampling/importance_sampling_ratio/max": 1.0976179838180542, "sampling/importance_sampling_ratio/mean": 0.967023491859436, "sampling/importance_sampling_ratio/min": 0.4762164354324341, "sampling/sampling_logp_difference/max": 0.7420268058776855, "sampling/sampling_logp_difference/mean": 0.026370080187916756, "step": 600, "step_time": 13.9431928289996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 328.25, "completions/mean_terminated_length": 328.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07718917564488947, "epoch": 0.01202, "frac_reward_zero_std": 0.0, "grad_norm": 0.38952895998954773, "kl": 1.2566988756880164, "learning_rate": 4.5142493729002316e-06, "loss": 0.0354, "num_tokens": 28842866.0, "reward": 0.125, "reward_std": 0.26011452078819275, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.2565276622772217, "sampling/importance_sampling_ratio/max": 1.165728211402893, "sampling/importance_sampling_ratio/mean": 1.003206729888916, "sampling/importance_sampling_ratio/min": 0.5403617024421692, "sampling/sampling_logp_difference/max": 0.6169028282165527, "sampling/sampling_logp_difference/mean": 0.012754691764712334, "step": 601, "step_time": 14.654770181008644 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 263.53125, "completions/mean_terminated_length": 263.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08958813641220331, "epoch": 0.01204, "frac_reward_zero_std": 0.0, "grad_norm": 0.3841589093208313, "kl": 0.48830701410770416, "learning_rate": 4.502188035950794e-06, "loss": -0.0025, "num_tokens": 28891015.0, "reward": 0.11562500894069672, "reward_std": 0.22131821513175964, "rewards/rollout_reward_func/mean": 0.11562500894069672, "rewards/rollout_reward_func/std": 0.27013659477233887, "sampling/importance_sampling_ratio/max": 1.1652724742889404, "sampling/importance_sampling_ratio/mean": 1.0041955709457397, "sampling/importance_sampling_ratio/min": 0.6465633511543274, "sampling/sampling_logp_difference/max": 0.4451589584350586, "sampling/sampling_logp_difference/mean": 0.014982730150222778, "step": 602, "step_time": 15.736633981003251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 224.4375, "completions/mean_terminated_length": 224.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06729364325292408, "epoch": 0.01206, "frac_reward_zero_std": 0.0, "grad_norm": 0.22201724350452423, "kl": 1.3804795872420073, "learning_rate": 4.490149753822689e-06, "loss": 0.0142, "num_tokens": 28938156.0, "reward": 0.05937499925494194, "reward_std": 0.26286065578460693, "rewards/rollout_reward_func/mean": 0.05937499925494194, "rewards/rollout_reward_func/std": 0.26379287242889404, "sampling/importance_sampling_ratio/max": 1.0986573696136475, "sampling/importance_sampling_ratio/mean": 0.9839386940002441, "sampling/importance_sampling_ratio/min": 0.4859493374824524, "sampling/sampling_logp_difference/max": 0.7216620445251465, "sampling/sampling_logp_difference/mean": 0.018898077309131622, "step": 603, "step_time": 13.69170311400012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 256.46875, "completions/mean_terminated_length": 256.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06640974199399352, "epoch": 0.01208, "frac_reward_zero_std": 0.0, "grad_norm": 0.2848304212093353, "kl": 0.8828150238841772, "learning_rate": 4.478134685308997e-06, "loss": 0.0247, "num_tokens": 28985206.0, "reward": 0.21562498807907104, "reward_std": 0.26494550704956055, "rewards/rollout_reward_func/mean": 0.21562498807907104, "rewards/rollout_reward_func/std": 0.2616441249847412, "sampling/importance_sampling_ratio/max": 1.504090666770935, "sampling/importance_sampling_ratio/mean": 1.0228264331817627, "sampling/importance_sampling_ratio/min": 0.5294685363769531, "sampling/sampling_logp_difference/max": 0.7323741912841797, "sampling/sampling_logp_difference/mean": 0.017823748290538788, "step": 604, "step_time": 13.607204001993523 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 278.96875, "completions/mean_terminated_length": 278.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08651589532382786, "epoch": 0.0121, "frac_reward_zero_std": 0.0, "grad_norm": 0.356105774641037, "kl": 0.5904457997530699, "learning_rate": 4.4661429888965894e-06, "loss": 0.0035, "num_tokens": 29032770.0, "reward": 0.11249999701976776, "reward_std": 0.3043878674507141, "rewards/rollout_reward_func/mean": 0.11249999701976776, "rewards/rollout_reward_func/std": 0.30240967869758606, "sampling/importance_sampling_ratio/max": 1.3156307935714722, "sampling/importance_sampling_ratio/mean": 0.9713050127029419, "sampling/importance_sampling_ratio/min": 0.3181038796901703, "sampling/sampling_logp_difference/max": 1.1481475830078125, "sampling/sampling_logp_difference/mean": 0.030672486871480942, "step": 605, "step_time": 13.144505378004396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 218.1875, "completions/mean_terminated_length": 218.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0863783317618072, "epoch": 0.01212, "frac_reward_zero_std": 0.0, "grad_norm": 0.3185900151729584, "kl": 1.092000275850296, "learning_rate": 4.454174822764049e-06, "loss": 0.0104, "num_tokens": 29077212.0, "reward": 0.11562500894069672, "reward_std": 0.24769334495067596, "rewards/rollout_reward_func/mean": 0.11562500894069672, "rewards/rollout_reward_func/std": 0.24111217260360718, "sampling/importance_sampling_ratio/max": 1.086989402770996, "sampling/importance_sampling_ratio/mean": 0.9589309692382812, "sampling/importance_sampling_ratio/min": 0.39990270137786865, "sampling/sampling_logp_difference/max": 0.9165122509002686, "sampling/sampling_logp_difference/mean": 0.03375384211540222, "step": 606, "step_time": 14.801438893999148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 335.8125, "completions/mean_terminated_length": 335.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06402776367031038, "epoch": 0.01214, "frac_reward_zero_std": 0.0, "grad_norm": 0.22535045444965363, "kl": 3.5126641178503633, "learning_rate": 4.4422303447795816e-06, "loss": 0.0191, "num_tokens": 29128687.0, "reward": 0.06562500447034836, "reward_std": 0.22164995968341827, "rewards/rollout_reward_func/mean": 0.06562500447034836, "rewards/rollout_reward_func/std": 0.23501458764076233, "sampling/importance_sampling_ratio/max": 1.0878530740737915, "sampling/importance_sampling_ratio/mean": 0.9345847368240356, "sampling/importance_sampling_ratio/min": 0.34423717856407166, "sampling/sampling_logp_difference/max": 1.070490837097168, "sampling/sampling_logp_difference/mean": 0.033176567405462265, "step": 607, "step_time": 13.838295016998018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 355.5625, "completions/mean_terminated_length": 355.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10790047864429653, "epoch": 0.01216, "frac_reward_zero_std": 0.0, "grad_norm": 0.6113014221191406, "kl": 0.6943326368927956, "learning_rate": 4.430309712498924e-06, "loss": 0.0007, "num_tokens": 29179539.0, "reward": 0.13437500596046448, "reward_std": 0.23547157645225525, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.2336379587650299, "sampling/importance_sampling_ratio/max": 1.2326829433441162, "sampling/importance_sampling_ratio/mean": 0.976852536201477, "sampling/importance_sampling_ratio/min": 0.5404141545295715, "sampling/sampling_logp_difference/max": 0.6154131889343262, "sampling/sampling_logp_difference/mean": 0.02554478496313095, "step": 608, "step_time": 15.862311924007372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 213.9375, "completions/mean_terminated_length": 220.32257080078125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11476095858961344, "epoch": 0.01218, "frac_reward_zero_std": 0.0, "grad_norm": 0.8664658665657043, "kl": 1.1322905411943793, "learning_rate": 4.418413083163273e-06, "loss": -0.0142, "num_tokens": 29224507.0, "reward": 0.15000000596046448, "reward_std": 0.19828446209430695, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.24362848699092865, "sampling/importance_sampling_ratio/max": 1.0831116437911987, "sampling/importance_sampling_ratio/mean": 0.9612081050872803, "sampling/importance_sampling_ratio/min": 0.47570642828941345, "sampling/sampling_logp_difference/max": 0.7430510520935059, "sampling/sampling_logp_difference/mean": 0.029050350189208984, "step": 609, "step_time": 13.448330000002898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.010416666977107525, "clip_ratio/region_mean": 0.010416666977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 351.5, "completions/mean_terminated_length": 351.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18046915833838284, "epoch": 0.0122, "frac_reward_zero_std": 0.0, "grad_norm": 0.9272422790527344, "kl": 2.0500618070364, "learning_rate": 4.406540613697213e-06, "loss": -0.0149, "num_tokens": 29276103.0, "reward": 0.125, "reward_std": 0.2904529571533203, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.3005371689796448, "sampling/importance_sampling_ratio/max": 1.734403133392334, "sampling/importance_sampling_ratio/mean": 0.987561047077179, "sampling/importance_sampling_ratio/min": 6.038781208417277e-08, "sampling/sampling_logp_difference/max": 16.46518898010254, "sampling/sampling_logp_difference/mean": 0.15388762950897217, "step": 610, "step_time": 15.406863016996795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 228.8125, "completions/mean_terminated_length": 228.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07222834113053977, "epoch": 0.01222, "frac_reward_zero_std": 0.0, "grad_norm": 0.2359233945608139, "kl": 0.521993275731802, "learning_rate": 4.394692460706643e-06, "loss": 0.0197, "num_tokens": 29321914.0, "reward": 0.20000000298023224, "reward_std": 0.21308378875255585, "rewards/rollout_reward_func/mean": 0.20000000298023224, "rewards/rollout_reward_func/std": 0.2243269383907318, "sampling/importance_sampling_ratio/max": 1.045714259147644, "sampling/importance_sampling_ratio/mean": 1.0122376680374146, "sampling/importance_sampling_ratio/min": 0.9782715439796448, "sampling/sampling_logp_difference/max": 0.0447792187333107, "sampling/sampling_logp_difference/mean": 0.004824436269700527, "step": 611, "step_time": 13.765338301000156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 315.78125, "completions/mean_terminated_length": 315.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0923267318867147, "epoch": 0.01224, "frac_reward_zero_std": 0.0, "grad_norm": 0.1893029361963272, "kl": 0.5846448969095945, "learning_rate": 4.382868780476709e-06, "loss": 0.0199, "num_tokens": 29372060.0, "reward": 0.18125000596046448, "reward_std": 0.22522231936454773, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.25832900404930115, "sampling/importance_sampling_ratio/max": 1.1150258779525757, "sampling/importance_sampling_ratio/mean": 1.0026638507843018, "sampling/importance_sampling_ratio/min": 0.3838961720466614, "sampling/sampling_logp_difference/max": 0.9574289321899414, "sampling/sampling_logp_difference/mean": 0.016577227041125298, "step": 612, "step_time": 14.013214762002463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 991.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 241.875, "completions/mean_terminated_length": 241.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08607906382530928, "epoch": 0.01226, "frac_reward_zero_std": 0.0, "grad_norm": 0.294829398393631, "kl": 0.52976318821311, "learning_rate": 4.371069728969745e-06, "loss": 0.0276, "num_tokens": 29418751.0, "reward": 0.14374999701976776, "reward_std": 0.2181677520275116, "rewards/rollout_reward_func/mean": 0.14374999701976776, "rewards/rollout_reward_func/std": 0.23546454310417175, "sampling/importance_sampling_ratio/max": 1.4378929138183594, "sampling/importance_sampling_ratio/mean": 1.017193078994751, "sampling/importance_sampling_ratio/min": 0.5669209361076355, "sampling/sampling_logp_difference/max": 0.6176385879516602, "sampling/sampling_logp_difference/mean": 0.0178899634629488, "step": 613, "step_time": 13.4175918140063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 229.84375, "completions/mean_terminated_length": 229.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09286120953038335, "epoch": 0.01228, "frac_reward_zero_std": 0.0, "grad_norm": 0.345842182636261, "kl": 0.4742646999657154, "learning_rate": 4.359295461823221e-06, "loss": 0.0087, "num_tokens": 29463669.0, "reward": 0.12812499701976776, "reward_std": 0.3260238766670227, "rewards/rollout_reward_func/mean": 0.12812499701976776, "rewards/rollout_reward_func/std": 0.32252466678619385, "sampling/importance_sampling_ratio/max": 1.3097147941589355, "sampling/importance_sampling_ratio/mean": 0.997251033782959, "sampling/importance_sampling_ratio/min": 0.6301340460777283, "sampling/sampling_logp_difference/max": 0.5434765815734863, "sampling/sampling_logp_difference/mean": 0.02253621630370617, "step": 614, "step_time": 14.371523537007306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 368.78125, "completions/mean_terminated_length": 368.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09949373616836965, "epoch": 0.0123, "frac_reward_zero_std": 0.0, "grad_norm": 0.25200605392456055, "kl": 0.5952727235853672, "learning_rate": 4.347546134347673e-06, "loss": 0.0137, "num_tokens": 29515414.0, "reward": 0.1562500149011612, "reward_std": 0.25682348012924194, "rewards/rollout_reward_func/mean": 0.1562500149011612, "rewards/rollout_reward_func/std": 0.28277143836021423, "sampling/importance_sampling_ratio/max": 1.3052862882614136, "sampling/importance_sampling_ratio/mean": 0.9992725253105164, "sampling/importance_sampling_ratio/min": 0.16701512038707733, "sampling/sampling_logp_difference/max": 1.135328769683838, "sampling/sampling_logp_difference/mean": 0.02630351297557354, "step": 615, "step_time": 15.815706301993487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 278.0, "completions/mean_terminated_length": 278.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07589671271853149, "epoch": 0.01232, "frac_reward_zero_std": 0.0, "grad_norm": 0.22349129617214203, "kl": 2.3063650224357843, "learning_rate": 4.33582190152468e-06, "loss": 0.0164, "num_tokens": 29564388.0, "reward": 0.15625, "reward_std": 0.24689370393753052, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.2564490735530853, "sampling/importance_sampling_ratio/max": 1.3335682153701782, "sampling/importance_sampling_ratio/mean": 0.9903564453125, "sampling/importance_sampling_ratio/min": 0.5770297646522522, "sampling/sampling_logp_difference/max": 0.5680499076843262, "sampling/sampling_logp_difference/mean": 0.019249554723501205, "step": 616, "step_time": 13.383722200993361 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 285.03125, "completions/mean_terminated_length": 285.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08557402365840971, "epoch": 0.01234, "frac_reward_zero_std": 0.0, "grad_norm": 0.40669527649879456, "kl": 0.6138104926794767, "learning_rate": 4.324122918004795e-06, "loss": -0.0059, "num_tokens": 29612749.0, "reward": 0.13750001788139343, "reward_std": 0.2567668855190277, "rewards/rollout_reward_func/mean": 0.13750001788139343, "rewards/rollout_reward_func/std": 0.2779533267021179, "sampling/importance_sampling_ratio/max": 1.165191411972046, "sampling/importance_sampling_ratio/mean": 0.9938145875930786, "sampling/importance_sampling_ratio/min": 0.5526726245880127, "sampling/sampling_logp_difference/max": 0.5959405899047852, "sampling/sampling_logp_difference/mean": 0.020614786073565483, "step": 617, "step_time": 14.622680284002854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1279.0, "completions/max_terminated_length": 1279.0, "completions/mean_length": 373.15625, "completions/mean_terminated_length": 373.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07855396065860987, "epoch": 0.01236, "frac_reward_zero_std": 0.0, "grad_norm": 0.935169517993927, "kl": 0.695180500857532, "learning_rate": 4.3124493381055204e-06, "loss": 0.0275, "num_tokens": 29663449.0, "reward": 0.12812499701976776, "reward_std": 0.3032187521457672, "rewards/rollout_reward_func/mean": 0.12812499701976776, "rewards/rollout_reward_func/std": 0.3123628795146942, "sampling/importance_sampling_ratio/max": 1.2245430946350098, "sampling/importance_sampling_ratio/mean": 1.0001112222671509, "sampling/importance_sampling_ratio/min": 0.610206663608551, "sampling/sampling_logp_difference/max": 0.4974946975708008, "sampling/sampling_logp_difference/mean": 0.01932252012193203, "step": 618, "step_time": 14.960557610007527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 319.03125, "completions/mean_terminated_length": 319.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09883459703996778, "epoch": 0.01238, "frac_reward_zero_std": 0.0, "grad_norm": 0.2805657684803009, "kl": 0.6348453164100647, "learning_rate": 4.300801315809264e-06, "loss": 0.0164, "num_tokens": 29712862.0, "reward": 0.140625, "reward_std": 0.23334603011608124, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.23120912909507751, "sampling/importance_sampling_ratio/max": 1.1289288997650146, "sampling/importance_sampling_ratio/mean": 0.9900593161582947, "sampling/importance_sampling_ratio/min": 0.35900044441223145, "sampling/sampling_logp_difference/max": 0.5520039796829224, "sampling/sampling_logp_difference/mean": 0.020344771444797516, "step": 619, "step_time": 13.790487969999958 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 253.84375, "completions/mean_terminated_length": 253.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0841301407199353, "epoch": 0.0124, "frac_reward_zero_std": 0.0, "grad_norm": 0.6963982582092285, "kl": 2.246057684533298, "learning_rate": 4.289179004761318e-06, "loss": 0.0308, "num_tokens": 29759184.0, "reward": 0.08125000447034836, "reward_std": 0.3204961121082306, "rewards/rollout_reward_func/mean": 0.08125000447034836, "rewards/rollout_reward_func/std": 0.3187045454978943, "sampling/importance_sampling_ratio/max": 1.2454149723052979, "sampling/importance_sampling_ratio/mean": 0.9808439016342163, "sampling/importance_sampling_ratio/min": 0.4926367700099945, "sampling/sampling_logp_difference/max": 0.712092399597168, "sampling/sampling_logp_difference/mean": 0.026681266725063324, "step": 620, "step_time": 14.569057674998476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 220.25, "completions/mean_terminated_length": 220.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1039287718012929, "epoch": 0.01242, "frac_reward_zero_std": 0.0, "grad_norm": 0.4003746509552002, "kl": 1.5727888606488705, "learning_rate": 4.277582558267819e-06, "loss": -0.0036, "num_tokens": 29804520.0, "reward": 0.17500001192092896, "reward_std": 0.2761698067188263, "rewards/rollout_reward_func/mean": 0.17500001192092896, "rewards/rollout_reward_func/std": 0.2747432589530945, "sampling/importance_sampling_ratio/max": 1.2068462371826172, "sampling/importance_sampling_ratio/mean": 0.988179624080658, "sampling/importance_sampling_ratio/min": 0.4195859134197235, "sampling/sampling_logp_difference/max": 0.8701839447021484, "sampling/sampling_logp_difference/mean": 0.02955992892384529, "step": 621, "step_time": 14.035847328003001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1328.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 229.6875, "completions/mean_terminated_length": 229.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09125212277285755, "epoch": 0.01244, "frac_reward_zero_std": 0.0, "grad_norm": 0.357950896024704, "kl": 1.0554689690470695, "learning_rate": 4.266012129293736e-06, "loss": -0.0032, "num_tokens": 29849685.0, "reward": 0.10000000894069672, "reward_std": 0.26747795939445496, "rewards/rollout_reward_func/mean": 0.10000000894069672, "rewards/rollout_reward_func/std": 0.28169989585876465, "sampling/importance_sampling_ratio/max": 1.3595178127288818, "sampling/importance_sampling_ratio/mean": 1.0064321756362915, "sampling/importance_sampling_ratio/min": 0.37869125604629517, "sampling/sampling_logp_difference/max": 0.9710474014282227, "sampling/sampling_logp_difference/mean": 0.024413323029875755, "step": 622, "step_time": 14.464041742994596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2205.0, "completions/max_terminated_length": 2205.0, "completions/mean_length": 401.34375, "completions/mean_terminated_length": 401.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09436764905694872, "epoch": 0.01246, "frac_reward_zero_std": 0.0, "grad_norm": 0.7641575336456299, "kl": 2.62423162907362, "learning_rate": 4.254467870460848e-06, "loss": 0.0311, "num_tokens": 29902228.0, "reward": 0.14374999701976776, "reward_std": 0.29412510991096497, "rewards/rollout_reward_func/mean": 0.14374999701976776, "rewards/rollout_reward_func/std": 0.3222276270389557, "sampling/importance_sampling_ratio/max": 1.7969261407852173, "sampling/importance_sampling_ratio/mean": 1.004648208618164, "sampling/importance_sampling_ratio/min": 0.3897795081138611, "sampling/sampling_logp_difference/max": 0.9968171119689941, "sampling/sampling_logp_difference/mean": 0.026797575876116753, "step": 623, "step_time": 20.37161265898976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 235.84375, "completions/mean_terminated_length": 235.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07215550797991455, "epoch": 0.01248, "frac_reward_zero_std": 0.0, "grad_norm": 0.5865734219551086, "kl": 0.7656648652628064, "learning_rate": 4.2429499340457344e-06, "loss": 0.0064, "num_tokens": 29948569.0, "reward": 0.18437500298023224, "reward_std": 0.22957445681095123, "rewards/rollout_reward_func/mean": 0.18437500298023224, "rewards/rollout_reward_func/std": 0.2315576672554016, "sampling/importance_sampling_ratio/max": 1.4254591464996338, "sampling/importance_sampling_ratio/mean": 1.0158811807632446, "sampling/importance_sampling_ratio/min": 0.5443712472915649, "sampling/sampling_logp_difference/max": 0.6081223487854004, "sampling/sampling_logp_difference/mean": 0.018815917894244194, "step": 624, "step_time": 13.5903610580026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 314.5, "completions/mean_terminated_length": 314.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08345687249675393, "epoch": 0.0125, "frac_reward_zero_std": 0.0, "grad_norm": 0.1815110594034195, "kl": 0.4698667787015438, "learning_rate": 4.231458471977764e-06, "loss": -0.0208, "num_tokens": 29998341.0, "reward": 0.21250000596046448, "reward_std": 0.20710602402687073, "rewards/rollout_reward_func/mean": 0.21250000596046448, "rewards/rollout_reward_func/std": 0.21812988817691803, "sampling/importance_sampling_ratio/max": 1.1135761737823486, "sampling/importance_sampling_ratio/mean": 0.9898751974105835, "sampling/importance_sampling_ratio/min": 0.6164879202842712, "sampling/sampling_logp_difference/max": 0.48838406801223755, "sampling/sampling_logp_difference/mean": 0.017474975436925888, "step": 625, "step_time": 13.649118458011799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 227.6875, "completions/mean_terminated_length": 227.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1062394694890827, "epoch": 0.01252, "frac_reward_zero_std": 0.0, "grad_norm": 0.37820541858673096, "kl": 1.9696416445076466, "learning_rate": 4.219993635837086e-06, "loss": 0.0036, "num_tokens": 30045153.0, "reward": 0.08437500149011612, "reward_std": 0.2916395366191864, "rewards/rollout_reward_func/mean": 0.08437500149011612, "rewards/rollout_reward_func/std": 0.2963317334651947, "sampling/importance_sampling_ratio/max": 1.4848823547363281, "sampling/importance_sampling_ratio/mean": 1.0422389507293701, "sampling/importance_sampling_ratio/min": 0.7856771945953369, "sampling/sampling_logp_difference/max": 0.34029102325439453, "sampling/sampling_logp_difference/mean": 0.020040128380060196, "step": 626, "step_time": 14.44446798500212 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 982.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 227.5625, "completions/mean_terminated_length": 227.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06796081061474979, "epoch": 0.01254, "frac_reward_zero_std": 0.0, "grad_norm": 0.4928034842014313, "kl": 2.6379589773714542, "learning_rate": 4.208555576852641e-06, "loss": 0.0093, "num_tokens": 30090702.0, "reward": 0.16562500596046448, "reward_std": 0.2224694937467575, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.2509457767009735, "sampling/importance_sampling_ratio/max": 1.2448663711547852, "sampling/importance_sampling_ratio/mean": 0.9745342135429382, "sampling/importance_sampling_ratio/min": 0.3346988558769226, "sampling/sampling_logp_difference/max": 1.0959687232971191, "sampling/sampling_logp_difference/mean": 0.027620581910014153, "step": 627, "step_time": 13.236153601996193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1769.0, "completions/max_terminated_length": 1769.0, "completions/mean_length": 369.03125, "completions/mean_terminated_length": 369.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.1102887240704149, "epoch": 0.01256, "frac_reward_zero_std": 0.0, "grad_norm": 0.31789731979370117, "kl": 0.4492895994335413, "learning_rate": 4.19714444590016e-06, "loss": 0.0209, "num_tokens": 30142835.0, "reward": 0.09062499552965164, "reward_std": 0.2907423973083496, "rewards/rollout_reward_func/mean": 0.09062499552965164, "rewards/rollout_reward_func/std": 0.3030589818954468, "sampling/importance_sampling_ratio/max": 1.1629621982574463, "sampling/importance_sampling_ratio/mean": 1.0044305324554443, "sampling/importance_sampling_ratio/min": 0.559146523475647, "sampling/sampling_logp_difference/max": 0.5936534404754639, "sampling/sampling_logp_difference/mean": 0.014864224009215832, "step": 628, "step_time": 16.60875803498857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1036.0, "completions/max_terminated_length": 1036.0, "completions/mean_length": 370.8125, "completions/mean_terminated_length": 370.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09422297705896199, "epoch": 0.01258, "frac_reward_zero_std": 0.0, "grad_norm": 0.11610864102840424, "kl": 0.6351947169750929, "learning_rate": 4.185760393500174e-06, "loss": -0.0107, "num_tokens": 30195956.0, "reward": 0.14374999701976776, "reward_std": 0.2543908357620239, "rewards/rollout_reward_func/mean": 0.14374999701976776, "rewards/rollout_reward_func/std": 0.24878737330436707, "sampling/importance_sampling_ratio/max": 1.399871826171875, "sampling/importance_sampling_ratio/mean": 1.0106163024902344, "sampling/importance_sampling_ratio/min": 0.5265010595321655, "sampling/sampling_logp_difference/max": 0.6487503051757812, "sampling/sampling_logp_difference/mean": 0.01892615482211113, "step": 629, "step_time": 14.202693310009636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 276.4375, "completions/mean_terminated_length": 276.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06630371324717999, "epoch": 0.0126, "frac_reward_zero_std": 0.0, "grad_norm": 0.08485102653503418, "kl": 0.4812748013064265, "learning_rate": 4.17440356981603e-06, "loss": -0.0197, "num_tokens": 30244366.0, "reward": 0.25, "reward_std": 0.18105056881904602, "rewards/rollout_reward_func/mean": 0.25, "rewards/rollout_reward_func/std": 0.1849149763584137, "sampling/importance_sampling_ratio/max": 1.1247930526733398, "sampling/importance_sampling_ratio/mean": 1.0033775568008423, "sampling/importance_sampling_ratio/min": 0.6136725544929504, "sampling/sampling_logp_difference/max": 0.49007081985473633, "sampling/sampling_logp_difference/mean": 0.010199839249253273, "step": 630, "step_time": 13.693883775995346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1009.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 288.625, "completions/mean_terminated_length": 288.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06863978179171681, "epoch": 0.01262, "frac_reward_zero_std": 0.0, "grad_norm": 0.051791585981845856, "kl": 0.5278339926153421, "learning_rate": 4.163074124651908e-06, "loss": 0.0161, "num_tokens": 30294536.0, "reward": 0.22187499701976776, "reward_std": 0.23465728759765625, "rewards/rollout_reward_func/mean": 0.22187499701976776, "rewards/rollout_reward_func/std": 0.23518608510494232, "sampling/importance_sampling_ratio/max": 1.0994148254394531, "sampling/importance_sampling_ratio/mean": 1.0188497304916382, "sampling/importance_sampling_ratio/min": 0.9793961644172668, "sampling/sampling_logp_difference/max": 0.09472343325614929, "sampling/sampling_logp_difference/mean": 0.005864623934030533, "step": 631, "step_time": 13.679535409009986 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 308.03125, "completions/mean_terminated_length": 308.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09662746312096715, "epoch": 0.01264, "frac_reward_zero_std": 0.0, "grad_norm": 0.5715571641921997, "kl": 2.599093271419406, "learning_rate": 4.15177220745085e-06, "loss": 0.0096, "num_tokens": 30345024.0, "reward": 0.11562500894069672, "reward_std": 0.2637738585472107, "rewards/rollout_reward_func/mean": 0.11562500894069672, "rewards/rollout_reward_func/std": 0.26287415623664856, "sampling/importance_sampling_ratio/max": 1.1914174556732178, "sampling/importance_sampling_ratio/mean": 0.9973599910736084, "sampling/importance_sampling_ratio/min": 0.609139084815979, "sampling/sampling_logp_difference/max": 0.49561071395874023, "sampling/sampling_logp_difference/mean": 0.021960657089948654, "step": 632, "step_time": 15.642379204997269 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 219.25, "completions/mean_terminated_length": 219.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06642826646566391, "epoch": 0.01266, "frac_reward_zero_std": 0.0, "grad_norm": 0.032051119953393936, "kl": 0.4907330609858036, "learning_rate": 4.140497967292782e-06, "loss": 0.0202, "num_tokens": 30391077.0, "reward": 0.19374999403953552, "reward_std": 0.20473575592041016, "rewards/rollout_reward_func/mean": 0.19374999403953552, "rewards/rollout_reward_func/std": 0.22134122252464294, "sampling/importance_sampling_ratio/max": 1.1731972694396973, "sampling/importance_sampling_ratio/mean": 1.020481824874878, "sampling/importance_sampling_ratio/min": 1.003954291343689, "sampling/sampling_logp_difference/max": 0.1369401216506958, "sampling/sampling_logp_difference/mean": 0.00616749981418252, "step": 633, "step_time": 13.492898456996045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 278.15625, "completions/mean_terminated_length": 278.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07977109774947166, "epoch": 0.01268, "frac_reward_zero_std": 0.0, "grad_norm": 0.2811845541000366, "kl": 2.9650392569601536, "learning_rate": 4.1292515528925584e-06, "loss": 0.0066, "num_tokens": 30438087.0, "reward": 0.17812499403953552, "reward_std": 0.25871700048446655, "rewards/rollout_reward_func/mean": 0.17812499403953552, "rewards/rollout_reward_func/std": 0.2599433660507202, "sampling/importance_sampling_ratio/max": 1.2287269830703735, "sampling/importance_sampling_ratio/mean": 0.9837837815284729, "sampling/importance_sampling_ratio/min": 0.4240591824054718, "sampling/sampling_logp_difference/max": 0.8578987121582031, "sampling/sampling_logp_difference/mean": 0.027220813557505608, "step": 634, "step_time": 14.540666364013305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 892.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 247.8125, "completions/mean_terminated_length": 247.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09271051501855254, "epoch": 0.0127, "frac_reward_zero_std": 0.0, "grad_norm": 0.2669813334941864, "kl": 0.5214745383709669, "learning_rate": 4.11803311259798e-06, "loss": 0.0418, "num_tokens": 30484770.0, "reward": 0.13125000894069672, "reward_std": 0.2312140017747879, "rewards/rollout_reward_func/mean": 0.13125000894069672, "rewards/rollout_reward_func/std": 0.24943485856056213, "sampling/importance_sampling_ratio/max": 1.0998035669326782, "sampling/importance_sampling_ratio/mean": 1.010246992111206, "sampling/importance_sampling_ratio/min": 0.5513219237327576, "sampling/sampling_logp_difference/max": 0.6023650169372559, "sampling/sampling_logp_difference/mean": 0.013759635388851166, "step": 635, "step_time": 12.60117704699951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2215.0, "completions/max_terminated_length": 2215.0, "completions/mean_length": 398.8125, "completions/mean_terminated_length": 398.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12348493747413158, "epoch": 0.01272, "frac_reward_zero_std": 0.0, "grad_norm": 0.91152024269104, "kl": 0.6435505263507366, "learning_rate": 4.106842794387864e-06, "loss": 0.0391, "num_tokens": 30537169.0, "reward": 0.0937500149011612, "reward_std": 0.33234545588493347, "rewards/rollout_reward_func/mean": 0.0937500149011612, "rewards/rollout_reward_func/std": 0.38599470257759094, "sampling/importance_sampling_ratio/max": 1.7064369916915894, "sampling/importance_sampling_ratio/mean": 1.020989179611206, "sampling/importance_sampling_ratio/min": 0.6268985867500305, "sampling/sampling_logp_difference/max": 0.5966343879699707, "sampling/sampling_logp_difference/mean": 0.023742325603961945, "step": 636, "step_time": 18.892305586017756 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1625.0, "completions/max_terminated_length": 1625.0, "completions/mean_length": 275.75, "completions/mean_terminated_length": 275.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06758371647447348, "epoch": 0.01274, "frac_reward_zero_std": 0.0, "grad_norm": 0.1433514505624771, "kl": 2.0992514584213495, "learning_rate": 4.095680745870069e-06, "loss": -0.0108, "num_tokens": 30585116.0, "reward": 0.18125000596046448, "reward_std": 0.28645801544189453, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.2810550332069397, "sampling/importance_sampling_ratio/max": 1.1386604309082031, "sampling/importance_sampling_ratio/mean": 0.9812308549880981, "sampling/importance_sampling_ratio/min": 0.3462313711643219, "sampling/sampling_logp_difference/max": 1.060624122619629, "sampling/sampling_logp_difference/mean": 0.025147819891572, "step": 637, "step_time": 16.13316774799023 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 415.65625, "completions/mean_terminated_length": 415.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08549656299874187, "epoch": 0.01276, "frac_reward_zero_std": 0.0, "grad_norm": 0.3804956078529358, "kl": 0.5604713726788759, "learning_rate": 4.084547114279564e-06, "loss": -0.0024, "num_tokens": 30639036.0, "reward": 0.14375001192092896, "reward_std": 0.2841108739376068, "rewards/rollout_reward_func/mean": 0.14375001192092896, "rewards/rollout_reward_func/std": 0.29175499081611633, "sampling/importance_sampling_ratio/max": 1.190338373184204, "sampling/importance_sampling_ratio/mean": 1.0164703130722046, "sampling/importance_sampling_ratio/min": 0.625544011592865, "sampling/sampling_logp_difference/max": 0.5800977945327759, "sampling/sampling_logp_difference/mean": 0.015341974794864655, "step": 638, "step_time": 15.032560365005338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 300.75, "completions/mean_terminated_length": 300.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08401272678747773, "epoch": 0.01278, "frac_reward_zero_std": 0.0, "grad_norm": 0.7411704659461975, "kl": 3.2902073338627815, "learning_rate": 4.073442046476472e-06, "loss": 0.0226, "num_tokens": 30688038.0, "reward": 0.16875000298023224, "reward_std": 0.2660861611366272, "rewards/rollout_reward_func/mean": 0.16875000298023224, "rewards/rollout_reward_func/std": 0.27290257811546326, "sampling/importance_sampling_ratio/max": 1.6499581336975098, "sampling/importance_sampling_ratio/mean": 1.0275993347167969, "sampling/importance_sampling_ratio/min": 0.4764190912246704, "sampling/sampling_logp_difference/max": 0.7413458824157715, "sampling/sampling_logp_difference/mean": 0.020288333296775818, "step": 639, "step_time": 13.684441913999763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1850.0, "completions/max_terminated_length": 1850.0, "completions/mean_length": 405.8125, "completions/mean_terminated_length": 405.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10484265047125518, "epoch": 0.0128, "frac_reward_zero_std": 0.0, "grad_norm": 0.5224176645278931, "kl": 0.5686115138232708, "learning_rate": 4.062365688944147e-06, "loss": 0.0259, "num_tokens": 30741729.0, "reward": 0.15000000596046448, "reward_std": 0.3420078158378601, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.35103532671928406, "sampling/importance_sampling_ratio/max": 1.2113524675369263, "sampling/importance_sampling_ratio/mean": 1.0001336336135864, "sampling/importance_sampling_ratio/min": 0.5260409712791443, "sampling/sampling_logp_difference/max": 0.6791567802429199, "sampling/sampling_logp_difference/mean": 0.01710602454841137, "step": 640, "step_time": 17.138458484001603 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1733.0, "completions/max_terminated_length": 1733.0, "completions/mean_length": 324.875, "completions/mean_terminated_length": 324.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07974468288011849, "epoch": 0.01282, "frac_reward_zero_std": 0.0, "grad_norm": 0.192866250872612, "kl": 0.7297353483736515, "learning_rate": 4.051318187787231e-06, "loss": 0.0175, "num_tokens": 30791518.0, "reward": 0.14375001192092896, "reward_std": 0.26020190119743347, "rewards/rollout_reward_func/mean": 0.14375001192092896, "rewards/rollout_reward_func/std": 0.2793281376361847, "sampling/importance_sampling_ratio/max": 1.4849098920822144, "sampling/importance_sampling_ratio/mean": 1.0328576564788818, "sampling/importance_sampling_ratio/min": 0.6912645101547241, "sampling/sampling_logp_difference/max": 0.37071657180786133, "sampling/sampling_logp_difference/mean": 0.013999786227941513, "step": 641, "step_time": 16.11719099099355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1535.0, "completions/max_terminated_length": 1535.0, "completions/mean_length": 356.90625, "completions/mean_terminated_length": 356.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14731439761817455, "epoch": 0.01284, "frac_reward_zero_std": 0.0, "grad_norm": 0.3725206255912781, "kl": 0.6178732682019472, "learning_rate": 4.040299688729734e-06, "loss": 0.0447, "num_tokens": 30842467.0, "reward": 0.15625, "reward_std": 0.24588096141815186, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.2381887286901474, "sampling/importance_sampling_ratio/max": 1.401324987411499, "sampling/importance_sampling_ratio/mean": 1.027167797088623, "sampling/importance_sampling_ratio/min": 0.6910053491592407, "sampling/sampling_logp_difference/max": 0.3696413040161133, "sampling/sampling_logp_difference/mean": 0.018131811171770096, "step": 642, "step_time": 15.397428242995375 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 276.40625, "completions/mean_terminated_length": 276.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06497865589335561, "epoch": 0.01286, "frac_reward_zero_std": 0.0, "grad_norm": 0.09339235723018646, "kl": 2.3835243051871657, "learning_rate": 4.029310337113106e-06, "loss": 0.0282, "num_tokens": 30891529.0, "reward": 0.14687500894069672, "reward_std": 0.23116852343082428, "rewards/rollout_reward_func/mean": 0.14687500894069672, "rewards/rollout_reward_func/std": 0.24094487726688385, "sampling/importance_sampling_ratio/max": 1.1464323997497559, "sampling/importance_sampling_ratio/mean": 0.9648295044898987, "sampling/importance_sampling_ratio/min": 0.41852766275405884, "sampling/sampling_logp_difference/max": 0.8710947036743164, "sampling/sampling_logp_difference/mean": 0.028356164693832397, "step": 643, "step_time": 14.305664946001343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1670.0, "completions/max_terminated_length": 1670.0, "completions/mean_length": 277.28125, "completions/mean_terminated_length": 277.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08568048803135753, "epoch": 0.01288, "frac_reward_zero_std": 0.0, "grad_norm": 0.6500256657600403, "kl": 3.008973811753094, "learning_rate": 4.018350277894324e-06, "loss": -0.0076, "num_tokens": 30940523.0, "reward": 0.19687499105930328, "reward_std": 0.23322367668151855, "rewards/rollout_reward_func/mean": 0.19687499105930328, "rewards/rollout_reward_func/std": 0.2533445656299591, "sampling/importance_sampling_ratio/max": 1.3089977502822876, "sampling/importance_sampling_ratio/mean": 1.0099761486053467, "sampling/importance_sampling_ratio/min": 0.3286125361919403, "sampling/sampling_logp_difference/max": 1.1127653121948242, "sampling/sampling_logp_difference/mean": 0.02061634324491024, "step": 644, "step_time": 15.821390015000361 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1045.0, "completions/max_terminated_length": 1045.0, "completions/mean_length": 229.65625, "completions/mean_terminated_length": 229.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06724301283247769, "epoch": 0.0129, "frac_reward_zero_std": 0.0, "grad_norm": 0.2007841169834137, "kl": 1.7571579236537218, "learning_rate": 4.007419655643977e-06, "loss": -0.0005, "num_tokens": 30985940.0, "reward": 0.22500000894069672, "reward_std": 0.2188940942287445, "rewards/rollout_reward_func/mean": 0.22500000894069672, "rewards/rollout_reward_func/std": 0.22143229842185974, "sampling/importance_sampling_ratio/max": 1.43014395236969, "sampling/importance_sampling_ratio/mean": 0.9935792684555054, "sampling/importance_sampling_ratio/min": 0.4280616044998169, "sampling/sampling_logp_difference/max": 0.8485026359558105, "sampling/sampling_logp_difference/mean": 0.025652073323726654, "step": 645, "step_time": 13.705462328995054 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 338.59375, "completions/mean_terminated_length": 338.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09631117945536971, "epoch": 0.01292, "frac_reward_zero_std": 0.0, "grad_norm": 0.34412211179733276, "kl": 0.6005287226289511, "learning_rate": 3.996518614544363e-06, "loss": -0.0029, "num_tokens": 31037447.0, "reward": 0.10312500596046448, "reward_std": 0.2782372832298279, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.2741372287273407, "sampling/importance_sampling_ratio/max": 1.226919412612915, "sampling/importance_sampling_ratio/mean": 0.9982923269271851, "sampling/importance_sampling_ratio/min": 0.5605020523071289, "sampling/sampling_logp_difference/max": 0.5789693593978882, "sampling/sampling_logp_difference/mean": 0.021927598863840103, "step": 646, "step_time": 14.1768445839989 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 219.84375, "completions/mean_terminated_length": 219.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06462823180481791, "epoch": 0.01294, "frac_reward_zero_std": 0.0, "grad_norm": 0.29254329204559326, "kl": 0.44958228059113026, "learning_rate": 3.985647298387584e-06, "loss": -0.0035, "num_tokens": 31082861.0, "reward": 0.140625, "reward_std": 0.25503769516944885, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.2486860603094101, "sampling/importance_sampling_ratio/max": 1.0447131395339966, "sampling/importance_sampling_ratio/mean": 0.9885344505310059, "sampling/importance_sampling_ratio/min": 0.637403130531311, "sampling/sampling_logp_difference/max": 0.45032739639282227, "sampling/sampling_logp_difference/mean": 0.013454818166792393, "step": 647, "step_time": 13.573915518005379 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1207.0, "completions/max_terminated_length": 1207.0, "completions/mean_length": 243.71875, "completions/mean_terminated_length": 243.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07309845765121281, "epoch": 0.01296, "frac_reward_zero_std": 0.0, "grad_norm": 0.30393561720848083, "kl": 0.4297513822093606, "learning_rate": 3.9748058505736474e-06, "loss": 0.0158, "num_tokens": 31129783.0, "reward": 0.12812501192092896, "reward_std": 0.2422930896282196, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.2555631101131439, "sampling/importance_sampling_ratio/max": 1.0880706310272217, "sampling/importance_sampling_ratio/mean": 0.9924541115760803, "sampling/importance_sampling_ratio/min": 0.4771564304828644, "sampling/sampling_logp_difference/max": 0.7399709224700928, "sampling/sampling_logp_difference/mean": 0.01378958486020565, "step": 648, "step_time": 14.13725491399964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1261.0, "completions/max_terminated_length": 1261.0, "completions/mean_length": 302.3125, "completions/mean_terminated_length": 302.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08127485611476004, "epoch": 0.01298, "frac_reward_zero_std": 0.0, "grad_norm": 0.5256503820419312, "kl": 0.7179849706590176, "learning_rate": 3.9639944141085785e-06, "loss": 0.0027, "num_tokens": 31179029.0, "reward": 0.15625, "reward_std": 0.25272297859191895, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.2526472806930542, "sampling/importance_sampling_ratio/max": 1.1738585233688354, "sampling/importance_sampling_ratio/mean": 0.9763051867485046, "sampling/importance_sampling_ratio/min": 0.4359237849712372, "sampling/sampling_logp_difference/max": 0.943976879119873, "sampling/sampling_logp_difference/mean": 0.025682775303721428, "step": 649, "step_time": 14.725559658003476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 334.46875, "completions/mean_terminated_length": 334.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0778485283954069, "epoch": 0.013, "frac_reward_zero_std": 0.0, "grad_norm": 0.3825784921646118, "kl": 1.4548332681879401, "learning_rate": 3.953213131602535e-06, "loss": -0.0145, "num_tokens": 31229945.0, "reward": 0.09375, "reward_std": 0.2670055627822876, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.2758418023586273, "sampling/importance_sampling_ratio/max": 1.2963500022888184, "sampling/importance_sampling_ratio/mean": 1.0025815963745117, "sampling/importance_sampling_ratio/min": 0.4966755211353302, "sampling/sampling_logp_difference/max": 0.7281889915466309, "sampling/sampling_logp_difference/mean": 0.01516060158610344, "step": 650, "step_time": 12.50087194499065 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1688.0, "completions/max_terminated_length": 1688.0, "completions/mean_length": 368.1875, "completions/mean_terminated_length": 368.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06813476176466793, "epoch": 0.01302, "frac_reward_zero_std": 0.0, "grad_norm": 0.3183026611804962, "kl": 0.5040226485580206, "learning_rate": 3.942462145267919e-06, "loss": 0.0106, "num_tokens": 31281958.0, "reward": 0.15937499701976776, "reward_std": 0.3174784779548645, "rewards/rollout_reward_func/mean": 0.15937499701976776, "rewards/rollout_reward_func/std": 0.30676183104515076, "sampling/importance_sampling_ratio/max": 1.2613762617111206, "sampling/importance_sampling_ratio/mean": 1.009900450706482, "sampling/importance_sampling_ratio/min": 0.5322263836860657, "sampling/sampling_logp_difference/max": 0.6415805816650391, "sampling/sampling_logp_difference/mean": 0.012866201810538769, "step": 651, "step_time": 16.58673006700701 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 293.03125, "completions/mean_terminated_length": 293.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06949014030396938, "epoch": 0.01304, "frac_reward_zero_std": 0.0, "grad_norm": 0.4379934072494507, "kl": 0.6172162648290396, "learning_rate": 3.93174159691751e-06, "loss": -0.0155, "num_tokens": 31331219.0, "reward": 0.14687499403953552, "reward_std": 0.3127886652946472, "rewards/rollout_reward_func/mean": 0.14687499403953552, "rewards/rollout_reward_func/std": 0.30372354388237, "sampling/importance_sampling_ratio/max": 1.2582595348358154, "sampling/importance_sampling_ratio/mean": 0.9581536650657654, "sampling/importance_sampling_ratio/min": 0.4787554442882538, "sampling/sampling_logp_difference/max": 0.737701416015625, "sampling/sampling_logp_difference/mean": 0.031954921782016754, "step": 652, "step_time": 14.53389841500757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 240.90625, "completions/mean_terminated_length": 240.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08208745217416435, "epoch": 0.01306, "frac_reward_zero_std": 0.0, "grad_norm": 0.35142290592193604, "kl": 1.4798208437860012, "learning_rate": 3.921051627962586e-06, "loss": 0.0008, "num_tokens": 31377471.0, "reward": 0.14374999701976776, "reward_std": 0.26285237073898315, "rewards/rollout_reward_func/mean": 0.14374999701976776, "rewards/rollout_reward_func/std": 0.2526472806930542, "sampling/importance_sampling_ratio/max": 1.1903350353240967, "sampling/importance_sampling_ratio/mean": 1.0167791843414307, "sampling/importance_sampling_ratio/min": 0.5202246308326721, "sampling/sampling_logp_difference/max": 0.6647405624389648, "sampling/sampling_logp_difference/mean": 0.021319925785064697, "step": 653, "step_time": 13.578565229006927 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1625.0, "completions/max_terminated_length": 1625.0, "completions/mean_length": 330.9375, "completions/mean_terminated_length": 330.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07700180425308645, "epoch": 0.01308, "frac_reward_zero_std": 0.0, "grad_norm": 0.33953261375427246, "kl": 0.5959988627582788, "learning_rate": 3.9103923794110645e-06, "loss": 0.0054, "num_tokens": 31427874.0, "reward": 0.15312501788139343, "reward_std": 0.26126331090927124, "rewards/rollout_reward_func/mean": 0.15312501788139343, "rewards/rollout_reward_func/std": 0.2699873149394989, "sampling/importance_sampling_ratio/max": 1.3994896411895752, "sampling/importance_sampling_ratio/mean": 1.0343286991119385, "sampling/importance_sampling_ratio/min": 0.9604280591011047, "sampling/sampling_logp_difference/max": 0.27504730224609375, "sampling/sampling_logp_difference/mean": 0.008540188893675804, "step": 654, "step_time": 16.42297099600546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 294.6875, "completions/mean_terminated_length": 294.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09125672653317451, "epoch": 0.0131, "frac_reward_zero_std": 0.0, "grad_norm": 0.8052202463150024, "kl": 5.214190953411162, "learning_rate": 3.89976399186564e-06, "loss": 0.0016, "num_tokens": 31476845.0, "reward": 0.07500000298023224, "reward_std": 0.33187657594680786, "rewards/rollout_reward_func/mean": 0.07500000298023224, "rewards/rollout_reward_func/std": 0.34172239899635315, "sampling/importance_sampling_ratio/max": 1.1841617822647095, "sampling/importance_sampling_ratio/mean": 0.9873553514480591, "sampling/importance_sampling_ratio/min": 0.36959102749824524, "sampling/sampling_logp_difference/max": 0.9954195022583008, "sampling/sampling_logp_difference/mean": 0.025986291468143463, "step": 655, "step_time": 14.750112170997454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1018.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 258.875, "completions/mean_terminated_length": 258.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08258496259804815, "epoch": 0.01312, "frac_reward_zero_std": 0.0, "grad_norm": 0.21226297318935394, "kl": 0.5708547215908766, "learning_rate": 3.8891666055219305e-06, "loss": 0.027, "num_tokens": 31523846.0, "reward": 0.09687499701976776, "reward_std": 0.2673876881599426, "rewards/rollout_reward_func/mean": 0.09687499701976776, "rewards/rollout_reward_func/std": 0.26938924193382263, "sampling/importance_sampling_ratio/max": 1.1176389455795288, "sampling/importance_sampling_ratio/mean": 1.0159763097763062, "sampling/importance_sampling_ratio/min": 0.8537948131561279, "sampling/sampling_logp_difference/max": 0.15997496247291565, "sampling/sampling_logp_difference/mean": 0.008163321763277054, "step": 656, "step_time": 13.389903619001416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 309.46875, "completions/mean_terminated_length": 309.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07930002920329571, "epoch": 0.01314, "frac_reward_zero_std": 0.0, "grad_norm": 0.3421439826488495, "kl": 1.9774435497820377, "learning_rate": 3.878600360166621e-06, "loss": -0.0189, "num_tokens": 31574254.0, "reward": 0.08125000447034836, "reward_std": 0.33879250288009644, "rewards/rollout_reward_func/mean": 0.08125000447034836, "rewards/rollout_reward_func/std": 0.3306274712085724, "sampling/importance_sampling_ratio/max": 1.2090047597885132, "sampling/importance_sampling_ratio/mean": 0.9886438846588135, "sampling/importance_sampling_ratio/min": 0.41655704379081726, "sampling/sampling_logp_difference/max": 0.8638019561767578, "sampling/sampling_logp_difference/mean": 0.021448833867907524, "step": 657, "step_time": 14.417632505002985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1616.0, "completions/max_terminated_length": 1616.0, "completions/mean_length": 360.90625, "completions/mean_terminated_length": 360.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08179692458361387, "epoch": 0.01316, "frac_reward_zero_std": 0.0, "grad_norm": 0.16981595754623413, "kl": 0.7096973340958357, "learning_rate": 3.868065395175636e-06, "loss": 0.0636, "num_tokens": 31624531.0, "reward": 0.10000000894069672, "reward_std": 0.2864142954349518, "rewards/rollout_reward_func/mean": 0.10000000894069672, "rewards/rollout_reward_func/std": 0.3005371689796448, "sampling/importance_sampling_ratio/max": 1.3651905059814453, "sampling/importance_sampling_ratio/mean": 1.0182863473892212, "sampling/importance_sampling_ratio/min": 0.5297560095787048, "sampling/sampling_logp_difference/max": 0.5523500442504883, "sampling/sampling_logp_difference/mean": 0.013720067217946053, "step": 658, "step_time": 16.176967031002278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 281.375, "completions/mean_terminated_length": 281.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06856737029738724, "epoch": 0.01318, "frac_reward_zero_std": 0.0, "grad_norm": 0.3457256555557251, "kl": 0.6202033143490553, "learning_rate": 3.857561849512283e-06, "loss": 0.035, "num_tokens": 31671771.0, "reward": 0.16562500596046448, "reward_std": 0.196319580078125, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.216436505317688, "sampling/importance_sampling_ratio/max": 1.3074442148208618, "sampling/importance_sampling_ratio/mean": 1.0166175365447998, "sampling/importance_sampling_ratio/min": 0.8875840902328491, "sampling/sampling_logp_difference/max": 0.19965702295303345, "sampling/sampling_logp_difference/mean": 0.009408483281731606, "step": 659, "step_time": 13.708710924001934 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 211.25, "completions/mean_terminated_length": 211.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07080847630277276, "epoch": 0.0132, "frac_reward_zero_std": 0.0, "grad_norm": 0.8501601219177246, "kl": 1.2092772126197815, "learning_rate": 3.847089861725429e-06, "loss": 0.0286, "num_tokens": 31716496.0, "reward": 0.16562500596046448, "reward_std": 0.24762141704559326, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.2707330286502838, "sampling/importance_sampling_ratio/max": 1.3928637504577637, "sampling/importance_sampling_ratio/mean": 0.9841927886009216, "sampling/importance_sampling_ratio/min": 0.4268806576728821, "sampling/sampling_logp_difference/max": 0.8638277053833008, "sampling/sampling_logp_difference/mean": 0.03199849650263786, "step": 660, "step_time": 14.32770051600528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1189.0, "completions/max_terminated_length": 1189.0, "completions/mean_length": 355.375, "completions/mean_terminated_length": 355.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09985569235868752, "epoch": 0.01322, "frac_reward_zero_std": 0.0, "grad_norm": 0.42654040455818176, "kl": 0.6713935062289238, "learning_rate": 3.836649569947673e-06, "loss": 0.0059, "num_tokens": 31768194.0, "reward": 0.12812501192092896, "reward_std": 0.270664244890213, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.27384287118911743, "sampling/importance_sampling_ratio/max": 1.1940232515335083, "sampling/importance_sampling_ratio/mean": 1.0053483247756958, "sampling/importance_sampling_ratio/min": 0.5656666159629822, "sampling/sampling_logp_difference/max": 0.5716145038604736, "sampling/sampling_logp_difference/mean": 0.018378201872110367, "step": 661, "step_time": 13.9817340329937 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 292.59375, "completions/mean_terminated_length": 292.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0777157531119883, "epoch": 0.01324, "frac_reward_zero_std": 0.0, "grad_norm": 0.27327096462249756, "kl": 2.7613928392529488, "learning_rate": 3.82624111189352e-06, "loss": -0.009, "num_tokens": 31816459.0, "reward": 0.16562500596046448, "reward_std": 0.2633688747882843, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.28010293841362, "sampling/importance_sampling_ratio/max": 1.2989530563354492, "sampling/importance_sampling_ratio/mean": 1.0183641910552979, "sampling/importance_sampling_ratio/min": 0.47846198081970215, "sampling/sampling_logp_difference/max": 0.7372479438781738, "sampling/sampling_logp_difference/mean": 0.01607545278966427, "step": 662, "step_time": 13.907536621991312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 346.40625, "completions/mean_terminated_length": 346.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09649794665165246, "epoch": 0.01326, "frac_reward_zero_std": 0.0, "grad_norm": 0.43508487939834595, "kl": 0.47156103514134884, "learning_rate": 3.81586462485757e-06, "loss": 0.0055, "num_tokens": 31867729.0, "reward": 0.18437500298023224, "reward_std": 0.243630513548851, "rewards/rollout_reward_func/mean": 0.18437500298023224, "rewards/rollout_reward_func/std": 0.24111218750476837, "sampling/importance_sampling_ratio/max": 1.2703680992126465, "sampling/importance_sampling_ratio/mean": 1.032846212387085, "sampling/importance_sampling_ratio/min": 0.7554313540458679, "sampling/sampling_logp_difference/max": 0.28278565406799316, "sampling/sampling_logp_difference/mean": 0.013846293091773987, "step": 663, "step_time": 14.267257498999243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 281.15625, "completions/mean_terminated_length": 281.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06726809032261372, "epoch": 0.01328, "frac_reward_zero_std": 0.0, "grad_norm": 0.7497368454933167, "kl": 4.794479693286121, "learning_rate": 3.8055202457127015e-06, "loss": -0.0087, "num_tokens": 31916758.0, "reward": 0.125, "reward_std": 0.2766566872596741, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.2782433331012726, "sampling/importance_sampling_ratio/max": 1.1234406232833862, "sampling/importance_sampling_ratio/mean": 0.9642957448959351, "sampling/importance_sampling_ratio/min": 0.48574161529541016, "sampling/sampling_logp_difference/max": 0.739051342010498, "sampling/sampling_logp_difference/mean": 0.027902118861675262, "step": 664, "step_time": 13.871896709009889 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 874.0, "completions/max_terminated_length": 874.0, "completions/mean_length": 237.09375, "completions/mean_terminated_length": 237.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08330749813467264, "epoch": 0.0133, "frac_reward_zero_std": 0.0, "grad_norm": 0.28414636850357056, "kl": 0.5422333292663097, "learning_rate": 3.795208110908265e-06, "loss": 0.0033, "num_tokens": 31963004.0, "reward": 0.171875, "reward_std": 0.23110955953598022, "rewards/rollout_reward_func/mean": 0.171875, "rewards/rollout_reward_func/std": 0.23587088286876678, "sampling/importance_sampling_ratio/max": 1.4157122373580933, "sampling/importance_sampling_ratio/mean": 1.0259772539138794, "sampling/importance_sampling_ratio/min": 0.7634701132774353, "sampling/sampling_logp_difference/max": 0.340948224067688, "sampling/sampling_logp_difference/mean": 0.012220838107168674, "step": 665, "step_time": 13.106692668003234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 233.875, "completions/mean_terminated_length": 233.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08415647456422448, "epoch": 0.01332, "frac_reward_zero_std": 0.0, "grad_norm": 0.40997394919395447, "kl": 0.496432987973094, "learning_rate": 3.784928356468293e-06, "loss": 0.0234, "num_tokens": 32010145.0, "reward": 0.14687500894069672, "reward_std": 0.2883329391479492, "rewards/rollout_reward_func/mean": 0.14687500894069672, "rewards/rollout_reward_func/std": 0.28958743810653687, "sampling/importance_sampling_ratio/max": 1.1683992147445679, "sampling/importance_sampling_ratio/mean": 1.0165915489196777, "sampling/importance_sampling_ratio/min": 0.8351106643676758, "sampling/sampling_logp_difference/max": 0.23735499382019043, "sampling/sampling_logp_difference/mean": 0.009771620854735374, "step": 666, "step_time": 14.596313027999713 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1625.0, "completions/max_terminated_length": 1625.0, "completions/mean_length": 294.875, "completions/mean_terminated_length": 294.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08866592589765787, "epoch": 0.01334, "frac_reward_zero_std": 0.0, "grad_norm": 0.23943597078323364, "kl": 0.9161999709904194, "learning_rate": 3.7746811179896937e-06, "loss": -0.0021, "num_tokens": 32057887.0, "reward": 0.17812500894069672, "reward_std": 0.2085741013288498, "rewards/rollout_reward_func/mean": 0.17812500894069672, "rewards/rollout_reward_func/std": 0.3087272047996521, "sampling/importance_sampling_ratio/max": 1.243952751159668, "sampling/importance_sampling_ratio/mean": 0.986772894859314, "sampling/importance_sampling_ratio/min": 0.43910330533981323, "sampling/sampling_logp_difference/max": 0.8525745868682861, "sampling/sampling_logp_difference/mean": 0.024132728576660156, "step": 667, "step_time": 15.772111885009508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 259.5625, "completions/mean_terminated_length": 259.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07432576594874263, "epoch": 0.01336, "frac_reward_zero_std": 0.0, "grad_norm": 0.12271731346845627, "kl": 0.5383207760751247, "learning_rate": 3.7644665306404726e-06, "loss": 0.002, "num_tokens": 32106455.0, "reward": 0.15312500298023224, "reward_std": 0.21621441841125488, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.2257380336523056, "sampling/importance_sampling_ratio/max": 1.1102581024169922, "sampling/importance_sampling_ratio/mean": 1.0198917388916016, "sampling/importance_sampling_ratio/min": 0.9993532299995422, "sampling/sampling_logp_difference/max": 0.10366427898406982, "sampling/sampling_logp_difference/mean": 0.005808438174426556, "step": 668, "step_time": 13.589821924004355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1652.0, "completions/max_terminated_length": 1652.0, "completions/mean_length": 312.53125, "completions/mean_terminated_length": 312.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09857139433734119, "epoch": 0.01338, "frac_reward_zero_std": 0.0, "grad_norm": 0.35192808508872986, "kl": 1.0466829910874367, "learning_rate": 3.7542847291579375e-06, "loss": 0.0188, "num_tokens": 32154975.0, "reward": 0.11874999850988388, "reward_std": 0.2939509153366089, "rewards/rollout_reward_func/mean": 0.11874999850988388, "rewards/rollout_reward_func/std": 0.2912016212940216, "sampling/importance_sampling_ratio/max": 1.2613810300827026, "sampling/importance_sampling_ratio/mean": 1.0068122148513794, "sampling/importance_sampling_ratio/min": 0.5480850338935852, "sampling/sampling_logp_difference/max": 0.6011903285980225, "sampling/sampling_logp_difference/mean": 0.022743459790945053, "step": 669, "step_time": 16.18749024299177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 274.0625, "completions/mean_terminated_length": 274.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08202972472645342, "epoch": 0.0134, "frac_reward_zero_std": 0.0, "grad_norm": 1.0114582777023315, "kl": 0.7862195856869221, "learning_rate": 3.744135847846937e-06, "loss": 0.0329, "num_tokens": 32202795.0, "reward": 0.10625000298023224, "reward_std": 0.2638593316078186, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.27231088280677795, "sampling/importance_sampling_ratio/max": 1.4504001140594482, "sampling/importance_sampling_ratio/mean": 1.0382039546966553, "sampling/importance_sampling_ratio/min": 0.8087974190711975, "sampling/sampling_logp_difference/max": 0.2413644790649414, "sampling/sampling_logp_difference/mean": 0.014911727048456669, "step": 670, "step_time": 13.700738605999504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1261.0, "completions/max_terminated_length": 1261.0, "completions/mean_length": 344.59375, "completions/mean_terminated_length": 344.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10626369202509522, "epoch": 0.01342, "frac_reward_zero_std": 0.0, "grad_norm": 0.4253480136394501, "kl": 0.5745512777939439, "learning_rate": 3.7340200205780724e-06, "loss": 0.0149, "num_tokens": 32253877.0, "reward": 0.09375, "reward_std": 0.28009259700775146, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.298315167427063, "sampling/importance_sampling_ratio/max": 1.113911509513855, "sampling/importance_sampling_ratio/mean": 0.9684915542602539, "sampling/importance_sampling_ratio/min": 0.39336198568344116, "sampling/sampling_logp_difference/max": 0.9431695938110352, "sampling/sampling_logp_difference/mean": 0.025672156363725662, "step": 671, "step_time": 14.750338513003953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1814.0, "completions/max_terminated_length": 1814.0, "completions/mean_length": 284.90625, "completions/mean_terminated_length": 284.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08972747181542218, "epoch": 0.01344, "frac_reward_zero_std": 0.0, "grad_norm": 0.6614704728126526, "kl": 1.4923964953050017, "learning_rate": 3.723937380785948e-06, "loss": 0.0351, "num_tokens": 32300693.0, "reward": 0.15937499701976776, "reward_std": 0.28359895944595337, "rewards/rollout_reward_func/mean": 0.15937499701976776, "rewards/rollout_reward_func/std": 0.29823067784309387, "sampling/importance_sampling_ratio/max": 1.1438055038452148, "sampling/importance_sampling_ratio/mean": 0.9911312460899353, "sampling/importance_sampling_ratio/min": 0.21110005676746368, "sampling/sampling_logp_difference/max": 1.5554118156433105, "sampling/sampling_logp_difference/mean": 0.026839757338166237, "step": 672, "step_time": 16.446300219002296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 303.28125, "completions/mean_terminated_length": 303.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06588433682918549, "epoch": 0.01346, "frac_reward_zero_std": 0.0, "grad_norm": 0.20162542164325714, "kl": 3.9953855127096176, "learning_rate": 3.7138880614673957e-06, "loss": 0.01, "num_tokens": 32350274.0, "reward": 0.12187500298023224, "reward_std": 0.27268263697624207, "rewards/rollout_reward_func/mean": 0.12187500298023224, "rewards/rollout_reward_func/std": 0.27207016944885254, "sampling/importance_sampling_ratio/max": 1.207758903503418, "sampling/importance_sampling_ratio/mean": 0.9695523977279663, "sampling/importance_sampling_ratio/min": 0.3958176374435425, "sampling/sampling_logp_difference/max": 0.9310364723205566, "sampling/sampling_logp_difference/mean": 0.027490392327308655, "step": 673, "step_time": 13.520105139999941 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 255.9375, "completions/mean_terminated_length": 255.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10086725722067058, "epoch": 0.01348, "frac_reward_zero_std": 0.0, "grad_norm": 0.48787054419517517, "kl": 0.5537121035158634, "learning_rate": 3.7038721951797318e-06, "loss": 0.0302, "num_tokens": 32397608.0, "reward": 0.08750000596046448, "reward_std": 0.301567018032074, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.3169918954372406, "sampling/importance_sampling_ratio/max": 1.3489993810653687, "sampling/importance_sampling_ratio/mean": 1.0248961448669434, "sampling/importance_sampling_ratio/min": 0.8160842657089233, "sampling/sampling_logp_difference/max": 0.2974250018596649, "sampling/sampling_logp_difference/mean": 0.015967894345521927, "step": 674, "step_time": 16.07134055899951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 242.03125, "completions/mean_terminated_length": 242.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07795783248730004, "epoch": 0.0135, "frac_reward_zero_std": 0.0, "grad_norm": 0.4278813600540161, "kl": 0.6380827706307173, "learning_rate": 3.6938899140390016e-06, "loss": 0.0042, "num_tokens": 32442813.0, "reward": 0.21562501788139343, "reward_std": 0.27171677350997925, "rewards/rollout_reward_func/mean": 0.21562501788139343, "rewards/rollout_reward_func/std": 0.2886110842227936, "sampling/importance_sampling_ratio/max": 1.2542940378189087, "sampling/importance_sampling_ratio/mean": 1.0127520561218262, "sampling/importance_sampling_ratio/min": 0.43884143233299255, "sampling/sampling_logp_difference/max": 0.8657922744750977, "sampling/sampling_logp_difference/mean": 0.019676025956869125, "step": 675, "step_time": 15.37078386500798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 937.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 258.03125, "completions/mean_terminated_length": 258.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07528498442843556, "epoch": 0.01352, "frac_reward_zero_std": 0.0, "grad_norm": 0.257956326007843, "kl": 0.62086390145123, "learning_rate": 3.683941349718244e-06, "loss": -0.0037, "num_tokens": 32488685.0, "reward": 0.24687498807907104, "reward_std": 0.19343560934066772, "rewards/rollout_reward_func/mean": 0.24687498807907104, "rewards/rollout_reward_func/std": 0.21400086581707, "sampling/importance_sampling_ratio/max": 1.4444854259490967, "sampling/importance_sampling_ratio/mean": 1.041522741317749, "sampling/importance_sampling_ratio/min": 1.0025521516799927, "sampling/sampling_logp_difference/max": 0.3492676019668579, "sampling/sampling_logp_difference/mean": 0.010736134834587574, "step": 676, "step_time": 12.809125278006832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 252.40625, "completions/mean_terminated_length": 252.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11712226551026106, "epoch": 0.01354, "frac_reward_zero_std": 0.0, "grad_norm": 0.8287718892097473, "kl": 0.5210258513689041, "learning_rate": 3.674026633445747e-06, "loss": 0.0221, "num_tokens": 32535963.0, "reward": 0.12812501192092896, "reward_std": 0.20143654942512512, "rewards/rollout_reward_func/mean": 0.12812501192092896, "rewards/rollout_reward_func/std": 0.23173172771930695, "sampling/importance_sampling_ratio/max": 1.2152186632156372, "sampling/importance_sampling_ratio/mean": 1.0127673149108887, "sampling/importance_sampling_ratio/min": 0.5388795733451843, "sampling/sampling_logp_difference/max": 0.6383349895477295, "sampling/sampling_logp_difference/mean": 0.016974620521068573, "step": 677, "step_time": 13.579957518006267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 365.25, "completions/mean_terminated_length": 365.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10514271631836891, "epoch": 0.01356, "frac_reward_zero_std": 0.0, "grad_norm": 0.6137698888778687, "kl": 0.6650973670184612, "learning_rate": 3.6641458960033194e-06, "loss": -0.0058, "num_tokens": 32587471.0, "reward": 0.09375, "reward_std": 0.30813106894493103, "rewards/rollout_reward_func/mean": 0.09375, "rewards/rollout_reward_func/std": 0.30789345502853394, "sampling/importance_sampling_ratio/max": 1.45233952999115, "sampling/importance_sampling_ratio/mean": 1.009356141090393, "sampling/importance_sampling_ratio/min": 0.5629655718803406, "sampling/sampling_logp_difference/max": 0.579432487487793, "sampling/sampling_logp_difference/mean": 0.018955839797854424, "step": 678, "step_time": 14.947389690001728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1297.0, "completions/max_terminated_length": 1297.0, "completions/mean_length": 339.8125, "completions/mean_terminated_length": 339.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10353803867474198, "epoch": 0.01358, "frac_reward_zero_std": 0.0, "grad_norm": 0.4055614471435547, "kl": 0.5629227180033922, "learning_rate": 3.654299267724567e-06, "loss": -0.0108, "num_tokens": 32636270.0, "reward": 0.26875001192092896, "reward_std": 0.2739757001399994, "rewards/rollout_reward_func/mean": 0.26875001192092896, "rewards/rollout_reward_func/std": 0.28447744250297546, "sampling/importance_sampling_ratio/max": 1.1779053211212158, "sampling/importance_sampling_ratio/mean": 1.02399480342865, "sampling/importance_sampling_ratio/min": 0.5990576148033142, "sampling/sampling_logp_difference/max": 0.5161310434341431, "sampling/sampling_logp_difference/mean": 0.015432408079504967, "step": 679, "step_time": 15.196106089981186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1760.0, "completions/max_terminated_length": 1760.0, "completions/mean_length": 272.65625, "completions/mean_terminated_length": 272.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12022162950597703, "epoch": 0.0136, "frac_reward_zero_std": 0.0, "grad_norm": 0.14187943935394287, "kl": 2.0613303296267986, "learning_rate": 3.6444868784931752e-06, "loss": -0.0018, "num_tokens": 32684991.0, "reward": 0.20937500894069672, "reward_std": 0.2011602818965912, "rewards/rollout_reward_func/mean": 0.20937500894069672, "rewards/rollout_reward_func/std": 0.2115371823310852, "sampling/importance_sampling_ratio/max": 1.1880073547363281, "sampling/importance_sampling_ratio/mean": 0.9768663048744202, "sampling/importance_sampling_ratio/min": 1.5657535489509156e-13, "sampling/sampling_logp_difference/max": 28.727092742919922, "sampling/sampling_logp_difference/mean": 0.25216352939605713, "step": 680, "step_time": 16.369625407005515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1607.0, "completions/max_terminated_length": 1607.0, "completions/mean_length": 237.53125, "completions/mean_terminated_length": 237.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11676038894802332, "epoch": 0.01362, "frac_reward_zero_std": 0.0, "grad_norm": 0.2926667034626007, "kl": 1.8251099828630686, "learning_rate": 3.6347088577411906e-06, "loss": 0.0155, "num_tokens": 32731428.0, "reward": 0.05000000074505806, "reward_std": 0.28338539600372314, "rewards/rollout_reward_func/mean": 0.05000000074505806, "rewards/rollout_reward_func/std": 0.2723849415779114, "sampling/importance_sampling_ratio/max": 1.2635992765426636, "sampling/importance_sampling_ratio/mean": 1.008707046508789, "sampling/importance_sampling_ratio/min": 0.6058204174041748, "sampling/sampling_logp_difference/max": 0.5011405944824219, "sampling/sampling_logp_difference/mean": 0.0254819393157959, "step": 681, "step_time": 15.827309896994848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 405.5625, "completions/mean_terminated_length": 405.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11129941279068589, "epoch": 0.01364, "frac_reward_zero_std": 0.0, "grad_norm": 0.3755848705768585, "kl": 1.132162094116211, "learning_rate": 3.6249653344473162e-06, "loss": -0.0087, "num_tokens": 32784381.0, "reward": 0.20625001192092896, "reward_std": 0.24627532064914703, "rewards/rollout_reward_func/mean": 0.20625001192092896, "rewards/rollout_reward_func/std": 0.2500806450843811, "sampling/importance_sampling_ratio/max": 1.6417295932769775, "sampling/importance_sampling_ratio/mean": 1.0069620609283447, "sampling/importance_sampling_ratio/min": 0.31043723225593567, "sampling/sampling_logp_difference/max": 0.9424068927764893, "sampling/sampling_logp_difference/mean": 0.029344137758016586, "step": 682, "step_time": 15.740700339003524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 369.15625, "completions/mean_terminated_length": 369.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07234975486062467, "epoch": 0.01366, "frac_reward_zero_std": 0.0, "grad_norm": 0.23146659135818481, "kl": 0.5212669521570206, "learning_rate": 3.61525643713521e-06, "loss": 0.0232, "num_tokens": 32835841.0, "reward": 0.14375001192092896, "reward_std": 0.29273927211761475, "rewards/rollout_reward_func/mean": 0.14375001192092896, "rewards/rollout_reward_func/std": 0.29175499081611633, "sampling/importance_sampling_ratio/max": 1.1352155208587646, "sampling/importance_sampling_ratio/mean": 1.022540807723999, "sampling/importance_sampling_ratio/min": 0.9701084494590759, "sampling/sampling_logp_difference/max": 0.08926302194595337, "sampling/sampling_logp_difference/mean": 0.00711361039429903, "step": 683, "step_time": 15.990708081997582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 246.09375, "completions/mean_terminated_length": 246.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0825917290057987, "epoch": 0.01368, "frac_reward_zero_std": 0.0, "grad_norm": 0.31209683418273926, "kl": 0.5109286680817604, "learning_rate": 3.6055822938717933e-06, "loss": 0.007, "num_tokens": 32882375.0, "reward": 0.19375000894069672, "reward_std": 0.2504085302352905, "rewards/rollout_reward_func/mean": 0.19375000894069672, "rewards/rollout_reward_func/std": 0.2626631557941437, "sampling/importance_sampling_ratio/max": 1.1683056354522705, "sampling/importance_sampling_ratio/mean": 1.0186707973480225, "sampling/importance_sampling_ratio/min": 0.9730969667434692, "sampling/sampling_logp_difference/max": 0.12285497784614563, "sampling/sampling_logp_difference/mean": 0.007878436706960201, "step": 684, "step_time": 13.458848242000386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 410.75, "completions/mean_terminated_length": 410.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08003069972619414, "epoch": 0.0137, "frac_reward_zero_std": 0.0, "grad_norm": 0.18829675018787384, "kl": 0.5195521013811231, "learning_rate": 3.595943032265554e-06, "loss": 0.0189, "num_tokens": 32935480.0, "reward": 0.13437500596046448, "reward_std": 0.31333714723587036, "rewards/rollout_reward_func/mean": 0.13437500596046448, "rewards/rollout_reward_func/std": 0.3137796223163605, "sampling/importance_sampling_ratio/max": 1.6671992540359497, "sampling/importance_sampling_ratio/mean": 1.019416093826294, "sampling/importance_sampling_ratio/min": 0.5994005799293518, "sampling/sampling_logp_difference/max": 0.5333631038665771, "sampling/sampling_logp_difference/mean": 0.017055554315447807, "step": 685, "step_time": 15.85512477001248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 350.625, "completions/mean_terminated_length": 350.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09006659290753305, "epoch": 0.01372, "frac_reward_zero_std": 0.0, "grad_norm": 0.5704389214515686, "kl": 3.155047943815589, "learning_rate": 3.5863387794648695e-06, "loss": 0.0318, "num_tokens": 32985536.0, "reward": 0.16249999403953552, "reward_std": 0.3454320728778839, "rewards/rollout_reward_func/mean": 0.16249999403953552, "rewards/rollout_reward_func/std": 0.34242966771125793, "sampling/importance_sampling_ratio/max": 1.4569616317749023, "sampling/importance_sampling_ratio/mean": 1.039618968963623, "sampling/importance_sampling_ratio/min": 0.8694828748703003, "sampling/sampling_logp_difference/max": 0.3763771057128906, "sampling/sampling_logp_difference/mean": 0.014297252520918846, "step": 686, "step_time": 15.179415957001765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 379.875, "completions/mean_terminated_length": 379.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12037304206751287, "epoch": 0.01374, "frac_reward_zero_std": 0.0, "grad_norm": 0.42523500323295593, "kl": 0.5296046268194914, "learning_rate": 3.5767696621563247e-06, "loss": 0.0491, "num_tokens": 33037256.0, "reward": 0.08750000596046448, "reward_std": 0.29596835374832153, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.30874353647232056, "sampling/importance_sampling_ratio/max": 2.9485936164855957, "sampling/importance_sampling_ratio/mean": 1.1040501594543457, "sampling/importance_sampling_ratio/min": 0.6990205645561218, "sampling/sampling_logp_difference/max": 1.0776941776275635, "sampling/sampling_logp_difference/mean": 0.029063351452350616, "step": 687, "step_time": 15.37230338999143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 573.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 244.5, "completions/mean_terminated_length": 244.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06826267577707767, "epoch": 0.01376, "frac_reward_zero_std": 0.0, "grad_norm": 0.1347615271806717, "kl": 1.0079071763902903, "learning_rate": 3.5672358065630486e-06, "loss": 0.0053, "num_tokens": 33085456.0, "reward": 0.15625, "reward_std": 0.22555457055568695, "rewards/rollout_reward_func/mean": 0.15625, "rewards/rollout_reward_func/std": 0.2227938324213028, "sampling/importance_sampling_ratio/max": 1.1155260801315308, "sampling/importance_sampling_ratio/mean": 0.9935876131057739, "sampling/importance_sampling_ratio/min": 0.5417547821998596, "sampling/sampling_logp_difference/max": 0.6181578636169434, "sampling/sampling_logp_difference/mean": 0.015611762180924416, "step": 688, "step_time": 11.887280796992854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 372.78125, "completions/mean_terminated_length": 372.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09369233390316367, "epoch": 0.01378, "frac_reward_zero_std": 0.0, "grad_norm": 0.5775775909423828, "kl": 0.7692460119724274, "learning_rate": 3.5577373384430397e-06, "loss": -0.0099, "num_tokens": 33138049.0, "reward": 0.20937499403953552, "reward_std": 0.23371851444244385, "rewards/rollout_reward_func/mean": 0.20937499403953552, "rewards/rollout_reward_func/std": 0.2704349756240845, "sampling/importance_sampling_ratio/max": 1.1627837419509888, "sampling/importance_sampling_ratio/mean": 1.0179334878921509, "sampling/importance_sampling_ratio/min": 0.478729784488678, "sampling/sampling_logp_difference/max": 0.7423539161682129, "sampling/sampling_logp_difference/mean": 0.018280068412423134, "step": 689, "step_time": 15.54312473200116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 255.5, "completions/mean_terminated_length": 255.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0964001554530114, "epoch": 0.0138, "frac_reward_zero_std": 0.0, "grad_norm": 0.3482077717781067, "kl": 3.1153259333223104, "learning_rate": 3.5482743830875137e-06, "loss": -0.034, "num_tokens": 33185740.0, "reward": 0.05312500149011612, "reward_std": 0.2934950590133667, "rewards/rollout_reward_func/mean": 0.05312500149011612, "rewards/rollout_reward_func/std": 0.30478376150131226, "sampling/importance_sampling_ratio/max": 1.169960618019104, "sampling/importance_sampling_ratio/mean": 0.9555801153182983, "sampling/importance_sampling_ratio/min": 0.4811622202396393, "sampling/sampling_logp_difference/max": 0.731583833694458, "sampling/sampling_logp_difference/mean": 0.02741929702460766, "step": 690, "step_time": 15.163878658997419 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 298.21875, "completions/mean_terminated_length": 298.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07876141672022641, "epoch": 0.01382, "frac_reward_zero_std": 0.0, "grad_norm": 0.2617143392562866, "kl": 0.6845591645687819, "learning_rate": 3.538847065319248e-06, "loss": 0.0272, "num_tokens": 33233938.0, "reward": 0.15312500298023224, "reward_std": 0.2430848479270935, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.26148998737335205, "sampling/importance_sampling_ratio/max": 1.1703546047210693, "sampling/importance_sampling_ratio/mean": 0.996581494808197, "sampling/importance_sampling_ratio/min": 0.5868226885795593, "sampling/sampling_logp_difference/max": 0.5329830646514893, "sampling/sampling_logp_difference/mean": 0.015468236990272999, "step": 691, "step_time": 14.019503802002873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1261.0, "completions/max_terminated_length": 1261.0, "completions/mean_length": 326.65625, "completions/mean_terminated_length": 326.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11380802583880723, "epoch": 0.01384, "frac_reward_zero_std": 0.0, "grad_norm": 0.44877639412879944, "kl": 0.796719616279006, "learning_rate": 3.5294555094909387e-06, "loss": 0.0156, "num_tokens": 33282906.0, "reward": 0.10625000298023224, "reward_std": 0.29731863737106323, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.30684396624565125, "sampling/importance_sampling_ratio/max": 1.1619279384613037, "sampling/importance_sampling_ratio/mean": 0.9766950607299805, "sampling/importance_sampling_ratio/min": 0.5011063814163208, "sampling/sampling_logp_difference/max": 0.7244415283203125, "sampling/sampling_logp_difference/mean": 0.02375468984246254, "step": 692, "step_time": 14.708127718004107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1679.0, "completions/max_terminated_length": 1679.0, "completions/mean_length": 417.21875, "completions/mean_terminated_length": 417.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08731440384872258, "epoch": 0.01386, "frac_reward_zero_std": 0.0, "grad_norm": 0.3232080936431885, "kl": 0.7322747437283397, "learning_rate": 3.520099839483554e-06, "loss": 0.0188, "num_tokens": 33336904.0, "reward": 0.14375001192092896, "reward_std": 0.3230913579463959, "rewards/rollout_reward_func/mean": 0.14375001192092896, "rewards/rollout_reward_func/std": 0.3320877254009247, "sampling/importance_sampling_ratio/max": 1.1277306079864502, "sampling/importance_sampling_ratio/mean": 0.9840373396873474, "sampling/importance_sampling_ratio/min": 0.6733002662658691, "sampling/sampling_logp_difference/max": 0.45261311531066895, "sampling/sampling_logp_difference/mean": 0.017012840136885643, "step": 693, "step_time": 16.317915658990387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1679.0, "completions/max_terminated_length": 1679.0, "completions/mean_length": 497.03125, "completions/mean_terminated_length": 497.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.13379867770709097, "epoch": 0.01388, "frac_reward_zero_std": 0.0, "grad_norm": 0.5033994913101196, "kl": 0.3994760075584054, "learning_rate": 3.5107801787047088e-06, "loss": -0.0319, "num_tokens": 33394295.0, "reward": 0.046875, "reward_std": 0.298280268907547, "rewards/rollout_reward_func/mean": 0.046875, "rewards/rollout_reward_func/std": 0.3232739567756653, "sampling/importance_sampling_ratio/max": 1.2650582790374756, "sampling/importance_sampling_ratio/mean": 0.9872687458992004, "sampling/importance_sampling_ratio/min": 0.4079256057739258, "sampling/sampling_logp_difference/max": 0.8966047763824463, "sampling/sampling_logp_difference/mean": 0.02919115498661995, "step": 694, "step_time": 16.266324378000718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 364.28125, "completions/mean_terminated_length": 364.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.12668911623768508, "epoch": 0.0139, "frac_reward_zero_std": 0.0, "grad_norm": 0.5617536306381226, "kl": 0.5953518459573388, "learning_rate": 3.5014966500870244e-06, "loss": 0.0112, "num_tokens": 33445204.0, "reward": 0.07187500596046448, "reward_std": 0.3252097964286804, "rewards/rollout_reward_func/mean": 0.07187500596046448, "rewards/rollout_reward_func/std": 0.3314040005207062, "sampling/importance_sampling_ratio/max": 1.3978315591812134, "sampling/importance_sampling_ratio/mean": 1.0399513244628906, "sampling/importance_sampling_ratio/min": 0.44190847873687744, "sampling/sampling_logp_difference/max": 0.8642520904541016, "sampling/sampling_logp_difference/mean": 0.026321785524487495, "step": 695, "step_time": 13.803115433995117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 352.1875, "completions/mean_terminated_length": 352.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08732667262665927, "epoch": 0.01392, "frac_reward_zero_std": 0.0, "grad_norm": 0.5586869716644287, "kl": 0.7635071650147438, "learning_rate": 3.492249376086523e-06, "loss": 0.0207, "num_tokens": 33495913.0, "reward": 0.18437500298023224, "reward_std": 0.309187650680542, "rewards/rollout_reward_func/mean": 0.18437500298023224, "rewards/rollout_reward_func/std": 0.31633931398391724, "sampling/importance_sampling_ratio/max": 1.1739972829818726, "sampling/importance_sampling_ratio/mean": 1.029107928276062, "sampling/importance_sampling_ratio/min": 0.9969422817230225, "sampling/sampling_logp_difference/max": 0.13495603203773499, "sampling/sampling_logp_difference/mean": 0.009190712124109268, "step": 696, "step_time": 14.161507421995339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1234.0, "completions/max_terminated_length": 1234.0, "completions/mean_length": 317.15625, "completions/mean_terminated_length": 317.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0795465970877558, "epoch": 0.01394, "frac_reward_zero_std": 0.0, "grad_norm": 0.22958944737911224, "kl": 0.7412720276042819, "learning_rate": 3.483038478680996e-06, "loss": 0.0182, "num_tokens": 33546539.0, "reward": 0.18437498807907104, "reward_std": 0.2777353823184967, "rewards/rollout_reward_func/mean": 0.18437498807907104, "rewards/rollout_reward_func/std": 0.2974183261394501, "sampling/importance_sampling_ratio/max": 1.216314435005188, "sampling/importance_sampling_ratio/mean": 1.024932861328125, "sampling/importance_sampling_ratio/min": 0.6611936092376709, "sampling/sampling_logp_difference/max": 0.4279690980911255, "sampling/sampling_logp_difference/mean": 0.013067751191556454, "step": 697, "step_time": 14.673378098003013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 330.96875, "completions/mean_terminated_length": 330.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07211396400816739, "epoch": 0.01396, "frac_reward_zero_std": 0.0, "grad_norm": 0.6958369612693787, "kl": 0.7349012065678835, "learning_rate": 3.4738640793684103e-06, "loss": 0.0175, "num_tokens": 33596163.0, "reward": 0.10312500596046448, "reward_std": 0.2895151674747467, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.29565057158470154, "sampling/importance_sampling_ratio/max": 1.4030171632766724, "sampling/importance_sampling_ratio/mean": 1.0153546333312988, "sampling/importance_sampling_ratio/min": 0.6509854793548584, "sampling/sampling_logp_difference/max": 0.43219709396362305, "sampling/sampling_logp_difference/mean": 0.014960877597332, "step": 698, "step_time": 15.52032565299669 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 326.6875, "completions/mean_terminated_length": 326.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07471470604650676, "epoch": 0.01398, "frac_reward_zero_std": 0.0, "grad_norm": 0.24158436059951782, "kl": 0.4675626661628485, "learning_rate": 3.464726299165293e-06, "loss": 0.0228, "num_tokens": 33647069.0, "reward": 0.18125000596046448, "reward_std": 0.23072972893714905, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.24813823401927948, "sampling/importance_sampling_ratio/max": 1.232552170753479, "sampling/importance_sampling_ratio/mean": 1.0302796363830566, "sampling/importance_sampling_ratio/min": 0.8612293601036072, "sampling/sampling_logp_difference/max": 0.26557719707489014, "sampling/sampling_logp_difference/mean": 0.011879839934408665, "step": 699, "step_time": 15.743725741005619 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 865.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 290.71875, "completions/mean_terminated_length": 290.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0755882877856493, "epoch": 0.014, "frac_reward_zero_std": 0.0, "grad_norm": 0.3403424620628357, "kl": 2.649496177211404, "learning_rate": 3.4556252586051436e-06, "loss": -0.0321, "num_tokens": 33697265.0, "reward": 0.15312498807907104, "reward_std": 0.30220067501068115, "rewards/rollout_reward_func/mean": 0.15312498807907104, "rewards/rollout_reward_func/std": 0.3089883327484131, "sampling/importance_sampling_ratio/max": 1.3691390752792358, "sampling/importance_sampling_ratio/mean": 1.0164927244186401, "sampling/importance_sampling_ratio/min": 0.3712345063686371, "sampling/sampling_logp_difference/max": 0.990847110748291, "sampling/sampling_logp_difference/mean": 0.018294919282197952, "step": 700, "step_time": 13.795197176990769 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 429.15625, "completions/mean_terminated_length": 429.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09603456873446703, "epoch": 0.01402, "frac_reward_zero_std": 0.0, "grad_norm": 0.2582351863384247, "kl": 1.0801123548299074, "learning_rate": 3.4465610777368364e-06, "loss": 0.021, "num_tokens": 33751523.0, "reward": 0.11874999850988388, "reward_std": 0.3393564224243164, "rewards/rollout_reward_func/mean": 0.11874999850988388, "rewards/rollout_reward_func/std": 0.33929479122161865, "sampling/importance_sampling_ratio/max": 1.2172080278396606, "sampling/importance_sampling_ratio/mean": 0.9836253523826599, "sampling/importance_sampling_ratio/min": 0.44071701169013977, "sampling/sampling_logp_difference/max": 0.8191823959350586, "sampling/sampling_logp_difference/mean": 0.022136982530355453, "step": 701, "step_time": 15.554808584009152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 280.375, "completions/mean_terminated_length": 280.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09351817704737186, "epoch": 0.01404, "frac_reward_zero_std": 0.0, "grad_norm": 0.661385715007782, "kl": 3.4141231533139944, "learning_rate": 3.4375338761230474e-06, "loss": 0.0223, "num_tokens": 33799860.0, "reward": 0.0937500074505806, "reward_std": 0.3017808794975281, "rewards/rollout_reward_func/mean": 0.0937500074505806, "rewards/rollout_reward_func/std": 0.32719480991363525, "sampling/importance_sampling_ratio/max": 1.1124299764633179, "sampling/importance_sampling_ratio/mean": 0.9527473449707031, "sampling/importance_sampling_ratio/min": 0.42044657468795776, "sampling/sampling_logp_difference/max": 0.8715014457702637, "sampling/sampling_logp_difference/mean": 0.03040476329624653, "step": 702, "step_time": 14.82722499900774 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 775.0, "completions/max_terminated_length": 775.0, "completions/mean_length": 244.9375, "completions/mean_terminated_length": 244.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06949070910923183, "epoch": 0.01406, "frac_reward_zero_std": 0.0, "grad_norm": 0.1526055634021759, "kl": 0.5109985833987594, "learning_rate": 3.428543772838667e-06, "loss": -0.0028, "num_tokens": 33847024.0, "reward": 0.2343750149011612, "reward_std": 0.21508051455020905, "rewards/rollout_reward_func/mean": 0.2343750149011612, "rewards/rollout_reward_func/std": 0.2208624929189682, "sampling/importance_sampling_ratio/max": 1.1508541107177734, "sampling/importance_sampling_ratio/mean": 1.0090718269348145, "sampling/importance_sampling_ratio/min": 0.8742736577987671, "sampling/sampling_logp_difference/max": 0.13652706146240234, "sampling/sampling_logp_difference/mean": 0.0067527685314416885, "step": 703, "step_time": 13.208115176003048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 201.03125, "completions/mean_terminated_length": 201.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08565344521775842, "epoch": 0.01408, "frac_reward_zero_std": 0.0, "grad_norm": 0.3009915053844452, "kl": 0.6737431343644857, "learning_rate": 3.4195908864692364e-06, "loss": 0.0089, "num_tokens": 33891697.0, "reward": 0.15937501192092896, "reward_std": 0.2503586411476135, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.27691778540611267, "sampling/importance_sampling_ratio/max": 1.099825382232666, "sampling/importance_sampling_ratio/mean": 0.9841320514678955, "sampling/importance_sampling_ratio/min": 0.6067962050437927, "sampling/sampling_logp_difference/max": 0.5438127517700195, "sampling/sampling_logp_difference/mean": 0.016893260180950165, "step": 704, "step_time": 13.65384328899745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 329.125, "completions/mean_terminated_length": 329.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08966473862528801, "epoch": 0.0141, "frac_reward_zero_std": 0.0, "grad_norm": 0.5287218689918518, "kl": 0.47113770712167025, "learning_rate": 3.410675335109376e-06, "loss": 0.0311, "num_tokens": 33941554.0, "reward": 0.13124999403953552, "reward_std": 0.2713460922241211, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.28447744250297546, "sampling/importance_sampling_ratio/max": 1.117904782295227, "sampling/importance_sampling_ratio/mean": 0.9999544620513916, "sampling/importance_sampling_ratio/min": 0.6507667303085327, "sampling/sampling_logp_difference/max": 0.4408869743347168, "sampling/sampling_logp_difference/mean": 0.012076318264007568, "step": 705, "step_time": 15.884239425999112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1346.0, "completions/max_terminated_length": 1346.0, "completions/mean_length": 350.59375, "completions/mean_terminated_length": 350.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0684196890797466, "epoch": 0.01412, "frac_reward_zero_std": 0.0, "grad_norm": 0.446332722902298, "kl": 0.7526134178042412, "learning_rate": 3.4017972363612374e-06, "loss": -0.0078, "num_tokens": 33991997.0, "reward": 0.22812500596046448, "reward_std": 0.324827641248703, "rewards/rollout_reward_func/mean": 0.22812500596046448, "rewards/rollout_reward_func/std": 0.32152295112609863, "sampling/importance_sampling_ratio/max": 1.2482420206069946, "sampling/importance_sampling_ratio/mean": 0.9887474775314331, "sampling/importance_sampling_ratio/min": 0.464999794960022, "sampling/sampling_logp_difference/max": 0.7656669616699219, "sampling/sampling_logp_difference/mean": 0.01963498443365097, "step": 706, "step_time": 15.401190879994829 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 338.0, "completions/mean_terminated_length": 338.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0823581179138273, "epoch": 0.01414, "frac_reward_zero_std": 0.0, "grad_norm": 0.7764065861701965, "kl": 2.3312543127685785, "learning_rate": 3.392956707332945e-06, "loss": 0.0088, "num_tokens": 34041161.0, "reward": 0.140625, "reward_std": 0.30925828218460083, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.31710317730903625, "sampling/importance_sampling_ratio/max": 1.2946810722351074, "sampling/importance_sampling_ratio/mean": 1.0130952596664429, "sampling/importance_sampling_ratio/min": 0.538442075252533, "sampling/sampling_logp_difference/max": 0.6191139221191406, "sampling/sampling_logp_difference/mean": 0.021101247519254684, "step": 707, "step_time": 14.647579579002922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2088.0, "completions/max_terminated_length": 2088.0, "completions/mean_length": 363.84375, "completions/mean_terminated_length": 363.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0869907233864069, "epoch": 0.01416, "frac_reward_zero_std": 0.0, "grad_norm": 13.020991325378418, "kl": 32.96610198915005, "learning_rate": 3.384153864637052e-06, "loss": 0.0923, "num_tokens": 34093171.0, "reward": 0.08750000596046448, "reward_std": 0.37765270471572876, "rewards/rollout_reward_func/mean": 0.08750000596046448, "rewards/rollout_reward_func/std": 0.36786216497421265, "sampling/importance_sampling_ratio/max": 1.0628063678741455, "sampling/importance_sampling_ratio/mean": 0.9708037972450256, "sampling/importance_sampling_ratio/min": 0.3300143778324127, "sampling/sampling_logp_difference/max": 1.11773681640625, "sampling/sampling_logp_difference/mean": 0.02478284202516079, "step": 708, "step_time": 17.458523251996667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 384.5625, "completions/mean_terminated_length": 384.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09421240561641753, "epoch": 0.01418, "frac_reward_zero_std": 0.0, "grad_norm": 0.8150283694267273, "kl": 0.5730013158172369, "learning_rate": 3.3753888243890024e-06, "loss": 0.0192, "num_tokens": 34145857.0, "reward": 0.125, "reward_std": 0.24449367821216583, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.2565276622772217, "sampling/importance_sampling_ratio/max": 1.1152527332305908, "sampling/importance_sampling_ratio/mean": 0.9740138649940491, "sampling/importance_sampling_ratio/min": 0.5376220941543579, "sampling/sampling_logp_difference/max": 0.7441525459289551, "sampling/sampling_logp_difference/mean": 0.022145375609397888, "step": 709, "step_time": 15.87685645700185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 252.25, "completions/mean_terminated_length": 252.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07539339968934655, "epoch": 0.0142, "frac_reward_zero_std": 0.0, "grad_norm": 0.4899541735649109, "kl": 0.4441987741738558, "learning_rate": 3.3666617022056063e-06, "loss": -0.0194, "num_tokens": 34193290.0, "reward": 0.21250000596046448, "reward_std": 0.25325626134872437, "rewards/rollout_reward_func/mean": 0.21250000596046448, "rewards/rollout_reward_func/std": 0.2624328136444092, "sampling/importance_sampling_ratio/max": 1.266735553741455, "sampling/importance_sampling_ratio/mean": 1.0208009481430054, "sampling/importance_sampling_ratio/min": 0.731755793094635, "sampling/sampling_logp_difference/max": 0.3278803825378418, "sampling/sampling_logp_difference/mean": 0.013732721097767353, "step": 710, "step_time": 15.237217556004907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 271.125, "completions/mean_terminated_length": 271.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07294421223923564, "epoch": 0.01422, "frac_reward_zero_std": 0.0, "grad_norm": 0.19239719212055206, "kl": 0.7615929283201694, "learning_rate": 3.357972613203503e-06, "loss": -0.005, "num_tokens": 34240147.0, "reward": 0.20000000298023224, "reward_std": 0.21164819598197937, "rewards/rollout_reward_func/mean": 0.20000000298023224, "rewards/rollout_reward_func/std": 0.22143229842185974, "sampling/importance_sampling_ratio/max": 1.1329641342163086, "sampling/importance_sampling_ratio/mean": 0.9842849969863892, "sampling/importance_sampling_ratio/min": 0.574006199836731, "sampling/sampling_logp_difference/max": 0.6070672273635864, "sampling/sampling_logp_difference/mean": 0.01972920075058937, "step": 711, "step_time": 14.176331315993593 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 294.40625, "completions/mean_terminated_length": 294.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08496927295345813, "epoch": 0.01424, "frac_reward_zero_std": 0.0, "grad_norm": 0.2774604856967926, "kl": 0.5064425822347403, "learning_rate": 3.3493216719976483e-06, "loss": 0.0084, "num_tokens": 34289420.0, "reward": 0.10624999552965164, "reward_std": 0.2917824983596802, "rewards/rollout_reward_func/mean": 0.10624999552965164, "rewards/rollout_reward_func/std": 0.2928585410118103, "sampling/importance_sampling_ratio/max": 1.3154032230377197, "sampling/importance_sampling_ratio/mean": 1.026364803314209, "sampling/importance_sampling_ratio/min": 0.8968915343284607, "sampling/sampling_logp_difference/max": 0.1622391939163208, "sampling/sampling_logp_difference/mean": 0.012123001739382744, "step": 712, "step_time": 13.621341602003668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 244.125, "completions/mean_terminated_length": 244.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07355754543095827, "epoch": 0.01426, "frac_reward_zero_std": 0.0, "grad_norm": 0.29396581649780273, "kl": 0.5470712436363101, "learning_rate": 3.340708992699808e-06, "loss": -0.0264, "num_tokens": 34336589.0, "reward": 0.18125000596046448, "reward_std": 0.2897203266620636, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.29009175300598145, "sampling/importance_sampling_ratio/max": 1.2839018106460571, "sampling/importance_sampling_ratio/mean": 1.024305820465088, "sampling/importance_sampling_ratio/min": 0.9253867268562317, "sampling/sampling_logp_difference/max": 0.2002103328704834, "sampling/sampling_logp_difference/mean": 0.009272102266550064, "step": 713, "step_time": 14.781427829002496 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 220.25, "completions/mean_terminated_length": 220.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07504082168452442, "epoch": 0.01428, "frac_reward_zero_std": 0.0, "grad_norm": 0.4661902189254761, "kl": 0.4897626219317317, "learning_rate": 3.3321346889170426e-06, "loss": 0.0356, "num_tokens": 34381547.0, "reward": 0.19687499105930328, "reward_std": 0.17667707800865173, "rewards/rollout_reward_func/mean": 0.19687499105930328, "rewards/rollout_reward_func/std": 0.19590051472187042, "sampling/importance_sampling_ratio/max": 1.4382742643356323, "sampling/importance_sampling_ratio/mean": 1.0234179496765137, "sampling/importance_sampling_ratio/min": 0.7182426452636719, "sampling/sampling_logp_difference/max": 0.33092832565307617, "sampling/sampling_logp_difference/mean": 0.01288563385605812, "step": 714, "step_time": 13.591775415003212 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1751.0, "completions/max_terminated_length": 1751.0, "completions/mean_length": 296.96875, "completions/mean_terminated_length": 296.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10049642878584564, "epoch": 0.0143, "frac_reward_zero_std": 0.0, "grad_norm": 0.7418267130851746, "kl": 3.3650187384337187, "learning_rate": 3.323598873750216e-06, "loss": 0.0169, "num_tokens": 34430343.0, "reward": 0.0312500037252903, "reward_std": 0.27408236265182495, "rewards/rollout_reward_func/mean": 0.0312500037252903, "rewards/rollout_reward_func/std": 0.3073691725730896, "sampling/importance_sampling_ratio/max": 1.2493853569030762, "sampling/importance_sampling_ratio/mean": 0.9924595355987549, "sampling/importance_sampling_ratio/min": 0.4733350872993469, "sampling/sampling_logp_difference/max": 0.7478446960449219, "sampling/sampling_logp_difference/mean": 0.029921947047114372, "step": 715, "step_time": 16.225810680003633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1099.0, "completions/max_terminated_length": 1099.0, "completions/mean_length": 238.21875, "completions/mean_terminated_length": 238.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05410571314860135, "epoch": 0.01432, "frac_reward_zero_std": 0.0, "grad_norm": 0.14682811498641968, "kl": 1.219583879224956, "learning_rate": 3.315101659792499e-06, "loss": 0.0213, "num_tokens": 34476111.0, "reward": 0.19062499701976776, "reward_std": 0.25672510266304016, "rewards/rollout_reward_func/mean": 0.19062499701976776, "rewards/rollout_reward_func/std": 0.2619521915912628, "sampling/importance_sampling_ratio/max": 1.137575626373291, "sampling/importance_sampling_ratio/mean": 1.0042978525161743, "sampling/importance_sampling_ratio/min": 0.47677555680274963, "sampling/sampling_logp_difference/max": 0.7427778244018555, "sampling/sampling_logp_difference/mean": 0.017146170139312744, "step": 716, "step_time": 13.909550552001747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 196.9375, "completions/mean_terminated_length": 196.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.09015425364486873, "epoch": 0.01434, "frac_reward_zero_std": 0.0, "grad_norm": 0.3190912902355194, "kl": 2.396584768779576, "learning_rate": 3.306643159127889e-06, "loss": 0.0049, "num_tokens": 34521085.0, "reward": 0.11875000596046448, "reward_std": 0.2841157615184784, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.2810550332069397, "sampling/importance_sampling_ratio/max": 1.560556173324585, "sampling/importance_sampling_ratio/mean": 1.0021389722824097, "sampling/importance_sampling_ratio/min": 0.5055513978004456, "sampling/sampling_logp_difference/max": 0.5580329895019531, "sampling/sampling_logp_difference/mean": 0.03180261328816414, "step": 717, "step_time": 11.434985248997691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 296.53125, "completions/mean_terminated_length": 296.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07893447508104146, "epoch": 0.01436, "frac_reward_zero_std": 0.0, "grad_norm": 0.4257518947124481, "kl": 0.6012880858033895, "learning_rate": 3.2982234833297273e-06, "loss": 0.0283, "num_tokens": 34569669.0, "reward": 0.109375, "reward_std": 0.27466052770614624, "rewards/rollout_reward_func/mean": 0.109375, "rewards/rollout_reward_func/std": 0.2644036114215851, "sampling/importance_sampling_ratio/max": 1.4240257740020752, "sampling/importance_sampling_ratio/mean": 1.0466704368591309, "sampling/importance_sampling_ratio/min": 1.002715826034546, "sampling/sampling_logp_difference/max": 0.3535726070404053, "sampling/sampling_logp_difference/mean": 0.012767171487212181, "step": 718, "step_time": 13.59782870699928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1090.0, "completions/max_terminated_length": 1090.0, "completions/mean_length": 315.65625, "completions/mean_terminated_length": 315.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06402449845336378, "epoch": 0.01438, "frac_reward_zero_std": 0.0, "grad_norm": 0.27705857157707214, "kl": 0.43981787469238043, "learning_rate": 3.28984274345923e-06, "loss": -0.0292, "num_tokens": 34618969.0, "reward": 0.24375000596046448, "reward_std": 0.2637477517127991, "rewards/rollout_reward_func/mean": 0.24375000596046448, "rewards/rollout_reward_func/std": 0.2781708836555481, "sampling/importance_sampling_ratio/max": 1.1378659009933472, "sampling/importance_sampling_ratio/mean": 1.0013062953948975, "sampling/importance_sampling_ratio/min": 0.6095014810562134, "sampling/sampling_logp_difference/max": 0.4950428009033203, "sampling/sampling_logp_difference/mean": 0.011632155627012253, "step": 719, "step_time": 14.282849235005415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1616.0, "completions/max_terminated_length": 1616.0, "completions/mean_length": 244.28125, "completions/mean_terminated_length": 244.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06073898682370782, "epoch": 0.0144, "frac_reward_zero_std": 0.0, "grad_norm": 0.5517264008522034, "kl": 2.289986737072468, "learning_rate": 3.281501050064022e-06, "loss": 0.0238, "num_tokens": 34666035.0, "reward": 0.10312500596046448, "reward_std": 0.31456810235977173, "rewards/rollout_reward_func/mean": 0.10312500596046448, "rewards/rollout_reward_func/std": 0.31570133566856384, "sampling/importance_sampling_ratio/max": 1.2293790578842163, "sampling/importance_sampling_ratio/mean": 0.9695577621459961, "sampling/importance_sampling_ratio/min": 0.4221303164958954, "sampling/sampling_logp_difference/max": 0.8687725067138672, "sampling/sampling_logp_difference/mean": 0.02886948734521866, "step": 720, "step_time": 16.652040898985433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 254.625, "completions/mean_terminated_length": 254.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06555091915652156, "epoch": 0.01442, "frac_reward_zero_std": 0.0, "grad_norm": 0.19265593588352203, "kl": 1.3792360350489616, "learning_rate": 3.2731985131766765e-06, "loss": -0.0199, "num_tokens": 34714665.0, "reward": 0.13124999403953552, "reward_std": 0.3121383488178253, "rewards/rollout_reward_func/mean": 0.13124999403953552, "rewards/rollout_reward_func/std": 0.30314216017723083, "sampling/importance_sampling_ratio/max": 1.2042630910873413, "sampling/importance_sampling_ratio/mean": 0.9740340709686279, "sampling/importance_sampling_ratio/min": 0.506336510181427, "sampling/sampling_logp_difference/max": 0.6806514263153076, "sampling/sampling_logp_difference/mean": 0.02351711317896843, "step": 721, "step_time": 13.573348940004507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 344.625, "completions/mean_terminated_length": 344.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07397077360656112, "epoch": 0.01444, "frac_reward_zero_std": 0.0, "grad_norm": 0.6038925051689148, "kl": 0.47863837517797947, "learning_rate": 3.2649352423132675e-06, "loss": 0.0296, "num_tokens": 34764028.0, "reward": 0.16562500596046448, "reward_std": 0.19239819049835205, "rewards/rollout_reward_func/mean": 0.16562500596046448, "rewards/rollout_reward_func/std": 0.19278833270072937, "sampling/importance_sampling_ratio/max": 1.0789875984191895, "sampling/importance_sampling_ratio/mean": 0.9856247901916504, "sampling/importance_sampling_ratio/min": 0.5509700179100037, "sampling/sampling_logp_difference/max": 0.59907066822052, "sampling/sampling_logp_difference/mean": 0.015758713707327843, "step": 722, "step_time": 14.079034214992134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1243.0, "completions/max_terminated_length": 1243.0, "completions/mean_length": 286.5625, "completions/mean_terminated_length": 286.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08725519012659788, "epoch": 0.01446, "frac_reward_zero_std": 0.0, "grad_norm": 0.5294378399848938, "kl": 0.5287246564403176, "learning_rate": 3.256711346471925e-06, "loss": 0.0004, "num_tokens": 34813023.0, "reward": 0.125, "reward_std": 0.2946776747703552, "rewards/rollout_reward_func/mean": 0.125, "rewards/rollout_reward_func/std": 0.3005371391773224, "sampling/importance_sampling_ratio/max": 1.5912435054779053, "sampling/importance_sampling_ratio/mean": 1.0413482189178467, "sampling/importance_sampling_ratio/min": 0.4308127760887146, "sampling/sampling_logp_difference/max": 0.8451676368713379, "sampling/sampling_logp_difference/mean": 0.026697153225541115, "step": 723, "step_time": 14.621824716999981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 274.0625, "completions/mean_terminated_length": 274.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06052038073539734, "epoch": 0.01448, "frac_reward_zero_std": 0.0, "grad_norm": 0.4391830265522003, "kl": 0.5094269514083862, "learning_rate": 3.2485269341313947e-06, "loss": 0.0095, "num_tokens": 34859879.0, "reward": 0.1875, "reward_std": 0.257315993309021, "rewards/rollout_reward_func/mean": 0.1875, "rewards/rollout_reward_func/std": 0.2612006664276123, "sampling/importance_sampling_ratio/max": 1.0641206502914429, "sampling/importance_sampling_ratio/mean": 0.9930245876312256, "sampling/importance_sampling_ratio/min": 0.39917048811912537, "sampling/sampling_logp_difference/max": 0.9235348701477051, "sampling/sampling_logp_difference/mean": 0.016527222469449043, "step": 724, "step_time": 14.09663207300764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 391.75, "completions/mean_terminated_length": 391.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06157432496547699, "epoch": 0.0145, "frac_reward_zero_std": 0.0, "grad_norm": 0.5629384517669678, "kl": 0.510784886777401, "learning_rate": 3.240382113249607e-06, "loss": 0.0191, "num_tokens": 34913322.0, "reward": 0.171875, "reward_std": 0.241410031914711, "rewards/rollout_reward_func/mean": 0.171875, "rewards/rollout_reward_func/std": 0.26788824796676636, "sampling/importance_sampling_ratio/max": 1.4172852039337158, "sampling/importance_sampling_ratio/mean": 1.0320062637329102, "sampling/importance_sampling_ratio/min": 0.8658204078674316, "sampling/sampling_logp_difference/max": 0.346996009349823, "sampling/sampling_logp_difference/mean": 0.009846426546573639, "step": 725, "step_time": 14.795887436001067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 260.03125, "completions/mean_terminated_length": 260.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07800773368217051, "epoch": 0.01452, "frac_reward_zero_std": 0.0, "grad_norm": 0.5596160292625427, "kl": 1.1575311291962862, "learning_rate": 3.232276991262254e-06, "loss": 0.0031, "num_tokens": 34959649.0, "reward": 0.15937501192092896, "reward_std": 0.2737002968788147, "rewards/rollout_reward_func/mean": 0.15937501192092896, "rewards/rollout_reward_func/std": 0.2625671923160553, "sampling/importance_sampling_ratio/max": 1.57026207447052, "sampling/importance_sampling_ratio/mean": 0.9892104268074036, "sampling/importance_sampling_ratio/min": 0.4440559446811676, "sampling/sampling_logp_difference/max": 0.8520469665527344, "sampling/sampling_logp_difference/mean": 0.03506273403763771, "step": 726, "step_time": 13.923785376999149 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1027.0, "completions/max_terminated_length": 1027.0, "completions/mean_length": 295.78125, "completions/mean_terminated_length": 295.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0722880718531087, "epoch": 0.01454, "frac_reward_zero_std": 0.0, "grad_norm": 0.3297201991081238, "kl": 0.5547821410000324, "learning_rate": 3.224211675081378e-06, "loss": 0.0108, "num_tokens": 35009786.0, "reward": 0.17500001192092896, "reward_std": 0.2376476228237152, "rewards/rollout_reward_func/mean": 0.17500001192092896, "rewards/rollout_reward_func/std": 0.2488684058189392, "sampling/importance_sampling_ratio/max": 1.1327816247940063, "sampling/importance_sampling_ratio/mean": 1.0124753713607788, "sampling/importance_sampling_ratio/min": 0.8234166502952576, "sampling/sampling_logp_difference/max": 0.20631647109985352, "sampling/sampling_logp_difference/mean": 0.009878152050077915, "step": 727, "step_time": 13.48651940900163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 340.8125, "completions/mean_terminated_length": 340.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07217709207907319, "epoch": 0.01456, "frac_reward_zero_std": 0.0, "grad_norm": 0.4170498549938202, "kl": 0.7630284233018756, "learning_rate": 3.216186271093948e-06, "loss": 0.0046, "num_tokens": 35060558.0, "reward": 0.08125000447034836, "reward_std": 0.30681389570236206, "rewards/rollout_reward_func/mean": 0.08125000447034836, "rewards/rollout_reward_func/std": 0.32472074031829834, "sampling/importance_sampling_ratio/max": 1.3242299556732178, "sampling/importance_sampling_ratio/mean": 0.977851152420044, "sampling/importance_sampling_ratio/min": 0.3061163127422333, "sampling/sampling_logp_difference/max": 1.1837859153747559, "sampling/sampling_logp_difference/mean": 0.02708788774907589, "step": 728, "step_time": 14.02589261601679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 337.125, "completions/mean_terminated_length": 337.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06609660433605313, "epoch": 0.01458, "frac_reward_zero_std": 0.0, "grad_norm": 0.4479507803916931, "kl": 2.1823239400982857, "learning_rate": 3.2082008851604685e-06, "loss": 0.0602, "num_tokens": 35110105.0, "reward": 0.19374999403953552, "reward_std": 0.17934346199035645, "rewards/rollout_reward_func/mean": 0.19374999403953552, "rewards/rollout_reward_func/std": 0.18128128349781036, "sampling/importance_sampling_ratio/max": 1.3596229553222656, "sampling/importance_sampling_ratio/mean": 0.994539737701416, "sampling/importance_sampling_ratio/min": 0.2579638361930847, "sampling/sampling_logp_difference/max": 1.4188423156738281, "sampling/sampling_logp_difference/mean": 0.028023090213537216, "step": 729, "step_time": 15.471986158001528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 306.65625, "completions/mean_terminated_length": 306.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08261753304395825, "epoch": 0.0146, "frac_reward_zero_std": 0.0, "grad_norm": 0.4999549090862274, "kl": 0.6095115691423416, "learning_rate": 3.200255622613578e-06, "loss": -0.0315, "num_tokens": 35158672.0, "reward": 0.15312500298023224, "reward_std": 0.30607807636260986, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.3015918731689453, "sampling/importance_sampling_ratio/max": 1.5411735773086548, "sampling/importance_sampling_ratio/mean": 1.030234694480896, "sampling/importance_sampling_ratio/min": 0.8278226256370544, "sampling/sampling_logp_difference/max": 0.3659555912017822, "sampling/sampling_logp_difference/mean": 0.013902842067182064, "step": 730, "step_time": 15.284067952004989 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1679.0, "completions/max_terminated_length": 1679.0, "completions/mean_length": 310.4375, "completions/mean_terminated_length": 310.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07233234390150756, "epoch": 0.01462, "frac_reward_zero_std": 0.0, "grad_norm": 0.6253811717033386, "kl": 0.5159391984343529, "learning_rate": 3.192350588256662e-06, "loss": 0.0474, "num_tokens": 35207836.0, "reward": 0.1875, "reward_std": 0.2685362696647644, "rewards/rollout_reward_func/mean": 0.1875, "rewards/rollout_reward_func/std": 0.2926519215106964, "sampling/importance_sampling_ratio/max": 1.3446433544158936, "sampling/importance_sampling_ratio/mean": 1.039379596710205, "sampling/importance_sampling_ratio/min": 0.9399374127388, "sampling/sampling_logp_difference/max": 0.1642220914363861, "sampling/sampling_logp_difference/mean": 0.01221272349357605, "step": 731, "step_time": 16.435310325996397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1679.0, "completions/max_terminated_length": 1679.0, "completions/mean_length": 345.71875, "completions/mean_terminated_length": 345.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.056730193668045104, "epoch": 0.01464, "frac_reward_zero_std": 0.0, "grad_norm": 0.36221930384635925, "kl": 1.1789015494287014, "learning_rate": 3.1844858863624657e-06, "loss": 0.0381, "num_tokens": 35258334.0, "reward": 0.15000000596046448, "reward_std": 0.25872188806533813, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.2747432589530945, "sampling/importance_sampling_ratio/max": 1.0676331520080566, "sampling/importance_sampling_ratio/mean": 0.9935331344604492, "sampling/importance_sampling_ratio/min": 0.48044759035110474, "sampling/sampling_logp_difference/max": 0.7426133155822754, "sampling/sampling_logp_difference/mean": 0.010300499387085438, "step": 732, "step_time": 16.083944751000672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 273.40625, "completions/mean_terminated_length": 273.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.0834544274257496, "epoch": 0.01466, "frac_reward_zero_std": 0.0, "grad_norm": 0.5210998058319092, "kl": 3.108927959576249, "learning_rate": 3.1766616206717248e-06, "loss": 0.0514, "num_tokens": 35304975.0, "reward": 0.15312500298023224, "reward_std": 0.2946496605873108, "rewards/rollout_reward_func/mean": 0.15312500298023224, "rewards/rollout_reward_func/std": 0.28622615337371826, "sampling/importance_sampling_ratio/max": 1.5524613857269287, "sampling/importance_sampling_ratio/mean": 0.9792554378509521, "sampling/importance_sampling_ratio/min": 0.28754034638404846, "sampling/sampling_logp_difference/max": 0.8666272163391113, "sampling/sampling_logp_difference/mean": 0.03489341214299202, "step": 733, "step_time": 16.156042148009874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 177.375, "completions/mean_terminated_length": 177.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.057837845641188323, "epoch": 0.01468, "frac_reward_zero_std": 0.0, "grad_norm": 0.3735807240009308, "kl": 5.081732971593738, "learning_rate": 3.168877894391792e-06, "loss": -0.0023, "num_tokens": 35349066.0, "reward": 0.19374999403953552, "reward_std": 0.2565785050392151, "rewards/rollout_reward_func/mean": 0.19374999403953552, "rewards/rollout_reward_func/std": 0.2526472806930542, "sampling/importance_sampling_ratio/max": 1.458504319190979, "sampling/importance_sampling_ratio/mean": 1.0148353576660156, "sampling/importance_sampling_ratio/min": 0.7158404588699341, "sampling/sampling_logp_difference/max": 0.37741851806640625, "sampling/sampling_logp_difference/mean": 0.014307107776403427, "step": 734, "step_time": 13.663168212991877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 348.21875, "completions/mean_terminated_length": 348.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08232056116685271, "epoch": 0.0147, "frac_reward_zero_std": 0.0, "grad_norm": 0.5722039341926575, "kl": 0.5972781823948026, "learning_rate": 3.16113481019528e-06, "loss": 0.0324, "num_tokens": 35400696.0, "reward": 0.16249999403953552, "reward_std": 0.2638382315635681, "rewards/rollout_reward_func/mean": 0.16249999403953552, "rewards/rollout_reward_func/std": 0.2612007260322571, "sampling/importance_sampling_ratio/max": 1.6529427766799927, "sampling/importance_sampling_ratio/mean": 1.0000468492507935, "sampling/importance_sampling_ratio/min": 0.47901004552841187, "sampling/sampling_logp_difference/max": 0.7446041107177734, "sampling/sampling_logp_difference/mean": 0.022442054003477097, "step": 735, "step_time": 13.951388084002247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 1679.0, "completions/max_terminated_length": 1679.0, "completions/mean_length": 298.90625, "completions/mean_terminated_length": 298.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07889521878678352, "epoch": 0.01472, "frac_reward_zero_std": 0.0, "grad_norm": 0.05345996096730232, "kl": 0.7374656815081835, "learning_rate": 3.1534324702187026e-06, "loss": 0.0211, "num_tokens": 35448946.0, "reward": 0.1875, "reward_std": 0.27642208337783813, "rewards/rollout_reward_func/mean": 0.1875, "rewards/rollout_reward_func/std": 0.28820914030075073, "sampling/importance_sampling_ratio/max": 1.106178879737854, "sampling/importance_sampling_ratio/mean": 0.9512564539909363, "sampling/importance_sampling_ratio/min": 1.441712385030769e-07, "sampling/sampling_logp_difference/max": 15.59681510925293, "sampling/sampling_logp_difference/mean": 0.15273502469062805, "step": 736, "step_time": 16.238666244997148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1328.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 271.65625, "completions/mean_terminated_length": 271.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07411987544037402, "epoch": 0.01474, "frac_reward_zero_std": 0.0, "grad_norm": 0.4671967923641205, "kl": 0.5640728315338492, "learning_rate": 3.1457709760611314e-06, "loss": 0.0463, "num_tokens": 35495523.0, "reward": 0.17812500894069672, "reward_std": 0.22829262912273407, "rewards/rollout_reward_func/mean": 0.17812500894069672, "rewards/rollout_reward_func/std": 0.22394464910030365, "sampling/importance_sampling_ratio/max": 1.2463793754577637, "sampling/importance_sampling_ratio/mean": 1.0207566022872925, "sampling/importance_sampling_ratio/min": 0.7666367888450623, "sampling/sampling_logp_difference/max": 0.37555551528930664, "sampling/sampling_logp_difference/mean": 0.014704802073538303, "step": 737, "step_time": 14.795198290998087 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 314.09375, "completions/mean_terminated_length": 314.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06116719963029027, "epoch": 0.01476, "frac_reward_zero_std": 0.0, "grad_norm": 0.18451736867427826, "kl": 0.5325828138738871, "learning_rate": 3.1381504287828525e-06, "loss": -0.01, "num_tokens": 35545536.0, "reward": 0.17812499403953552, "reward_std": 0.3083047866821289, "rewards/rollout_reward_func/mean": 0.17812499403953552, "rewards/rollout_reward_func/std": 0.30557653307914734, "sampling/importance_sampling_ratio/max": 1.1112847328186035, "sampling/importance_sampling_ratio/mean": 0.9993399977684021, "sampling/importance_sampling_ratio/min": 0.5469672083854675, "sampling/sampling_logp_difference/max": 0.6177449226379395, "sampling/sampling_logp_difference/mean": 0.010916639119386673, "step": 738, "step_time": 13.758529834987712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 973.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 224.875, "completions/mean_terminated_length": 224.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.05718097067438066, "epoch": 0.01478, "frac_reward_zero_std": 0.0, "grad_norm": 0.12622526288032532, "kl": 0.5327123012393713, "learning_rate": 3.130570928904039e-06, "loss": 0.0263, "num_tokens": 35591138.0, "reward": 0.16875000298023224, "reward_std": 0.24125313758850098, "rewards/rollout_reward_func/mean": 0.16875000298023224, "rewards/rollout_reward_func/std": 0.25328487157821655, "sampling/importance_sampling_ratio/max": 1.1799721717834473, "sampling/importance_sampling_ratio/mean": 1.0222804546356201, "sampling/importance_sampling_ratio/min": 1.0043359994888306, "sampling/sampling_logp_difference/max": 0.1580486297607422, "sampling/sampling_logp_difference/mean": 0.0067388336174190044, "step": 739, "step_time": 13.432386776985368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 158.71875, "completions/mean_terminated_length": 158.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06801438378170133, "epoch": 0.0148, "frac_reward_zero_std": 0.0, "grad_norm": 0.21238267421722412, "kl": 5.2727034986019135, "learning_rate": 3.1230325764034134e-06, "loss": 0.0206, "num_tokens": 35633547.0, "reward": 0.18125000596046448, "reward_std": 0.2487179934978485, "rewards/rollout_reward_func/mean": 0.18125000596046448, "rewards/rollout_reward_func/std": 0.2520080506801605, "sampling/importance_sampling_ratio/max": 1.2877014875411987, "sampling/importance_sampling_ratio/mean": 1.0171754360198975, "sampling/importance_sampling_ratio/min": 0.6153978109359741, "sampling/sampling_logp_difference/max": 0.4854860305786133, "sampling/sampling_logp_difference/mean": 0.01759124919772148, "step": 740, "step_time": 13.324789451991819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 910.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 313.78125, "completions/mean_terminated_length": 313.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07437656307592988, "epoch": 0.01482, "frac_reward_zero_std": 0.0, "grad_norm": 0.2276623249053955, "kl": 1.846197634935379, "learning_rate": 3.1155354707169467e-06, "loss": -0.0081, "num_tokens": 35683031.0, "reward": 0.14375001192092896, "reward_std": 0.2852361798286438, "rewards/rollout_reward_func/mean": 0.14375001192092896, "rewards/rollout_reward_func/std": 0.287298321723938, "sampling/importance_sampling_ratio/max": 1.1991996765136719, "sampling/importance_sampling_ratio/mean": 0.9945727586746216, "sampling/importance_sampling_ratio/min": 0.42331185936927795, "sampling/sampling_logp_difference/max": 0.8595621585845947, "sampling/sampling_logp_difference/mean": 0.026869533583521843, "step": 741, "step_time": 13.590328911996039 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1225.0, "completions/max_terminated_length": 1225.0, "completions/mean_length": 322.65625, "completions/mean_terminated_length": 322.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06780850025825202, "epoch": 0.01484, "frac_reward_zero_std": 0.0, "grad_norm": 0.19318847358226776, "kl": 1.3144314419478178, "learning_rate": 3.108079710736526e-06, "loss": 0.0182, "num_tokens": 35733531.0, "reward": 0.10625000298023224, "reward_std": 0.2413165122270584, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.3262074291706085, "sampling/importance_sampling_ratio/max": 1.393316626548767, "sampling/importance_sampling_ratio/mean": 1.018615961074829, "sampling/importance_sampling_ratio/min": 0.5017217397689819, "sampling/sampling_logp_difference/max": 0.7456579208374023, "sampling/sampling_logp_difference/mean": 0.020415503531694412, "step": 742, "step_time": 14.681052355008433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 1697.0, "completions/max_terminated_length": 1697.0, "completions/mean_length": 455.125, "completions/mean_terminated_length": 455.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08750752685591578, "epoch": 0.01486, "frac_reward_zero_std": 0.0, "grad_norm": 0.29087892174720764, "kl": 0.8447283571586013, "learning_rate": 3.1006653948086677e-06, "loss": 0.0127, "num_tokens": 35789096.0, "reward": 0.11875000596046448, "reward_std": 0.31855309009552, "rewards/rollout_reward_func/mean": 0.11875000596046448, "rewards/rollout_reward_func/std": 0.31971508264541626, "sampling/importance_sampling_ratio/max": 1.1257063150405884, "sampling/importance_sampling_ratio/mean": 0.9701961278915405, "sampling/importance_sampling_ratio/min": 0.37373360991477966, "sampling/sampling_logp_difference/max": 0.5321557521820068, "sampling/sampling_logp_difference/mean": 0.02566223219037056, "step": 743, "step_time": 16.821727979997377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1144.0, "completions/max_terminated_length": 1144.0, "completions/mean_length": 371.5625, "completions/mean_terminated_length": 371.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.10254930122755468, "epoch": 0.01488, "frac_reward_zero_std": 0.0, "grad_norm": 1.354527235031128, "kl": 0.5380904972553253, "learning_rate": 3.0932926207332104e-06, "loss": 0.0022, "num_tokens": 35840940.0, "reward": 0.140625, "reward_std": 0.26419201493263245, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.2803907096385956, "sampling/importance_sampling_ratio/max": 1.2397156953811646, "sampling/importance_sampling_ratio/mean": 1.0389363765716553, "sampling/importance_sampling_ratio/min": 0.5045088529586792, "sampling/sampling_logp_difference/max": 0.6842942237854004, "sampling/sampling_logp_difference/mean": 0.024945735931396484, "step": 744, "step_time": 14.21025266999277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1108.0, "completions/max_terminated_length": 1108.0, "completions/mean_length": 345.40625, "completions/mean_terminated_length": 345.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07451425469480455, "epoch": 0.0149, "frac_reward_zero_std": 0.0, "grad_norm": 0.5365945100784302, "kl": 1.5923424689099193, "learning_rate": 3.0859614857620308e-06, "loss": -0.0025, "num_tokens": 35891224.0, "reward": 0.15000000596046448, "reward_std": 0.2957053482532501, "rewards/rollout_reward_func/mean": 0.15000000596046448, "rewards/rollout_reward_func/std": 0.286243736743927, "sampling/importance_sampling_ratio/max": 1.128164529800415, "sampling/importance_sampling_ratio/mean": 0.967120885848999, "sampling/importance_sampling_ratio/min": 0.44411423802375793, "sampling/sampling_logp_difference/max": 0.8258342742919922, "sampling/sampling_logp_difference/mean": 0.02931564673781395, "step": 745, "step_time": 14.087587534006161 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1081.0, "completions/max_terminated_length": 1081.0, "completions/mean_length": 208.625, "completions/mean_terminated_length": 208.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.07871050829999149, "epoch": 0.01492, "frac_reward_zero_std": 0.0, "grad_norm": 1.7983015775680542, "kl": 0.676696302369237, "learning_rate": 3.0786720865977503e-06, "loss": 0.034, "num_tokens": 35936216.0, "reward": 0.140625, "reward_std": 0.2573782801628113, "rewards/rollout_reward_func/mean": 0.140625, "rewards/rollout_reward_func/std": 0.27923786640167236, "sampling/importance_sampling_ratio/max": 2.261899948120117, "sampling/importance_sampling_ratio/mean": 1.034138560295105, "sampling/importance_sampling_ratio/min": 0.3185334801673889, "sampling/sampling_logp_difference/max": 1.1462130546569824, "sampling/sampling_logp_difference/mean": 0.02982713095843792, "step": 746, "step_time": 13.406921165998938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 219.96875, "completions/mean_terminated_length": 219.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08035343373194337, "epoch": 0.01494, "frac_reward_zero_std": 0.0, "grad_norm": 0.2276349812746048, "kl": 4.730347933247685, "learning_rate": 3.0714245193924757e-06, "loss": 0.0105, "num_tokens": 35983026.0, "reward": 0.08749999850988388, "reward_std": 0.2515285909175873, "rewards/rollout_reward_func/mean": 0.08749999850988388, "rewards/rollout_reward_func/std": 0.27911147475242615, "sampling/importance_sampling_ratio/max": 1.346832036972046, "sampling/importance_sampling_ratio/mean": 0.9775313138961792, "sampling/importance_sampling_ratio/min": 0.2256796956062317, "sampling/sampling_logp_difference/max": 1.4885730743408203, "sampling/sampling_logp_difference/mean": 0.04010660946369171, "step": 747, "step_time": 14.00673028199526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 222.53125, "completions/mean_terminated_length": 222.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.056863421807065606, "epoch": 0.01496, "frac_reward_zero_std": 0.0, "grad_norm": 0.05657491460442543, "kl": 1.438655935227871, "learning_rate": 3.064218879746515e-06, "loss": 0.0049, "num_tokens": 36027606.0, "reward": 0.21562501788139343, "reward_std": 0.22632242739200592, "rewards/rollout_reward_func/mean": 0.21562501788139343, "rewards/rollout_reward_func/std": 0.2356998771429062, "sampling/importance_sampling_ratio/max": 1.1254150867462158, "sampling/importance_sampling_ratio/mean": 1.0069940090179443, "sampling/importance_sampling_ratio/min": 0.4769962430000305, "sampling/sampling_logp_difference/max": 0.7402639389038086, "sampling/sampling_logp_difference/mean": 0.015927240252494812, "step": 748, "step_time": 13.71531208700253 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 409.125, "completions/mean_terminated_length": 409.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.08456945116631687, "epoch": 0.01498, "frac_reward_zero_std": 0.0, "grad_norm": 0.5110464096069336, "kl": 4.911416158080101, "learning_rate": 3.057055262707128e-06, "loss": 0.0016, "num_tokens": 36082290.0, "reward": 0.10625000298023224, "reward_std": 0.25588005781173706, "rewards/rollout_reward_func/mean": 0.10625000298023224, "rewards/rollout_reward_func/std": 0.2770088016986847, "sampling/importance_sampling_ratio/max": 2.9714479446411133, "sampling/importance_sampling_ratio/mean": 1.0355756282806396, "sampling/importance_sampling_ratio/min": 0.30359598994255066, "sampling/sampling_logp_difference/max": 1.1919307708740234, "sampling/sampling_logp_difference/mean": 0.033246636390686035, "step": 749, "step_time": 14.000790006994066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1162.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 302.4375, "completions/mean_terminated_length": 302.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06611151399556547, "epoch": 0.015, "frac_reward_zero_std": 0.0, "grad_norm": 0.37379372119903564, "kl": 1.2151980176568031, "learning_rate": 3.0499337627672667e-06, "loss": 0.0308, "num_tokens": 36130789.0, "reward": 0.17500001192092896, "reward_std": 0.20694312453269958, "rewards/rollout_reward_func/mean": 0.17500001192092896, "rewards/rollout_reward_func/std": 0.22576037049293518, "sampling/importance_sampling_ratio/max": 1.1244364976882935, "sampling/importance_sampling_ratio/mean": 0.9865869283676147, "sampling/importance_sampling_ratio/min": 0.29474741220474243, "sampling/sampling_logp_difference/max": 1.2432832717895508, "sampling/sampling_logp_difference/mean": 0.020596303045749664, "step": 750, "step_time": 14.628401365011086 } ], "logging_steps": 1.0, "max_steps": 900, "num_input_tokens_seen": 36130789, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }