{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.012, "eval_steps": 500, "global_step": 1200, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 305.71875, "completions/mean_terminated_length": 305.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.261664554476738, "epoch": 1e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.8789594769477844, "kl": 0.0, "learning_rate": 0.0, "loss": -0.0047, "num_tokens": 37107.0, "reward": -0.519927978515625, "reward_std": 0.6779485940933228, "rewards/rollout_reward_func/mean": -0.519927978515625, "rewards/rollout_reward_func/std": 0.83356773853302, "sampling/importance_sampling_ratio/max": 1.4258222579956055, "sampling/importance_sampling_ratio/mean": 0.9935756921768188, "sampling/importance_sampling_ratio/min": 0.584108829498291, "sampling/sampling_logp_difference/max": 0.37037962675094604, "sampling/sampling_logp_difference/mean": 0.043884649872779846, "step": 1, "step_time": 9.895889502000045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 1.261664554476738, "epoch": 2e-05, "grad_norm": 0.8788484930992126, "kl": 0.0, "learning_rate": 6.999999999999999e-07, "loss": -0.0047, "step": 2, "step_time": 4.858421015999852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2038.0, "completions/max_terminated_length": 2038.0, "completions/mean_length": 470.03125, "completions/mean_terminated_length": 470.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3582956194877625, "epoch": 3e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.7954035401344299, "kl": 0.002395751653239131, "learning_rate": 1.3999999999999997e-06, "loss": -0.0306, "num_tokens": 82123.0, "reward": -0.4554394483566284, "reward_std": 0.6988565921783447, "rewards/rollout_reward_func/mean": -0.4554394483566284, "rewards/rollout_reward_func/std": 0.8687976598739624, "sampling/importance_sampling_ratio/max": 1.305112361907959, "sampling/importance_sampling_ratio/mean": 1.0275447368621826, "sampling/importance_sampling_ratio/min": 0.7409394979476929, "sampling/sampling_logp_difference/max": 0.2630273699760437, "sampling/sampling_logp_difference/mean": 0.037770919501781464, "step": 3, "step_time": 10.647064562000082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 1.3649093806743622, "epoch": 4e-05, "grad_norm": 0.807948112487793, "kl": 0.0027232170978095382, "learning_rate": 2.1e-06, "loss": -0.0295, "step": 4, "step_time": 5.883250726000142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1497.0, "completions/max_terminated_length": 1497.0, "completions/mean_length": 336.875, "completions/mean_terminated_length": 336.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2589170187711716, "epoch": 5e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.8169269561767578, "kl": 0.0019431657929089852, "learning_rate": 2.7999999999999994e-06, "loss": -0.0398, "num_tokens": 120257.0, "reward": -0.6562670469284058, "reward_std": 0.6941578388214111, "rewards/rollout_reward_func/mean": -0.6562670469284058, "rewards/rollout_reward_func/std": 0.6891330480575562, "sampling/importance_sampling_ratio/max": 1.49407958984375, "sampling/importance_sampling_ratio/mean": 0.9657339453697205, "sampling/importance_sampling_ratio/min": 0.5723897218704224, "sampling/sampling_logp_difference/max": 0.23616790771484375, "sampling/sampling_logp_difference/mean": 0.04215988516807556, "step": 5, "step_time": 10.266031794000241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 1.2580353617668152, "epoch": 6e-05, "grad_norm": 0.8186739683151245, "kl": 0.0029452544258674607, "learning_rate": 3.499999999999999e-06, "loss": -0.0398, "step": 6, "step_time": 4.867109590999917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1704.0, "completions/max_terminated_length": 1704.0, "completions/mean_length": 424.5, "completions/mean_terminated_length": 424.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2683640867471695, "epoch": 7e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.7719575762748718, "kl": 0.003543023398378864, "learning_rate": 4.2e-06, "loss": 0.0034, "num_tokens": 162672.0, "reward": -0.5453242063522339, "reward_std": 0.6895341277122498, "rewards/rollout_reward_func/mean": -0.5453242063522339, "rewards/rollout_reward_func/std": 0.8311666250228882, "sampling/importance_sampling_ratio/max": 1.4253408908843994, "sampling/importance_sampling_ratio/mean": 1.0015804767608643, "sampling/importance_sampling_ratio/min": 0.622477114200592, "sampling/sampling_logp_difference/max": 0.36527371406555176, "sampling/sampling_logp_difference/mean": 0.05207929387688637, "step": 7, "step_time": 9.70477523000011 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 1.2616931945085526, "epoch": 8e-05, "grad_norm": 0.7961030602455139, "kl": 0.003623228520154953, "learning_rate": 4.9e-06, "loss": 0.003, "step": 8, "step_time": 5.2451538569998775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1578.0, "completions/max_terminated_length": 1578.0, "completions/mean_length": 417.0, "completions/mean_terminated_length": 417.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.211885616183281, "epoch": 9e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.0260124206542969, "kl": 0.008170805522240698, "learning_rate": 5.599999999999999e-06, "loss": -0.0099, "num_tokens": 204689.0, "reward": -0.501650869846344, "reward_std": 0.8058703541755676, "rewards/rollout_reward_func/mean": -0.501650869846344, "rewards/rollout_reward_func/std": 0.7887821197509766, "sampling/importance_sampling_ratio/max": 1.7708544731140137, "sampling/importance_sampling_ratio/mean": 1.0173282623291016, "sampling/importance_sampling_ratio/min": 0.2637992799282074, "sampling/sampling_logp_difference/max": 1.3825066089630127, "sampling/sampling_logp_difference/mean": 0.06186160817742348, "step": 9, "step_time": 9.367817396000078 }, { "clip_ratio/high_max": 0.023958333767950535, "clip_ratio/high_mean": 0.011979166883975267, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.022395833861082792, "entropy": 1.1711883246898651, "epoch": 0.0001, "grad_norm": 0.847097635269165, "kl": 0.012437917292118073, "learning_rate": 6.299999999999998e-06, "loss": -0.0114, "step": 10, "step_time": 5.179228130999832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 354.4375, "completions/mean_terminated_length": 354.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2003123611211777, "epoch": 0.00011, "frac_reward_zero_std": 0.0, "grad_norm": 1.2896397113800049, "kl": 0.025134430266916752, "learning_rate": 6.999999999999998e-06, "loss": -0.018, "num_tokens": 245064.0, "reward": -0.28835198283195496, "reward_std": 0.8705775737762451, "rewards/rollout_reward_func/mean": -0.28835198283195496, "rewards/rollout_reward_func/std": 0.8755360841751099, "sampling/importance_sampling_ratio/max": 1.4172606468200684, "sampling/importance_sampling_ratio/mean": 1.039669394493103, "sampling/importance_sampling_ratio/min": 0.6282202005386353, "sampling/sampling_logp_difference/max": 0.5389535427093506, "sampling/sampling_logp_difference/mean": 0.04542597383260727, "step": 11, "step_time": 9.08208966399991 }, { "clip_ratio/high_max": 0.09791666828095913, "clip_ratio/high_mean": 0.048958334140479565, "clip_ratio/low_mean": 0.06093749962747097, "clip_ratio/low_min": 0.033333334140479565, "clip_ratio/region_mean": 0.10989583469927311, "entropy": 1.1347978934645653, "epoch": 0.00012, "grad_norm": 0.6478531956672668, "kl": 0.05974239856004715, "learning_rate": 7.699999999999999e-06, "loss": -0.0199, "step": 12, "step_time": 6.223873179999714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2988.0, "completions/max_terminated_length": 2988.0, "completions/mean_length": 370.0625, "completions/mean_terminated_length": 370.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0859129577875137, "epoch": 0.00013, "frac_reward_zero_std": 0.0, "grad_norm": 1.264985203742981, "kl": 0.05964707979001105, "learning_rate": 8.4e-06, "loss": -0.0407, "num_tokens": 285190.0, "reward": -0.3505691885948181, "reward_std": 0.8377879858016968, "rewards/rollout_reward_func/mean": -0.3505691885948181, "rewards/rollout_reward_func/std": 0.8537156581878662, "sampling/importance_sampling_ratio/max": 1.537920594215393, "sampling/importance_sampling_ratio/mean": 0.9729344844818115, "sampling/importance_sampling_ratio/min": 0.608290433883667, "sampling/sampling_logp_difference/max": 0.4884665012359619, "sampling/sampling_logp_difference/mean": 0.0685725063085556, "step": 13, "step_time": 13.074789307000174 }, { "clip_ratio/high_max": 0.17708333395421505, "clip_ratio/high_mean": 0.0927083333954215, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1552083333954215, "entropy": 0.9927851781249046, "epoch": 0.00014, "grad_norm": 0.6746919751167297, "kl": 0.11430336348712444, "learning_rate": 9.1e-06, "loss": -0.0433, "step": 14, "step_time": 7.632122699999968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2110.0, "completions/max_terminated_length": 2110.0, "completions/mean_length": 556.96875, "completions/mean_terminated_length": 556.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8692496716976166, "epoch": 0.00015, "frac_reward_zero_std": 0.0, "grad_norm": 0.9823276400566101, "kl": 0.22837077733129263, "learning_rate": 9.8e-06, "loss": -0.0258, "num_tokens": 332388.0, "reward": 0.09249570965766907, "reward_std": 0.9926256537437439, "rewards/rollout_reward_func/mean": 0.09249570965766907, "rewards/rollout_reward_func/std": 0.954261302947998, "sampling/importance_sampling_ratio/max": 2.0588552951812744, "sampling/importance_sampling_ratio/mean": 0.9280470013618469, "sampling/importance_sampling_ratio/min": 0.26143553853034973, "sampling/sampling_logp_difference/max": 1.1686904430389404, "sampling/sampling_logp_difference/mean": 0.10911379009485245, "step": 15, "step_time": 11.36535592500013 }, { "clip_ratio/high_max": 0.07199754938483238, "clip_ratio/high_mean": 0.03599877469241619, "clip_ratio/low_mean": 0.0875133560039103, "clip_ratio/low_min": 0.04447115492075682, "clip_ratio/region_mean": 0.12351213116198778, "entropy": 0.7740946784615517, "epoch": 0.00016, "grad_norm": 0.7059059739112854, "kl": 0.5299063604325056, "learning_rate": 1.0499999999999998e-05, "loss": -0.0273, "step": 16, "step_time": 6.150490589000128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2317.0, "completions/max_terminated_length": 2317.0, "completions/mean_length": 198.40625, "completions/mean_terminated_length": 198.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.584294568747282, "epoch": 0.00017, "frac_reward_zero_std": 0.0, "grad_norm": 0.5173925757408142, "kl": 0.48647167533636093, "learning_rate": 1.1199999999999998e-05, "loss": -0.0191, "num_tokens": 365386.0, "reward": -0.04590633884072304, "reward_std": 0.9509698748588562, "rewards/rollout_reward_func/mean": -0.04590633884072304, "rewards/rollout_reward_func/std": 0.9301521182060242, "sampling/importance_sampling_ratio/max": 1.5783610343933105, "sampling/importance_sampling_ratio/mean": 0.9292216300964355, "sampling/importance_sampling_ratio/min": 0.28379011154174805, "sampling/sampling_logp_difference/max": 1.1246838569641113, "sampling/sampling_logp_difference/mean": 0.1568412184715271, "step": 17, "step_time": 10.849809261999894 }, { "clip_ratio/high_max": 0.01923076994717121, "clip_ratio/high_mean": 0.009615384973585606, "clip_ratio/low_mean": 0.1471153860911727, "clip_ratio/low_min": 0.04583333432674408, "clip_ratio/region_mean": 0.1567307710647583, "entropy": 0.49616197496652603, "epoch": 0.00018, "grad_norm": 0.4425205588340759, "kl": 0.9584114253520966, "learning_rate": 1.1899999999999998e-05, "loss": -0.0194, "step": 18, "step_time": 6.553163497000014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1758.0, "completions/max_terminated_length": 1758.0, "completions/mean_length": 356.875, "completions/mean_terminated_length": 356.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5646495558321476, "epoch": 0.00019, "frac_reward_zero_std": 0.0, "grad_norm": 0.8411095142364502, "kl": 1.636976963840425, "learning_rate": 1.2599999999999996e-05, "loss": -0.0382, "num_tokens": 404682.0, "reward": 0.287761390209198, "reward_std": 0.9719667434692383, "rewards/rollout_reward_func/mean": 0.287761390209198, "rewards/rollout_reward_func/std": 0.946099579334259, "sampling/importance_sampling_ratio/max": 2.437194585800171, "sampling/importance_sampling_ratio/mean": 0.8835034966468811, "sampling/importance_sampling_ratio/min": 0.10234309732913971, "sampling/sampling_logp_difference/max": 2.2794411182403564, "sampling/sampling_logp_difference/mean": 0.21359863877296448, "step": 19, "step_time": 10.355787499000144 }, { "clip_ratio/high_max": 0.03645833395421505, "clip_ratio/high_mean": 0.018229166977107525, "clip_ratio/low_mean": 0.06510416697710752, "clip_ratio/low_min": 0.026041666977107525, "clip_ratio/region_mean": 0.08333333395421505, "entropy": 0.5245265737175941, "epoch": 0.0002, "grad_norm": 1.0523966550827026, "kl": 2.180280338972807, "learning_rate": 1.3299999999999996e-05, "loss": -0.0376, "step": 20, "step_time": 5.332927825999832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1720.0, "completions/max_terminated_length": 1720.0, "completions/mean_length": 447.84375, "completions/mean_terminated_length": 447.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5796486241742969, "epoch": 0.00021, "frac_reward_zero_std": 0.0, "grad_norm": 1.080851674079895, "kl": 0.962547155097127, "learning_rate": 1.3999999999999996e-05, "loss": 0.0023, "num_tokens": 448335.0, "reward": 0.45302435755729675, "reward_std": 0.6030290126800537, "rewards/rollout_reward_func/mean": 0.45302435755729675, "rewards/rollout_reward_func/std": 0.7655353546142578, "sampling/importance_sampling_ratio/max": 2.9810757637023926, "sampling/importance_sampling_ratio/mean": 1.1785697937011719, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2141153812408447, "sampling/sampling_logp_difference/mean": 0.123544842004776, "step": 21, "step_time": 10.068056481999974 }, { "clip_ratio/high_max": 0.039549911860376596, "clip_ratio/high_mean": 0.019774955930188298, "clip_ratio/low_mean": 0.031250000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.05102495686151087, "entropy": 0.5646582413464785, "epoch": 0.00022, "grad_norm": 0.8135606050491333, "kl": 1.1583675295114517, "learning_rate": 1.4699999999999996e-05, "loss": 0.0019, "step": 22, "step_time": 5.378046836000067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2083.0, "completions/max_terminated_length": 2083.0, "completions/mean_length": 273.5625, "completions/mean_terminated_length": 273.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4709456395357847, "epoch": 0.00023, "frac_reward_zero_std": 0.0, "grad_norm": 1.6809691190719604, "kl": 1.9399101547896862, "learning_rate": 1.5399999999999998e-05, "loss": 0.002, "num_tokens": 484432.0, "reward": 0.31531083583831787, "reward_std": 0.7944824695587158, "rewards/rollout_reward_func/mean": 0.31531083583831787, "rewards/rollout_reward_func/std": 0.8013672828674316, "sampling/importance_sampling_ratio/max": 1.43058180809021, "sampling/importance_sampling_ratio/mean": 0.8337316513061523, "sampling/importance_sampling_ratio/min": 3.4227787271134957e-09, "sampling/sampling_logp_difference/max": 19.432300567626953, "sampling/sampling_logp_difference/mean": 0.3592582941055298, "step": 23, "step_time": 10.35214462099998 }, { "clip_ratio/high_max": 0.06770833395421505, "clip_ratio/high_mean": 0.033854166977107525, "clip_ratio/low_mean": 0.029513888992369175, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.06336805550381541, "entropy": 0.48450588807463646, "epoch": 0.00024, "grad_norm": 0.6914796233177185, "kl": 1.3615021537989378, "learning_rate": 1.61e-05, "loss": -0.0025, "step": 24, "step_time": 5.9152006660000325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2012.0, "completions/max_terminated_length": 2012.0, "completions/mean_length": 507.28125, "completions/mean_terminated_length": 507.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5083212945610285, "epoch": 0.00025, "frac_reward_zero_std": 0.0, "grad_norm": 0.5812759399414062, "kl": 1.190410004928708, "learning_rate": 1.68e-05, "loss": -0.0309, "num_tokens": 528591.0, "reward": 0.5166890621185303, "reward_std": 0.8014365434646606, "rewards/rollout_reward_func/mean": 0.5166890621185303, "rewards/rollout_reward_func/std": 0.7726183533668518, "sampling/importance_sampling_ratio/max": 2.132061719894409, "sampling/importance_sampling_ratio/mean": 0.9254764318466187, "sampling/importance_sampling_ratio/min": 0.012618557550013065, "sampling/sampling_logp_difference/max": 2.5172066688537598, "sampling/sampling_logp_difference/mean": 0.17420633137226105, "step": 25, "step_time": 11.301692467000066 }, { "clip_ratio/high_max": 0.04761904897168279, "clip_ratio/high_mean": 0.023809524485841393, "clip_ratio/low_mean": 0.018601190531626344, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04241071501746774, "entropy": 0.5279924459755421, "epoch": 0.00026, "grad_norm": 0.4087564945220947, "kl": 1.1582919210195541, "learning_rate": 1.75e-05, "loss": -0.0315, "step": 26, "step_time": 6.539828205000049 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 896.0, "completions/max_terminated_length": 896.0, "completions/mean_length": 278.4375, "completions/mean_terminated_length": 278.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5139981396496296, "epoch": 0.00027, "frac_reward_zero_std": 0.0, "grad_norm": 0.7417870163917542, "kl": 0.9947727508842945, "learning_rate": 1.82e-05, "loss": -0.013, "num_tokens": 564592.0, "reward": 0.14556676149368286, "reward_std": 0.8973702192306519, "rewards/rollout_reward_func/mean": 0.14556676149368286, "rewards/rollout_reward_func/std": 0.9176846146583557, "sampling/importance_sampling_ratio/max": 1.5780967473983765, "sampling/importance_sampling_ratio/mean": 0.882154643535614, "sampling/importance_sampling_ratio/min": 0.22991909086704254, "sampling/sampling_logp_difference/max": 1.4501347541809082, "sampling/sampling_logp_difference/mean": 0.14282521605491638, "step": 27, "step_time": 7.269042330999923 }, { "clip_ratio/high_max": 0.04261363670229912, "clip_ratio/high_mean": 0.02130681835114956, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02911931835114956, "entropy": 0.5455751530826092, "epoch": 0.00028, "grad_norm": 0.5108001828193665, "kl": 0.8285699784755707, "learning_rate": 1.89e-05, "loss": -0.0136, "step": 28, "step_time": 3.889183408000008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 347.1875, "completions/mean_terminated_length": 347.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5671474151313305, "epoch": 0.00029, "frac_reward_zero_std": 0.0, "grad_norm": 0.35496625304222107, "kl": 0.6603694520890713, "learning_rate": 1.96e-05, "loss": -0.0147, "num_tokens": 602957.0, "reward": 0.6254887580871582, "reward_std": 0.7725589275360107, "rewards/rollout_reward_func/mean": 0.6254887580871582, "rewards/rollout_reward_func/std": 0.7428649663925171, "sampling/importance_sampling_ratio/max": 1.3871275186538696, "sampling/importance_sampling_ratio/mean": 0.973051905632019, "sampling/importance_sampling_ratio/min": 0.3346633017063141, "sampling/sampling_logp_difference/max": 1.066899299621582, "sampling/sampling_logp_difference/mean": 0.11233532428741455, "step": 29, "step_time": 9.284204646000035 }, { "clip_ratio/high_max": 0.07107371930032969, "clip_ratio/high_mean": 0.04595352662727237, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04595352662727237, "entropy": 0.590064249932766, "epoch": 0.0003, "grad_norm": 0.32286766171455383, "kl": 0.6321401782333851, "learning_rate": 2.03e-05, "loss": -0.0151, "step": 30, "step_time": 5.145461570999942 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 885.0, "completions/max_terminated_length": 885.0, "completions/mean_length": 301.875, "completions/mean_terminated_length": 289.7419128417969, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6691174618899822, "epoch": 0.00031, "frac_reward_zero_std": 0.0, "grad_norm": 0.48589563369750977, "kl": 0.45596247538924217, "learning_rate": 2.0999999999999995e-05, "loss": 0.0303, "num_tokens": 641126.0, "reward": 0.4661554992198944, "reward_std": 0.8632375597953796, "rewards/rollout_reward_func/mean": 0.4661554992198944, "rewards/rollout_reward_func/std": 0.8352718353271484, "sampling/importance_sampling_ratio/max": 2.286161184310913, "sampling/importance_sampling_ratio/mean": 1.0162770748138428, "sampling/importance_sampling_ratio/min": 0.24833691120147705, "sampling/sampling_logp_difference/max": 1.249307632446289, "sampling/sampling_logp_difference/mean": 0.08137969672679901, "step": 31, "step_time": 7.586736330000122 }, { "clip_ratio/high_max": 0.0416666679084301, "clip_ratio/high_mean": 0.02083333395421505, "clip_ratio/low_mean": 0.020067402278073132, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04090073623228818, "entropy": 0.6702573858201504, "epoch": 0.00032, "grad_norm": 0.38940948247909546, "kl": 0.43659770488739014, "learning_rate": 2.1699999999999996e-05, "loss": 0.0278, "step": 32, "step_time": 5.225580574999981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1633.0, "completions/max_terminated_length": 1633.0, "completions/mean_length": 452.59375, "completions/mean_terminated_length": 452.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6992867514491081, "epoch": 0.00033, "frac_reward_zero_std": 0.25, "grad_norm": 0.30734241008758545, "kl": 0.6430733911693096, "learning_rate": 2.2399999999999996e-05, "loss": -0.0201, "num_tokens": 685085.0, "reward": 0.3771618902683258, "reward_std": 0.7437170147895813, "rewards/rollout_reward_func/mean": 0.3771618902683258, "rewards/rollout_reward_func/std": 0.9015460014343262, "sampling/importance_sampling_ratio/max": 1.7714743614196777, "sampling/importance_sampling_ratio/mean": 0.9153831005096436, "sampling/importance_sampling_ratio/min": 0.32849931716918945, "sampling/sampling_logp_difference/max": 1.113156795501709, "sampling/sampling_logp_difference/mean": 0.09098587930202484, "step": 33, "step_time": 9.618119152000077 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.048390152398496866, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.05620265332981944, "entropy": 0.6778096407651901, "epoch": 0.00034, "grad_norm": 0.2629814147949219, "kl": 0.8332904549315572, "learning_rate": 2.3099999999999996e-05, "loss": -0.0203, "step": 34, "step_time": 5.351734105999867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 822.0, "completions/max_terminated_length": 822.0, "completions/mean_length": 337.9375, "completions/mean_terminated_length": 337.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5696994103491306, "epoch": 0.00035, "frac_reward_zero_std": 0.0, "grad_norm": 1.0129225254058838, "kl": 2.422546725720167, "learning_rate": 2.3799999999999996e-05, "loss": -0.0018, "num_tokens": 724127.0, "reward": 0.45748478174209595, "reward_std": 0.8362337350845337, "rewards/rollout_reward_func/mean": 0.45748478174209595, "rewards/rollout_reward_func/std": 0.8049970865249634, "sampling/importance_sampling_ratio/max": 1.6235036849975586, "sampling/importance_sampling_ratio/mean": 0.9821629524230957, "sampling/importance_sampling_ratio/min": 0.2444143146276474, "sampling/sampling_logp_difference/max": 1.3539834022521973, "sampling/sampling_logp_difference/mean": 0.0972641184926033, "step": 35, "step_time": 7.135632953999902 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.5695516802370548, "epoch": 0.00036, "grad_norm": 0.5174685120582581, "kl": 1.735250562429428, "learning_rate": 2.4499999999999996e-05, "loss": -0.0046, "step": 36, "step_time": 3.7916482090000727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1569.0, "completions/max_terminated_length": 1569.0, "completions/mean_length": 373.8125, "completions/mean_terminated_length": 373.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6458892896771431, "epoch": 0.00037, "frac_reward_zero_std": 0.0, "grad_norm": 0.5318168997764587, "kl": 0.8101392164826393, "learning_rate": 2.5199999999999993e-05, "loss": -0.0209, "num_tokens": 764087.0, "reward": 0.5123010277748108, "reward_std": 0.6879165172576904, "rewards/rollout_reward_func/mean": 0.5123010277748108, "rewards/rollout_reward_func/std": 0.7307071685791016, "sampling/importance_sampling_ratio/max": 2.353904962539673, "sampling/importance_sampling_ratio/mean": 1.0601342916488647, "sampling/importance_sampling_ratio/min": 0.4304255247116089, "sampling/sampling_logp_difference/max": 0.8530892133712769, "sampling/sampling_logp_difference/mean": 0.06941355019807816, "step": 37, "step_time": 9.443088265999904 }, { "clip_ratio/high_max": 0.03219697065651417, "clip_ratio/high_mean": 0.016098485328257084, "clip_ratio/low_mean": 0.031250000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04734848625957966, "entropy": 0.6517101153731346, "epoch": 0.00038, "grad_norm": 0.279819130897522, "kl": 0.8386905677616596, "learning_rate": 2.5899999999999996e-05, "loss": -0.0226, "step": 38, "step_time": 5.212772643000335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 371.03125, "completions/mean_terminated_length": 371.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6471690461039543, "epoch": 0.00039, "frac_reward_zero_std": 0.0, "grad_norm": 0.4146345853805542, "kl": 0.5213250620290637, "learning_rate": 2.6599999999999993e-05, "loss": 0.0138, "num_tokens": 804244.0, "reward": 0.24227820336818695, "reward_std": 0.8518268465995789, "rewards/rollout_reward_func/mean": 0.24227820336818695, "rewards/rollout_reward_func/std": 0.8743859529495239, "sampling/importance_sampling_ratio/max": 1.252670168876648, "sampling/importance_sampling_ratio/mean": 0.9353030920028687, "sampling/importance_sampling_ratio/min": 0.27799496054649353, "sampling/sampling_logp_difference/max": 0.7183585166931152, "sampling/sampling_logp_difference/mean": 0.07006009668111801, "step": 39, "step_time": 10.487295580999898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.009765625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009765625, "entropy": 0.648285236209631, "epoch": 0.0004, "grad_norm": 0.4161662757396698, "kl": 0.5154504245147109, "learning_rate": 2.7299999999999996e-05, "loss": 0.0127, "step": 40, "step_time": 5.171773260000123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 361.78125, "completions/mean_terminated_length": 361.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7153925113379955, "epoch": 0.00041, "frac_reward_zero_std": 0.0, "grad_norm": 0.4973072111606598, "kl": 0.6529207359999418, "learning_rate": 2.7999999999999993e-05, "loss": -0.0218, "num_tokens": 844678.0, "reward": 0.3696245551109314, "reward_std": 0.8395380973815918, "rewards/rollout_reward_func/mean": 0.3696245551109314, "rewards/rollout_reward_func/std": 0.8695901036262512, "sampling/importance_sampling_ratio/max": 1.708664059638977, "sampling/importance_sampling_ratio/mean": 0.9626280069351196, "sampling/importance_sampling_ratio/min": 0.45028790831565857, "sampling/sampling_logp_difference/max": 0.7366434335708618, "sampling/sampling_logp_difference/mean": 0.07328729331493378, "step": 41, "step_time": 9.346013239000058 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.04010416753590107, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04010416753590107, "entropy": 0.7066530771553516, "epoch": 0.00042, "grad_norm": 0.43947821855545044, "kl": 0.7846181578934193, "learning_rate": 2.8699999999999996e-05, "loss": -0.0235, "step": 42, "step_time": 5.169217666999998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 885.0, "completions/max_terminated_length": 885.0, "completions/mean_length": 307.90625, "completions/mean_terminated_length": 307.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6140940934419632, "epoch": 0.00043, "frac_reward_zero_std": 0.0, "grad_norm": 0.2699756920337677, "kl": 1.4204679615795612, "learning_rate": 2.9399999999999993e-05, "loss": -0.0124, "num_tokens": 883009.0, "reward": 0.39327478408813477, "reward_std": 0.7385755777359009, "rewards/rollout_reward_func/mean": 0.39327478408813477, "rewards/rollout_reward_func/std": 0.7975078225135803, "sampling/importance_sampling_ratio/max": 2.480621337890625, "sampling/importance_sampling_ratio/mean": 1.0348066091537476, "sampling/importance_sampling_ratio/min": 0.4264654815196991, "sampling/sampling_logp_difference/max": 0.8521008491516113, "sampling/sampling_logp_difference/mean": 0.07076883316040039, "step": 43, "step_time": 7.383370778999847 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.03645833395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04427083395421505, "entropy": 0.5966673120856285, "epoch": 0.00044, "grad_norm": 0.24958068132400513, "kl": 1.525636799633503, "learning_rate": 3.0099999999999996e-05, "loss": -0.0133, "step": 44, "step_time": 3.9524661020000167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1569.0, "completions/max_terminated_length": 1569.0, "completions/mean_length": 382.28125, "completions/mean_terminated_length": 382.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5850842632353306, "epoch": 0.00045, "frac_reward_zero_std": 0.0, "grad_norm": 0.2686639428138733, "kl": 0.4823284484446049, "learning_rate": 3.0799999999999996e-05, "loss": 0.0061, "num_tokens": 924176.0, "reward": 0.6984595656394958, "reward_std": 0.5527162551879883, "rewards/rollout_reward_func/mean": 0.6984595656394958, "rewards/rollout_reward_func/std": 0.5506800413131714, "sampling/importance_sampling_ratio/max": 2.034982204437256, "sampling/importance_sampling_ratio/mean": 1.0773773193359375, "sampling/importance_sampling_ratio/min": 0.7587000131607056, "sampling/sampling_logp_difference/max": 0.6249358654022217, "sampling/sampling_logp_difference/mean": 0.042881034314632416, "step": 45, "step_time": 9.454016895999757 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.6024033650755882, "epoch": 0.00046, "grad_norm": 0.25619491934776306, "kl": 0.47822780907154083, "learning_rate": 3.15e-05, "loss": 0.0064, "step": 46, "step_time": 6.143063694999682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2232.0, "completions/max_terminated_length": 2232.0, "completions/mean_length": 375.53125, "completions/mean_terminated_length": 375.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5815742462873459, "epoch": 0.00047, "frac_reward_zero_std": 0.0, "grad_norm": 0.5717251300811768, "kl": 1.3818266168236732, "learning_rate": 3.22e-05, "loss": -0.003, "num_tokens": 963933.0, "reward": 0.2744709253311157, "reward_std": 0.8958877325057983, "rewards/rollout_reward_func/mean": 0.2744709253311157, "rewards/rollout_reward_func/std": 0.8847177028656006, "sampling/importance_sampling_ratio/max": 1.5530790090560913, "sampling/importance_sampling_ratio/mean": 0.9414335489273071, "sampling/importance_sampling_ratio/min": 0.4300411641597748, "sampling/sampling_logp_difference/max": 0.8549981117248535, "sampling/sampling_logp_difference/mean": 0.06860444694757462, "step": 47, "step_time": 11.308755530000099 }, { "clip_ratio/high_max": 0.01785714365541935, "clip_ratio/high_mean": 0.008928571827709675, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008928571827709675, "entropy": 0.610357079654932, "epoch": 0.00048, "grad_norm": 0.3430391550064087, "kl": 0.8880793359130621, "learning_rate": 3.289999999999999e-05, "loss": -0.0051, "step": 48, "step_time": 6.297034845000098 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1704.0, "completions/max_terminated_length": 1704.0, "completions/mean_length": 340.40625, "completions/mean_terminated_length": 340.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5246803164482117, "epoch": 0.00049, "frac_reward_zero_std": 0.25, "grad_norm": 0.3835832476615906, "kl": 0.3953630458563566, "learning_rate": 3.36e-05, "loss": -0.053, "num_tokens": 1003184.0, "reward": 0.5471705198287964, "reward_std": 0.6104870438575745, "rewards/rollout_reward_func/mean": 0.5471705198287964, "rewards/rollout_reward_func/std": 0.7285730838775635, "sampling/importance_sampling_ratio/max": 2.2740707397460938, "sampling/importance_sampling_ratio/mean": 1.054465651512146, "sampling/importance_sampling_ratio/min": 0.6042937636375427, "sampling/sampling_logp_difference/max": 0.7734719514846802, "sampling/sampling_logp_difference/mean": 0.03311889246106148, "step": 49, "step_time": 9.906691997000053 }, { "clip_ratio/high_max": 0.029458992183208466, "clip_ratio/high_mean": 0.014729496091604233, "clip_ratio/low_mean": 0.02130681835114956, "clip_ratio/low_min": 0.005681818351149559, "clip_ratio/region_mean": 0.03603631444275379, "entropy": 0.5177083276212215, "epoch": 0.0005, "grad_norm": 0.16407334804534912, "kl": 0.42381778452545404, "learning_rate": 3.4299999999999993e-05, "loss": -0.055, "step": 50, "step_time": 5.278826424000272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1319.0, "completions/max_terminated_length": 1319.0, "completions/mean_length": 426.4375, "completions/mean_terminated_length": 426.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5570813603699207, "epoch": 0.00051, "frac_reward_zero_std": 0.25, "grad_norm": 0.4011284410953522, "kl": 1.1282444819808006, "learning_rate": 3.5e-05, "loss": 0.0293, "num_tokens": 1045520.0, "reward": 0.5908991694450378, "reward_std": 0.6321189403533936, "rewards/rollout_reward_func/mean": 0.5908991694450378, "rewards/rollout_reward_func/std": 0.7496492862701416, "sampling/importance_sampling_ratio/max": 1.3104177713394165, "sampling/importance_sampling_ratio/mean": 0.9521603584289551, "sampling/importance_sampling_ratio/min": 0.3754626214504242, "sampling/sampling_logp_difference/max": 0.8456258773803711, "sampling/sampling_logp_difference/mean": 0.05747090280056, "step": 51, "step_time": 8.817306928000562 }, { "clip_ratio/high_max": 0.030357143841683865, "clip_ratio/high_mean": 0.015178571920841932, "clip_ratio/low_mean": 0.016666667070239782, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.031845238991081715, "entropy": 0.566676065325737, "epoch": 0.00052, "grad_norm": 0.4241269826889038, "kl": 0.6556614749133587, "learning_rate": 3.5699999999999994e-05, "loss": 0.0287, "step": 52, "step_time": 5.310260251999807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2263.0, "completions/max_terminated_length": 2263.0, "completions/mean_length": 516.21875, "completions/mean_terminated_length": 516.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.49229545518755913, "epoch": 0.00053, "frac_reward_zero_std": 0.25, "grad_norm": 0.35610097646713257, "kl": 0.32822963036596775, "learning_rate": 3.64e-05, "loss": 0.0101, "num_tokens": 1090419.0, "reward": 0.734697699546814, "reward_std": 0.427322655916214, "rewards/rollout_reward_func/mean": 0.734697699546814, "rewards/rollout_reward_func/std": 0.5280843377113342, "sampling/importance_sampling_ratio/max": 1.4068918228149414, "sampling/importance_sampling_ratio/mean": 0.9838685393333435, "sampling/importance_sampling_ratio/min": 0.6134293079376221, "sampling/sampling_logp_difference/max": 0.47754764556884766, "sampling/sampling_logp_difference/mean": 0.02736492082476616, "step": 53, "step_time": 11.950595314000111 }, { "clip_ratio/high_max": 0.06428571464493871, "clip_ratio/high_mean": 0.032142857322469354, "clip_ratio/low_mean": 0.018702652072533965, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.05084551032632589, "entropy": 0.4900171272456646, "epoch": 0.00054, "grad_norm": 0.2704615294933319, "kl": 0.33605330996215343, "learning_rate": 3.7099999999999994e-05, "loss": 0.0084, "step": 54, "step_time": 6.427439306000224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2133.0, "completions/max_terminated_length": 2133.0, "completions/mean_length": 493.5, "completions/mean_terminated_length": 493.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5469484254717827, "epoch": 0.00055, "frac_reward_zero_std": 0.0, "grad_norm": 0.46761012077331543, "kl": 0.9315161276608706, "learning_rate": 3.78e-05, "loss": -0.0506, "num_tokens": 1133961.0, "reward": 0.504819929599762, "reward_std": 0.8262927532196045, "rewards/rollout_reward_func/mean": 0.504819929599762, "rewards/rollout_reward_func/std": 0.8273648023605347, "sampling/importance_sampling_ratio/max": 1.4178036451339722, "sampling/importance_sampling_ratio/mean": 0.9473260641098022, "sampling/importance_sampling_ratio/min": 0.35218551754951477, "sampling/sampling_logp_difference/max": 0.7030744552612305, "sampling/sampling_logp_difference/mean": 0.039900828152894974, "step": 55, "step_time": 11.955930813000123 }, { "clip_ratio/high_max": 0.00844594556838274, "clip_ratio/high_mean": 0.00422297278419137, "clip_ratio/low_mean": 0.03843750013038516, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04266047244891524, "entropy": 0.5416288543492556, "epoch": 0.00056, "grad_norm": 0.310720294713974, "kl": 1.0212288200855255, "learning_rate": 3.8499999999999994e-05, "loss": -0.0531, "step": 56, "step_time": 6.293462571000418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1560.0, "completions/max_terminated_length": 1560.0, "completions/mean_length": 401.875, "completions/mean_terminated_length": 401.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6370696276426315, "epoch": 0.00057, "frac_reward_zero_std": 0.0, "grad_norm": 0.4104721248149872, "kl": 0.9147022105753422, "learning_rate": 3.92e-05, "loss": 0.003, "num_tokens": 1175660.0, "reward": 0.36384594440460205, "reward_std": 0.864052951335907, "rewards/rollout_reward_func/mean": 0.36384594440460205, "rewards/rollout_reward_func/std": 0.8291681408882141, "sampling/importance_sampling_ratio/max": 1.2632278203964233, "sampling/importance_sampling_ratio/mean": 0.9862111806869507, "sampling/importance_sampling_ratio/min": 0.47024795413017273, "sampling/sampling_logp_difference/max": 0.5524387359619141, "sampling/sampling_logp_difference/mean": 0.03683824837207794, "step": 57, "step_time": 9.438028336999878 }, { "clip_ratio/high_max": 0.0031250000465661287, "clip_ratio/high_mean": 0.0015625000232830644, "clip_ratio/low_mean": 0.017187500023283064, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01875000004656613, "entropy": 0.6218053214251995, "epoch": 0.00058, "grad_norm": 0.2907359004020691, "kl": 0.9313262607902288, "learning_rate": 3.9899999999999994e-05, "loss": 0.0015, "step": 58, "step_time": 5.0690294840001116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 394.96875, "completions/mean_terminated_length": 394.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.49436085671186447, "epoch": 0.00059, "frac_reward_zero_std": 0.0, "grad_norm": 0.2927047312259674, "kl": 0.5024137096479535, "learning_rate": 4.06e-05, "loss": -0.0123, "num_tokens": 1216450.0, "reward": 0.6840561032295227, "reward_std": 0.6365723609924316, "rewards/rollout_reward_func/mean": 0.6840561032295227, "rewards/rollout_reward_func/std": 0.6415673494338989, "sampling/importance_sampling_ratio/max": 1.256005883216858, "sampling/importance_sampling_ratio/mean": 0.9498515129089355, "sampling/importance_sampling_ratio/min": 0.5531520247459412, "sampling/sampling_logp_difference/max": 0.5923171043395996, "sampling/sampling_logp_difference/mean": 0.040586017072200775, "step": 59, "step_time": 10.564330916999552 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.028645833488553762, "clip_ratio/low_min": 0.0052083334885537624, "clip_ratio/region_mean": 0.028645833488553762, "entropy": 0.47580323927104473, "epoch": 0.0006, "grad_norm": 0.25059637427330017, "kl": 0.6123338118195534, "learning_rate": 4.1299999999999994e-05, "loss": -0.013, "step": 60, "step_time": 4.930093829000043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2155.0, "completions/max_terminated_length": 2155.0, "completions/mean_length": 472.84375, "completions/mean_terminated_length": 472.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5367695055902004, "epoch": 0.00061, "frac_reward_zero_std": 0.0, "grad_norm": 0.5520923137664795, "kl": 0.44945060834288597, "learning_rate": 4.199999999999999e-05, "loss": 0.0503, "num_tokens": 1260348.0, "reward": 0.5375505685806274, "reward_std": 0.5773008465766907, "rewards/rollout_reward_func/mean": 0.5375505685806274, "rewards/rollout_reward_func/std": 0.6407954692840576, "sampling/importance_sampling_ratio/max": 1.4726043939590454, "sampling/importance_sampling_ratio/mean": 1.0269801616668701, "sampling/importance_sampling_ratio/min": 0.6010439991950989, "sampling/sampling_logp_difference/max": 0.3899970054626465, "sampling/sampling_logp_difference/mean": 0.028261784464120865, "step": 61, "step_time": 11.120504472999983 }, { "clip_ratio/high_max": 0.03354614693671465, "clip_ratio/high_mean": 0.016773073468357325, "clip_ratio/low_mean": 0.014448924921452999, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0312219993211329, "entropy": 0.5129078440368176, "epoch": 0.00062, "grad_norm": 0.3464604616165161, "kl": 0.492124168202281, "learning_rate": 4.2699999999999994e-05, "loss": 0.0475, "step": 62, "step_time": 6.178636681000398 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1382.0, "completions/max_terminated_length": 1382.0, "completions/mean_length": 370.53125, "completions/mean_terminated_length": 370.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.40959035232663155, "epoch": 0.00063, "frac_reward_zero_std": 0.25, "grad_norm": 0.1639973372220993, "kl": 0.5022875368595123, "learning_rate": 4.339999999999999e-05, "loss": -0.0093, "num_tokens": 1300760.0, "reward": 0.7992902994155884, "reward_std": 0.39145028591156006, "rewards/rollout_reward_func/mean": 0.7992902994155884, "rewards/rollout_reward_func/std": 0.4952199161052704, "sampling/importance_sampling_ratio/max": 1.2639648914337158, "sampling/importance_sampling_ratio/mean": 1.023637294769287, "sampling/importance_sampling_ratio/min": 0.6356881260871887, "sampling/sampling_logp_difference/max": 0.46201443672180176, "sampling/sampling_logp_difference/mean": 0.028952516615390778, "step": 63, "step_time": 8.636554564999642 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.016666667070239782, "entropy": 0.38771606236696243, "epoch": 0.00064, "grad_norm": 0.05583963543176651, "kl": 0.48581767827272415, "learning_rate": 4.4099999999999995e-05, "loss": -0.0099, "step": 64, "step_time": 4.72187211700043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 323.71875, "completions/mean_terminated_length": 323.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3421889692544937, "epoch": 0.00065, "frac_reward_zero_std": 0.0, "grad_norm": 0.1805817037820816, "kl": 0.521814052015543, "learning_rate": 4.479999999999999e-05, "loss": 0.008, "num_tokens": 1337890.0, "reward": 0.7406772375106812, "reward_std": 0.5136924982070923, "rewards/rollout_reward_func/mean": 0.7406772375106812, "rewards/rollout_reward_func/std": 0.5787789225578308, "sampling/importance_sampling_ratio/max": 1.2137353420257568, "sampling/importance_sampling_ratio/mean": 1.0055501461029053, "sampling/importance_sampling_ratio/min": 0.6156193017959595, "sampling/sampling_logp_difference/max": 0.4779789447784424, "sampling/sampling_logp_difference/mean": 0.025179781019687653, "step": 65, "step_time": 8.791412391999756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.008928571827709675, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008928571827709675, "entropy": 0.34326307475566864, "epoch": 0.00066, "grad_norm": 0.1993580460548401, "kl": 0.5941803902387619, "learning_rate": 4.5499999999999995e-05, "loss": 0.0077, "step": 66, "step_time": 5.96691887799966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1292.0, "completions/max_terminated_length": 1292.0, "completions/mean_length": 356.59375, "completions/mean_terminated_length": 356.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4288066104054451, "epoch": 0.00067, "frac_reward_zero_std": 0.25, "grad_norm": 0.20984771847724915, "kl": 0.3854968659579754, "learning_rate": 4.619999999999999e-05, "loss": -0.0064, "num_tokens": 1378884.0, "reward": 0.8612989187240601, "reward_std": 0.32517996430397034, "rewards/rollout_reward_func/mean": 0.8612989187240601, "rewards/rollout_reward_func/std": 0.3728911876678467, "sampling/importance_sampling_ratio/max": 1.1552592515945435, "sampling/importance_sampling_ratio/mean": 1.0281764268875122, "sampling/importance_sampling_ratio/min": 0.5710732340812683, "sampling/sampling_logp_difference/max": 0.5644137859344482, "sampling/sampling_logp_difference/mean": 0.024440273642539978, "step": 67, "step_time": 8.369410511000297 }, { "clip_ratio/high_max": 0.01785714365541935, "clip_ratio/high_mean": 0.008928571827709675, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.032366071827709675, "entropy": 0.4539686106145382, "epoch": 0.00068, "grad_norm": 0.09087157994508743, "kl": 0.384282398968935, "learning_rate": 4.6899999999999995e-05, "loss": -0.0078, "step": 68, "step_time": 4.562634150999429 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2387.0, "completions/max_terminated_length": 2387.0, "completions/mean_length": 424.8125, "completions/mean_terminated_length": 424.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.515124998986721, "epoch": 0.00069, "frac_reward_zero_std": 0.0, "grad_norm": 0.4971536695957184, "kl": 0.4903581514954567, "learning_rate": 4.759999999999999e-05, "loss": 0.0026, "num_tokens": 1421511.0, "reward": 0.49469029903411865, "reward_std": 0.7985262274742126, "rewards/rollout_reward_func/mean": 0.49469029903411865, "rewards/rollout_reward_func/std": 0.7979835271835327, "sampling/importance_sampling_ratio/max": 1.1188106536865234, "sampling/importance_sampling_ratio/mean": 0.9666975736618042, "sampling/importance_sampling_ratio/min": 0.624027669429779, "sampling/sampling_logp_difference/max": 0.5713698863983154, "sampling/sampling_logp_difference/mean": 0.03594612702727318, "step": 69, "step_time": 11.47663711999985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.06666666781529784, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.06666666781529784, "entropy": 0.5281698293983936, "epoch": 0.0007, "grad_norm": 0.28421175479888916, "kl": 0.5763002540916204, "learning_rate": 4.8299999999999995e-05, "loss": 0.0023, "step": 70, "step_time": 6.561869999999999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1659.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 412.28125, "completions/mean_terminated_length": 412.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6652674749493599, "epoch": 0.00071, "frac_reward_zero_std": 0.0, "grad_norm": 0.45789670944213867, "kl": 0.45381924510002136, "learning_rate": 4.899999999999999e-05, "loss": 0.023, "num_tokens": 1463312.0, "reward": 0.312690407037735, "reward_std": 0.9014500379562378, "rewards/rollout_reward_func/mean": 0.312690407037735, "rewards/rollout_reward_func/std": 0.8848817944526672, "sampling/importance_sampling_ratio/max": 2.231497049331665, "sampling/importance_sampling_ratio/mean": 1.0541712045669556, "sampling/importance_sampling_ratio/min": 0.8277871608734131, "sampling/sampling_logp_difference/max": 0.6575703620910645, "sampling/sampling_logp_difference/mean": 0.039049867540597916, "step": 71, "step_time": 9.969856163999566 }, { "clip_ratio/high_max": 0.03645833395421505, "clip_ratio/high_mean": 0.018229166977107525, "clip_ratio/low_mean": 0.05156250111758709, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.06979166809469461, "entropy": 0.6291546784341335, "epoch": 0.00072, "grad_norm": 0.30403563380241394, "kl": 0.5789325144141912, "learning_rate": 4.899991951821627e-05, "loss": 0.02, "step": 72, "step_time": 5.880795620000072 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1722.0, "completions/max_terminated_length": 1722.0, "completions/mean_length": 436.25, "completions/mean_terminated_length": 436.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4070567786693573, "epoch": 0.00073, "frac_reward_zero_std": 0.0, "grad_norm": 0.3155321180820465, "kl": 0.4537731744349003, "learning_rate": 4.899967807348718e-05, "loss": 0.0026, "num_tokens": 1506004.0, "reward": 0.7694225311279297, "reward_std": 0.5220502614974976, "rewards/rollout_reward_func/mean": 0.7694225311279297, "rewards/rollout_reward_func/std": 0.5109183192253113, "sampling/importance_sampling_ratio/max": 1.2453076839447021, "sampling/importance_sampling_ratio/mean": 0.9966861009597778, "sampling/importance_sampling_ratio/min": 0.5352568626403809, "sampling/sampling_logp_difference/max": 0.6253266334533691, "sampling/sampling_logp_difference/mean": 0.02308061346411705, "step": 73, "step_time": 10.220542977000378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02083333395421505, "entropy": 0.3973471149802208, "epoch": 0.00074, "grad_norm": 0.26673704385757446, "kl": 0.5531922094523907, "learning_rate": 4.899927566767893e-05, "loss": 0.0022, "step": 74, "step_time": 5.3083821689999695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1265.0, "completions/max_terminated_length": 1265.0, "completions/mean_length": 420.375, "completions/mean_terminated_length": 420.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4230531416833401, "epoch": 0.00075, "frac_reward_zero_std": 0.0, "grad_norm": 0.24852482974529266, "kl": 1.01990083232522, "learning_rate": 4.899871230390185e-05, "loss": 0.0058, "num_tokens": 1548189.0, "reward": 0.7095742225646973, "reward_std": 0.5819222927093506, "rewards/rollout_reward_func/mean": 0.7095742225646973, "rewards/rollout_reward_func/std": 0.5867263674736023, "sampling/importance_sampling_ratio/max": 1.1410356760025024, "sampling/importance_sampling_ratio/mean": 0.9742614030838013, "sampling/importance_sampling_ratio/min": 0.5787585377693176, "sampling/sampling_logp_difference/max": 0.46645617485046387, "sampling/sampling_logp_difference/mean": 0.027309508994221687, "step": 75, "step_time": 8.282782382000278 }, { "clip_ratio/high_max": 0.0416666679084301, "clip_ratio/high_mean": 0.02083333395421505, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02083333395421505, "entropy": 0.43744034692645073, "epoch": 0.00076, "grad_norm": 0.19275036454200745, "kl": 0.9085519909858704, "learning_rate": 4.899798798651038e-05, "loss": 0.0045, "step": 76, "step_time": 4.459019883000337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1409.0, "completions/max_terminated_length": 1409.0, "completions/mean_length": 496.8125, "completions/mean_terminated_length": 496.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5104015916585922, "epoch": 0.00077, "frac_reward_zero_std": 0.0, "grad_norm": 0.2423262894153595, "kl": 0.6945657283067703, "learning_rate": 4.8997102721103e-05, "loss": 0.0015, "num_tokens": 1593207.0, "reward": 0.5669847726821899, "reward_std": 0.8111416101455688, "rewards/rollout_reward_func/mean": 0.5669847726821899, "rewards/rollout_reward_func/std": 0.785930871963501, "sampling/importance_sampling_ratio/max": 1.417889952659607, "sampling/importance_sampling_ratio/mean": 1.000488042831421, "sampling/importance_sampling_ratio/min": 0.6728711724281311, "sampling/sampling_logp_difference/max": 0.3859211206436157, "sampling/sampling_logp_difference/mean": 0.02619420737028122, "step": 77, "step_time": 8.944382845000291 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.02864583395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03645833395421505, "entropy": 0.5059718824923038, "epoch": 0.00078, "grad_norm": 0.22946427762508392, "kl": 0.7896372973918915, "learning_rate": 4.899605651452225e-05, "loss": 0.0008, "step": 78, "step_time": 4.863985955000544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 391.375, "completions/mean_terminated_length": 391.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.44990584440529346, "epoch": 0.00079, "frac_reward_zero_std": 0.25, "grad_norm": 0.2751384377479553, "kl": 0.6439568549394608, "learning_rate": 4.899484937485461e-05, "loss": -0.0022, "num_tokens": 1633664.0, "reward": 0.6388370990753174, "reward_std": 0.5074847936630249, "rewards/rollout_reward_func/mean": 0.6388370990753174, "rewards/rollout_reward_func/std": 0.6214325428009033, "sampling/importance_sampling_ratio/max": 1.3939692974090576, "sampling/importance_sampling_ratio/mean": 1.0125408172607422, "sampling/importance_sampling_ratio/min": 0.7555236220359802, "sampling/sampling_logp_difference/max": 0.32885807752609253, "sampling/sampling_logp_difference/mean": 0.024129528552293777, "step": 79, "step_time": 9.857838364999907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.4216177202761173, "epoch": 0.0008, "grad_norm": 0.19295018911361694, "kl": 0.6440215185284615, "learning_rate": 4.899348131143047e-05, "loss": -0.0029, "step": 80, "step_time": 4.933378074000075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1615.0, "completions/max_terminated_length": 1615.0, "completions/mean_length": 412.21875, "completions/mean_terminated_length": 412.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32923791091889143, "epoch": 0.00081, "frac_reward_zero_std": 0.25, "grad_norm": 0.22175629436969757, "kl": 0.8220385573804379, "learning_rate": 4.8991952334824084e-05, "loss": 0.0059, "num_tokens": 1674469.0, "reward": 0.6284059286117554, "reward_std": 0.6391177773475647, "rewards/rollout_reward_func/mean": 0.6284059286117554, "rewards/rollout_reward_func/std": 0.742104172706604, "sampling/importance_sampling_ratio/max": 1.0724939107894897, "sampling/importance_sampling_ratio/mean": 1.0000009536743164, "sampling/importance_sampling_ratio/min": 0.8615776300430298, "sampling/sampling_logp_difference/max": 0.16149115562438965, "sampling/sampling_logp_difference/mean": 0.015708979219198227, "step": 81, "step_time": 9.364669605000017 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.03480113670229912, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04261363670229912, "entropy": 0.3149419268593192, "epoch": 0.00082, "grad_norm": 0.29863590002059937, "kl": 0.9766829460859299, "learning_rate": 4.899026245685341e-05, "loss": 0.0053, "step": 82, "step_time": 5.232645025999773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1632.0, "completions/max_terminated_length": 1632.0, "completions/mean_length": 414.46875, "completions/mean_terminated_length": 414.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3820315217599273, "epoch": 0.00083, "frac_reward_zero_std": 0.0, "grad_norm": 0.2077123075723648, "kl": 0.6875754073262215, "learning_rate": 4.8988411690580104e-05, "loss": 0.0026, "num_tokens": 1715075.0, "reward": 0.6403211951255798, "reward_std": 0.6127077341079712, "rewards/rollout_reward_func/mean": 0.6403211951255798, "rewards/rollout_reward_func/std": 0.6147496104240417, "sampling/importance_sampling_ratio/max": 1.183364987373352, "sampling/importance_sampling_ratio/mean": 1.0068573951721191, "sampling/importance_sampling_ratio/min": 0.7390010356903076, "sampling/sampling_logp_difference/max": 0.28660011291503906, "sampling/sampling_logp_difference/mean": 0.017997726798057556, "step": 83, "step_time": 9.539955769000244 }, { "clip_ratio/high_max": 0.01785714365541935, "clip_ratio/high_mean": 0.008928571827709675, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008928571827709675, "entropy": 0.39719980861991644, "epoch": 0.00084, "grad_norm": 0.20877410471439362, "kl": 0.6843762546777725, "learning_rate": 4.898640005030938e-05, "loss": 0.002, "step": 84, "step_time": 5.32367764699984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 298.625, "completions/mean_terminated_length": 298.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3185494616627693, "epoch": 0.00085, "frac_reward_zero_std": 0.0, "grad_norm": 0.29013341665267944, "kl": 0.559021569788456, "learning_rate": 4.898422755158989e-05, "loss": 0.0033, "num_tokens": 1751991.0, "reward": 0.5058728456497192, "reward_std": 0.7923943400382996, "rewards/rollout_reward_func/mean": 0.5058728456497192, "rewards/rollout_reward_func/std": 0.825049877166748, "sampling/importance_sampling_ratio/max": 1.5640169382095337, "sampling/importance_sampling_ratio/mean": 1.0250962972640991, "sampling/importance_sampling_ratio/min": 0.8095297813415527, "sampling/sampling_logp_difference/max": 0.33199453353881836, "sampling/sampling_logp_difference/mean": 0.01975112222135067, "step": 85, "step_time": 8.680368445000113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.026041666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.026041666977107525, "entropy": 0.3168821148574352, "epoch": 0.00086, "grad_norm": 0.13579989969730377, "kl": 0.564352011308074, "learning_rate": 4.898189421121362e-05, "loss": 0.0015, "step": 86, "step_time": 5.847938584999838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 528.375, "completions/mean_terminated_length": 528.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.40844342298805714, "epoch": 0.00087, "frac_reward_zero_std": 0.5, "grad_norm": 0.2187075912952423, "kl": 0.6964751742780209, "learning_rate": 4.897940004721576e-05, "loss": -0.001, "num_tokens": 1798220.0, "reward": 0.7521907091140747, "reward_std": 0.3741147220134735, "rewards/rollout_reward_func/mean": 0.7521907091140747, "rewards/rollout_reward_func/std": 0.6084619164466858, "sampling/importance_sampling_ratio/max": 1.1737929582595825, "sampling/importance_sampling_ratio/mean": 0.9630006551742554, "sampling/importance_sampling_ratio/min": 0.667667031288147, "sampling/sampling_logp_difference/max": 0.3075978755950928, "sampling/sampling_logp_difference/mean": 0.024598699063062668, "step": 87, "step_time": 9.567039445000091 }, { "clip_ratio/high_max": 0.02777777798473835, "clip_ratio/high_mean": 0.013888888992369175, "clip_ratio/low_mean": 0.02924107201397419, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.043129961006343365, "entropy": 0.40063840337097645, "epoch": 0.00088, "grad_norm": 0.1486017405986786, "kl": 0.767490841448307, "learning_rate": 4.897674507887456e-05, "loss": -0.0014, "step": 88, "step_time": 5.090399330000082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1373.0, "completions/max_terminated_length": 1373.0, "completions/mean_length": 462.125, "completions/mean_terminated_length": 462.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.43817245960235596, "epoch": 0.00089, "frac_reward_zero_std": 0.25, "grad_norm": 0.25970736145973206, "kl": 0.7830124385654926, "learning_rate": 4.897392932671118e-05, "loss": 0.0008, "num_tokens": 1841961.0, "reward": 0.7074525356292725, "reward_std": 0.500070333480835, "rewards/rollout_reward_func/mean": 0.7074525356292725, "rewards/rollout_reward_func/std": 0.5895466208457947, "sampling/importance_sampling_ratio/max": 1.226859450340271, "sampling/importance_sampling_ratio/mean": 0.966083288192749, "sampling/importance_sampling_ratio/min": 0.31705018877983093, "sampling/sampling_logp_difference/max": 1.0913150310516357, "sampling/sampling_logp_difference/mean": 0.03238843008875847, "step": 89, "step_time": 8.917119897000248 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.03149801678955555, "clip_ratio/low_min": 0.013888888992369175, "clip_ratio/region_mean": 0.03931051678955555, "entropy": 0.4182748533785343, "epoch": 0.0009, "grad_norm": 0.394379198551178, "kl": 0.8382480926811695, "learning_rate": 4.897095281248954e-05, "loss": -0.0007, "step": 90, "step_time": 4.794355753000218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1678.0, "completions/max_terminated_length": 1678.0, "completions/mean_length": 404.09375, "completions/mean_terminated_length": 404.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3878983333706856, "epoch": 0.00091, "frac_reward_zero_std": 0.0, "grad_norm": 0.3177123963832855, "kl": 0.9751700796186924, "learning_rate": 4.896781555921612e-05, "loss": 0.0043, "num_tokens": 1883325.0, "reward": 0.5675632953643799, "reward_std": 0.5968608856201172, "rewards/rollout_reward_func/mean": 0.5675632953643799, "rewards/rollout_reward_func/std": 0.586036205291748, "sampling/importance_sampling_ratio/max": 1.5584123134613037, "sampling/importance_sampling_ratio/mean": 1.025990605354309, "sampling/importance_sampling_ratio/min": 0.8289830088615417, "sampling/sampling_logp_difference/max": 0.42086267471313477, "sampling/sampling_logp_difference/mean": 0.02880079299211502, "step": 91, "step_time": 9.654203116999497 }, { "clip_ratio/high_max": 0.041666666977107525, "clip_ratio/high_mean": 0.020833333488553762, "clip_ratio/low_mean": 0.03298611147329211, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0538194440305233, "entropy": 0.3738677240908146, "epoch": 0.00092, "grad_norm": 0.19807910919189453, "kl": 0.9962515309453011, "learning_rate": 4.896451759113986e-05, "loss": 0.004, "step": 92, "step_time": 5.273799608000672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 904.0, "completions/max_terminated_length": 904.0, "completions/mean_length": 296.65625, "completions/mean_terminated_length": 296.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3376490995287895, "epoch": 0.00093, "frac_reward_zero_std": 0.0, "grad_norm": 0.27011072635650635, "kl": 0.946666844189167, "learning_rate": 4.896105893375185e-05, "loss": 0.0054, "num_tokens": 1919838.0, "reward": 0.5483165383338928, "reward_std": 0.7140357494354248, "rewards/rollout_reward_func/mean": 0.5483165383338928, "rewards/rollout_reward_func/std": 0.7231590151786804, "sampling/importance_sampling_ratio/max": 1.1489921808242798, "sampling/importance_sampling_ratio/mean": 0.992035984992981, "sampling/importance_sampling_ratio/min": 0.774271547794342, "sampling/sampling_logp_difference/max": 0.21634197235107422, "sampling/sampling_logp_difference/mean": 0.016783270984888077, "step": 93, "step_time": 8.599308044999589 }, { "clip_ratio/high_max": 0.030357143841683865, "clip_ratio/high_mean": 0.015178571920841932, "clip_ratio/low_mean": 0.04642857285216451, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.06160714477300644, "entropy": 0.33248958736658096, "epoch": 0.00094, "grad_norm": 0.2604471743106842, "kl": 1.2364847101271152, "learning_rate": 4.895743961378526e-05, "loss": 0.0048, "step": 94, "step_time": 3.9234093270003996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 913.0, "completions/max_terminated_length": 913.0, "completions/mean_length": 300.40625, "completions/mean_terminated_length": 283.19354248046875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5370058435946703, "epoch": 0.00095, "frac_reward_zero_std": 0.0, "grad_norm": 0.4523778557777405, "kl": 0.8504671975970268, "learning_rate": 4.895365965921506e-05, "loss": -0.0322, "num_tokens": 1956744.0, "reward": 0.6645392179489136, "reward_std": 0.5561251640319824, "rewards/rollout_reward_func/mean": 0.6645392179489136, "rewards/rollout_reward_func/std": 0.563211977481842, "sampling/importance_sampling_ratio/max": 2.3387820720672607, "sampling/importance_sampling_ratio/mean": 1.0176174640655518, "sampling/importance_sampling_ratio/min": 0.6243864893913269, "sampling/sampling_logp_difference/max": 0.7486313581466675, "sampling/sampling_logp_difference/mean": 0.04233343154191971, "step": 95, "step_time": 7.66211360699981 }, { "clip_ratio/high_max": 0.031250000931322575, "clip_ratio/high_mean": 0.015625000465661287, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125000046566129, "entropy": 0.539300974458456, "epoch": 0.00096, "grad_norm": 0.38931262493133545, "kl": 0.8498810715973377, "learning_rate": 4.8949719099257796e-05, "loss": -0.0357, "step": 96, "step_time": 3.93919282299953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 868.0, "completions/max_terminated_length": 868.0, "completions/mean_length": 337.15625, "completions/mean_terminated_length": 337.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32130019925534725, "epoch": 0.00097, "frac_reward_zero_std": 0.0, "grad_norm": 0.3449684977531433, "kl": 0.697452500462532, "learning_rate": 4.8945617964371426e-05, "loss": -0.0089, "num_tokens": 1996381.0, "reward": 0.6907978057861328, "reward_std": 0.5098336935043335, "rewards/rollout_reward_func/mean": 0.6907978057861328, "rewards/rollout_reward_func/std": 0.5023015737533569, "sampling/importance_sampling_ratio/max": 1.6109434366226196, "sampling/importance_sampling_ratio/mean": 1.0275741815567017, "sampling/importance_sampling_ratio/min": 0.6992088556289673, "sampling/sampling_logp_difference/max": 0.5313293933868408, "sampling/sampling_logp_difference/mean": 0.02283405140042305, "step": 97, "step_time": 7.569694327999741 }, { "clip_ratio/high_max": 0.08964646700769663, "clip_ratio/high_mean": 0.044823233503848314, "clip_ratio/low_mean": 0.024801588151603937, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.06962482165545225, "entropy": 0.30479357950389385, "epoch": 0.00098, "grad_norm": 0.08972884714603424, "kl": 0.6897191107273102, "learning_rate": 4.894135628625501e-05, "loss": -0.0109, "step": 98, "step_time": 3.879929144000016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 420.71875, "completions/mean_terminated_length": 420.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.34708523005247116, "epoch": 0.00099, "frac_reward_zero_std": 0.0, "grad_norm": 0.283402681350708, "kl": 1.0675002336502075, "learning_rate": 4.8936934097848537e-05, "loss": 0.0181, "num_tokens": 2038738.0, "reward": 0.48658907413482666, "reward_std": 0.7091055512428284, "rewards/rollout_reward_func/mean": 0.48658907413482666, "rewards/rollout_reward_func/std": 0.7662201523780823, "sampling/importance_sampling_ratio/max": 1.2019175291061401, "sampling/importance_sampling_ratio/mean": 0.981643557548523, "sampling/importance_sampling_ratio/min": 0.6659961938858032, "sampling/sampling_logp_difference/max": 0.4303722381591797, "sampling/sampling_logp_difference/mean": 0.025750668719410896, "step": 99, "step_time": 10.053810656999985 }, { "clip_ratio/high_max": 0.028125000186264515, "clip_ratio/high_mean": 0.014062500093132257, "clip_ratio/low_mean": 0.053927432745695114, "clip_ratio/low_min": 0.03219697065651417, "clip_ratio/region_mean": 0.06798993283882737, "entropy": 0.34952089935541153, "epoch": 0.001, "grad_norm": 0.21945352852344513, "kl": 1.0356716588139534, "learning_rate": 4.893235143333261e-05, "loss": 0.0167, "step": 100, "step_time": 5.759075586999643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 332.0, "completions/mean_terminated_length": 332.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29248981177806854, "epoch": 0.00101, "frac_reward_zero_std": 0.25, "grad_norm": 0.348386287689209, "kl": 0.7971875295042992, "learning_rate": 4.8927608328128215e-05, "loss": 0.0033, "num_tokens": 2076502.0, "reward": 0.6723272204399109, "reward_std": 0.5028443932533264, "rewards/rollout_reward_func/mean": 0.6723272204399109, "rewards/rollout_reward_func/std": 0.609272837638855, "sampling/importance_sampling_ratio/max": 1.4461146593093872, "sampling/importance_sampling_ratio/mean": 1.010133981704712, "sampling/importance_sampling_ratio/min": 0.6710712909698486, "sampling/sampling_logp_difference/max": 0.3692469596862793, "sampling/sampling_logp_difference/mean": 0.020797623321413994, "step": 101, "step_time": 8.821346739999854 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.046875000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.054687500931322575, "entropy": 0.30501324124634266, "epoch": 0.00102, "grad_norm": 0.08744289726018906, "kl": 0.7859054505825043, "learning_rate": 4.892270481889644e-05, "loss": 0.002, "step": 102, "step_time": 4.868404262000013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1704.0, "completions/max_terminated_length": 1704.0, "completions/mean_length": 458.3125, "completions/mean_terminated_length": 458.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.41537507623434067, "epoch": 0.00103, "frac_reward_zero_std": 0.0, "grad_norm": 0.1858036071062088, "kl": 0.7898042891174555, "learning_rate": 4.8917640943538186e-05, "loss": 0.0049, "num_tokens": 2120647.0, "reward": 0.5818654894828796, "reward_std": 0.7007519006729126, "rewards/rollout_reward_func/mean": 0.5818654894828796, "rewards/rollout_reward_func/std": 0.6735209226608276, "sampling/importance_sampling_ratio/max": 1.364448070526123, "sampling/importance_sampling_ratio/mean": 1.0089147090911865, "sampling/importance_sampling_ratio/min": 0.8006302714347839, "sampling/sampling_logp_difference/max": 0.20440959930419922, "sampling/sampling_logp_difference/mean": 0.01901872083544731, "step": 103, "step_time": 9.746085252999592 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.014062500093132257, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.029687500093132257, "entropy": 0.46384981647133827, "epoch": 0.00104, "grad_norm": 0.1592055708169937, "kl": 0.7630265727639198, "learning_rate": 4.891241674119389e-05, "loss": 0.004, "step": 104, "step_time": 5.237503078999907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1550.0, "completions/max_terminated_length": 1550.0, "completions/mean_length": 494.0, "completions/mean_terminated_length": 494.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4149354249238968, "epoch": 0.00105, "frac_reward_zero_std": 0.0, "grad_norm": 0.2243860363960266, "kl": 0.8705751895904541, "learning_rate": 4.89070322522432e-05, "loss": -0.0135, "num_tokens": 2164798.0, "reward": 0.6421203017234802, "reward_std": 0.6729966402053833, "rewards/rollout_reward_func/mean": 0.6421203017234802, "rewards/rollout_reward_func/std": 0.667771577835083, "sampling/importance_sampling_ratio/max": 1.0738722085952759, "sampling/importance_sampling_ratio/mean": 0.9859483242034912, "sampling/importance_sampling_ratio/min": 0.7573080658912659, "sampling/sampling_logp_difference/max": 0.3744626045227051, "sampling/sampling_logp_difference/mean": 0.01800359971821308, "step": 105, "step_time": 9.203155722000247 }, { "clip_ratio/high_max": 0.04583333432674408, "clip_ratio/high_mean": 0.026388889644294977, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04722222359851003, "entropy": 0.46597128361463547, "epoch": 0.00106, "grad_norm": 0.13234105706214905, "kl": 0.8086090385913849, "learning_rate": 4.890148751830467e-05, "loss": -0.0146, "step": 106, "step_time": 6.097045929999922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1435.0, "completions/max_terminated_length": 1435.0, "completions/mean_length": 375.28125, "completions/mean_terminated_length": 375.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4657408334314823, "epoch": 0.00107, "frac_reward_zero_std": 0.0, "grad_norm": 0.5190496444702148, "kl": 0.7547367475926876, "learning_rate": 4.8895782582235457e-05, "loss": -0.0064, "num_tokens": 2205033.0, "reward": 0.48707592487335205, "reward_std": 0.7952094078063965, "rewards/rollout_reward_func/mean": 0.48707592487335205, "rewards/rollout_reward_func/std": 0.769834041595459, "sampling/importance_sampling_ratio/max": 1.2341859340667725, "sampling/importance_sampling_ratio/mean": 1.0312683582305908, "sampling/importance_sampling_ratio/min": 0.8317000865936279, "sampling/sampling_logp_difference/max": 0.2644919157028198, "sampling/sampling_logp_difference/mean": 0.018736835569143295, "step": 107, "step_time": 8.791207673999907 }, { "clip_ratio/high_max": 0.06770833395421505, "clip_ratio/high_mean": 0.033854166977107525, "clip_ratio/low_mean": 0.10315205622464418, "clip_ratio/low_min": 0.05005411431193352, "clip_ratio/region_mean": 0.13700622506439686, "entropy": 0.4854657016694546, "epoch": 0.00108, "grad_norm": 0.16963227093219757, "kl": 0.6951050534844398, "learning_rate": 4.888991748813098e-05, "loss": -0.0095, "step": 108, "step_time": 4.746613444999639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1677.0, "completions/max_terminated_length": 1677.0, "completions/mean_length": 389.28125, "completions/mean_terminated_length": 389.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.51156085729599, "epoch": 0.00109, "frac_reward_zero_std": 0.25, "grad_norm": 0.5092929601669312, "kl": 0.5310743637382984, "learning_rate": 4.8883892281324544e-05, "loss": 0.0238, "num_tokens": 2245073.0, "reward": 0.6805089712142944, "reward_std": 0.5119286775588989, "rewards/rollout_reward_func/mean": 0.6805089712142944, "rewards/rollout_reward_func/std": 0.6464856863021851, "sampling/importance_sampling_ratio/max": 1.1216039657592773, "sampling/importance_sampling_ratio/mean": 1.0093355178833008, "sampling/importance_sampling_ratio/min": 0.9002077579498291, "sampling/sampling_logp_difference/max": 0.4685549736022949, "sampling/sampling_logp_difference/mean": 0.019869904965162277, "step": 109, "step_time": 9.446661794999727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.01854395680129528, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01854395680129528, "entropy": 0.49046940729022026, "epoch": 0.0011, "grad_norm": 2.091674566268921, "kl": 2.1614133957773447, "learning_rate": 4.887770700838704e-05, "loss": 0.036, "step": 110, "step_time": 5.199965765000115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1409.0, "completions/max_terminated_length": 1409.0, "completions/mean_length": 401.15625, "completions/mean_terminated_length": 385.45159912109375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5895019881427288, "epoch": 0.00111, "frac_reward_zero_std": 0.0, "grad_norm": 0.5211045145988464, "kl": 0.9284237064421177, "learning_rate": 4.887136171712659e-05, "loss": 0.0407, "num_tokens": 2287010.0, "reward": 0.6681247353553772, "reward_std": 0.5936824083328247, "rewards/rollout_reward_func/mean": 0.6681247353553772, "rewards/rollout_reward_func/std": 0.6150612831115723, "sampling/importance_sampling_ratio/max": 1.147656798362732, "sampling/importance_sampling_ratio/mean": 0.9910237789154053, "sampling/importance_sampling_ratio/min": 0.6744851469993591, "sampling/sampling_logp_difference/max": 0.23152399063110352, "sampling/sampling_logp_difference/mean": 0.022590596228837967, "step": 111, "step_time": 8.720140223000044 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.021875000093132257, "entropy": 0.5971599109470844, "epoch": 0.00112, "grad_norm": 0.32093295454978943, "kl": 0.5909958239644766, "learning_rate": 4.886485645658808e-05, "loss": 0.035, "step": 112, "step_time": 4.732070240999747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2384.0, "completions/max_terminated_length": 2384.0, "completions/mean_length": 531.625, "completions/mean_terminated_length": 531.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6610807999968529, "epoch": 0.00113, "frac_reward_zero_std": 0.25, "grad_norm": 0.3196781575679779, "kl": 0.3455932103097439, "learning_rate": 4.885819127705295e-05, "loss": -0.0113, "num_tokens": 2333327.0, "reward": 0.6677836179733276, "reward_std": 0.42867428064346313, "rewards/rollout_reward_func/mean": 0.6677836179733276, "rewards/rollout_reward_func/std": 0.5596867799758911, "sampling/importance_sampling_ratio/max": 1.2739051580429077, "sampling/importance_sampling_ratio/mean": 0.9938734173774719, "sampling/importance_sampling_ratio/min": 0.7522222399711609, "sampling/sampling_logp_difference/max": 0.17239916324615479, "sampling/sampling_logp_difference/mean": 0.019473666325211525, "step": 113, "step_time": 12.457034938999868 }, { "clip_ratio/high_max": 0.04062500037252903, "clip_ratio/high_mean": 0.020312500186264515, "clip_ratio/low_mean": 0.021875000093132257, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04218750027939677, "entropy": 0.6656127385795116, "epoch": 0.00114, "grad_norm": 0.20381465554237366, "kl": 0.4438517093658447, "learning_rate": 4.885136623003862e-05, "loss": -0.0125, "step": 114, "step_time": 6.543336224999848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 376.15625, "completions/mean_terminated_length": 376.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.686970230191946, "epoch": 0.00115, "frac_reward_zero_std": 0.0, "grad_norm": 0.2844456732273102, "kl": 0.3724983148276806, "learning_rate": 4.8844381368298244e-05, "loss": -0.0156, "num_tokens": 2374401.0, "reward": 0.5769393444061279, "reward_std": 0.6980675458908081, "rewards/rollout_reward_func/mean": 0.5769393444061279, "rewards/rollout_reward_func/std": 0.68136066198349, "sampling/importance_sampling_ratio/max": 1.176881194114685, "sampling/importance_sampling_ratio/mean": 0.9943292140960693, "sampling/importance_sampling_ratio/min": 0.8016956448554993, "sampling/sampling_logp_difference/max": 0.22097015380859375, "sampling/sampling_logp_difference/mean": 0.012247934006154537, "step": 115, "step_time": 8.87338338099994 }, { "clip_ratio/high_max": 0.030624999664723873, "clip_ratio/high_mean": 0.015312499832361937, "clip_ratio/low_mean": 0.046875000931322575, "clip_ratio/low_min": 0.02083333395421505, "clip_ratio/region_mean": 0.06218750076368451, "entropy": 0.6880388557910919, "epoch": 0.00116, "grad_norm": 0.13635769486427307, "kl": 0.36872768215835094, "learning_rate": 4.88372367458202e-05, "loss": -0.0181, "step": 116, "step_time": 4.81137733200012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1587.0, "completions/max_terminated_length": 1587.0, "completions/mean_length": 608.75, "completions/mean_terminated_length": 608.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7028424069285393, "epoch": 0.00117, "frac_reward_zero_std": 0.0, "grad_norm": 0.24298715591430664, "kl": 0.25010690093040466, "learning_rate": 4.882993241782774e-05, "loss": -0.0032, "num_tokens": 2423140.0, "reward": 0.5656996369361877, "reward_std": 0.7921057939529419, "rewards/rollout_reward_func/mean": 0.5656996369361877, "rewards/rollout_reward_func/std": 0.7872902154922485, "sampling/importance_sampling_ratio/max": 1.1782552003860474, "sampling/importance_sampling_ratio/mean": 1.0191056728363037, "sampling/importance_sampling_ratio/min": 0.8348630666732788, "sampling/sampling_logp_difference/max": 0.16581392288208008, "sampling/sampling_logp_difference/mean": 0.02004200592637062, "step": 117, "step_time": 9.74947971000006 }, { "clip_ratio/high_max": 0.06458333507180214, "clip_ratio/high_mean": 0.03229166753590107, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.047916668467223644, "entropy": 0.6947145015001297, "epoch": 0.00118, "grad_norm": 0.1751384437084198, "kl": 0.2865264043211937, "learning_rate": 4.88224684407785e-05, "loss": -0.0045, "step": 118, "step_time": 5.267383991000088 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 454.90625, "completions/mean_terminated_length": 454.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5391334891319275, "epoch": 0.00119, "frac_reward_zero_std": 0.5, "grad_norm": 0.30329176783561707, "kl": 0.41470838710665703, "learning_rate": 4.881484487236413e-05, "loss": 0.0091, "num_tokens": 2466229.0, "reward": 0.8402458429336548, "reward_std": 0.366376131772995, "rewards/rollout_reward_func/mean": 0.8402458429336548, "rewards/rollout_reward_func/std": 0.5211698412895203, "sampling/importance_sampling_ratio/max": 1.1877601146697998, "sampling/importance_sampling_ratio/mean": 1.0036559104919434, "sampling/importance_sampling_ratio/min": 0.8798485994338989, "sampling/sampling_logp_difference/max": 0.15340805053710938, "sampling/sampling_logp_difference/mean": 0.017243709415197372, "step": 119, "step_time": 9.327996637999604 }, { "clip_ratio/high_max": 0.025240384973585606, "clip_ratio/high_mean": 0.012620192486792803, "clip_ratio/low_mean": 0.025240384973585606, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03786057699471712, "entropy": 0.5177800320088863, "epoch": 0.0012, "grad_norm": 0.11791674792766571, "kl": 0.5205857176333666, "learning_rate": 4.8807061771509805e-05, "loss": 0.0082, "step": 120, "step_time": 6.138786563999702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1587.0, "completions/max_terminated_length": 1587.0, "completions/mean_length": 548.96875, "completions/mean_terminated_length": 548.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5884120725095272, "epoch": 0.00121, "frac_reward_zero_std": 0.0, "grad_norm": 0.39420831203460693, "kl": 0.3251846395432949, "learning_rate": 4.879911919837378e-05, "loss": -0.0017, "num_tokens": 2513146.0, "reward": 0.5382916331291199, "reward_std": 0.838129997253418, "rewards/rollout_reward_func/mean": 0.5382916331291199, "rewards/rollout_reward_func/std": 0.8265777230262756, "sampling/importance_sampling_ratio/max": 1.1946109533309937, "sampling/importance_sampling_ratio/mean": 1.0127087831497192, "sampling/importance_sampling_ratio/min": 0.9002626538276672, "sampling/sampling_logp_difference/max": 0.11806881427764893, "sampling/sampling_logp_difference/mean": 0.014233984053134918, "step": 121, "step_time": 9.541757071999655 }, { "clip_ratio/high_max": 0.15729167126119137, "clip_ratio/high_mean": 0.08645833656191826, "clip_ratio/low_mean": 0.09375000279396772, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1802083384245634, "entropy": 0.49986959993839264, "epoch": 0.00122, "grad_norm": 0.10756990313529968, "kl": 0.42414139956235886, "learning_rate": 4.8791017214346926e-05, "loss": -0.0057, "step": 122, "step_time": 5.197961399999713 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 318.875, "completions/mean_terminated_length": 318.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3749117311090231, "epoch": 0.00123, "frac_reward_zero_std": 0.0, "grad_norm": 0.2530548870563507, "kl": 0.34815033711493015, "learning_rate": 4.878275588205226e-05, "loss": 0.0024, "num_tokens": 2551220.0, "reward": 0.5038632750511169, "reward_std": 0.8508215546607971, "rewards/rollout_reward_func/mean": 0.5038632750511169, "rewards/rollout_reward_func/std": 0.8275800943374634, "sampling/importance_sampling_ratio/max": 1.1606080532073975, "sampling/importance_sampling_ratio/mean": 1.0029714107513428, "sampling/importance_sampling_ratio/min": 0.8633275032043457, "sampling/sampling_logp_difference/max": 0.14810752868652344, "sampling/sampling_logp_difference/mean": 0.01141263172030449, "step": 123, "step_time": 8.80498739199993 }, { "clip_ratio/high_max": 0.05208333395421505, "clip_ratio/high_mean": 0.026041666977107525, "clip_ratio/low_mean": 0.049479166977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.07552083395421505, "entropy": 0.32310711592435837, "epoch": 0.00124, "grad_norm": 0.1520189642906189, "kl": 0.49093029648065567, "learning_rate": 4.8774335265344455e-05, "loss": 0.0011, "step": 124, "step_time": 4.907066567999664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 484.59375, "completions/mean_terminated_length": 484.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.30211961828172207, "epoch": 0.00125, "frac_reward_zero_std": 0.0, "grad_norm": 0.31396931409835815, "kl": 0.5337911732494831, "learning_rate": 4.876575542930933e-05, "loss": -0.0049, "num_tokens": 2595467.0, "reward": 0.7804510593414307, "reward_std": 0.6209782361984253, "rewards/rollout_reward_func/mean": 0.7804510593414307, "rewards/rollout_reward_func/std": 0.6066991090774536, "sampling/importance_sampling_ratio/max": 1.1829818487167358, "sampling/importance_sampling_ratio/mean": 1.0016114711761475, "sampling/importance_sampling_ratio/min": 0.617740273475647, "sampling/sampling_logp_difference/max": 0.4819817543029785, "sampling/sampling_logp_difference/mean": 0.01940152794122696, "step": 125, "step_time": 9.39302363100046 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.032291666604578495, "clip_ratio/low_min": 0.012500000186264515, "clip_ratio/region_mean": 0.032291666604578495, "entropy": 0.25741456635296345, "epoch": 0.00126, "grad_norm": 0.23076754808425903, "kl": 0.7318570390343666, "learning_rate": 4.87570164402634e-05, "loss": -0.0061, "step": 126, "step_time": 5.127240323999786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 417.625, "completions/mean_terminated_length": 417.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17685433896258473, "epoch": 0.00127, "frac_reward_zero_std": 0.5, "grad_norm": 0.0678916946053505, "kl": 0.3959597535431385, "learning_rate": 4.874811836575329e-05, "loss": 0.0007, "num_tokens": 2637041.0, "reward": 0.7829650640487671, "reward_std": 0.3960499167442322, "rewards/rollout_reward_func/mean": 0.7829650640487671, "rewards/rollout_reward_func/std": 0.5972782969474792, "sampling/importance_sampling_ratio/max": 1.1238112449645996, "sampling/importance_sampling_ratio/mean": 1.011080265045166, "sampling/importance_sampling_ratio/min": 0.9508895874023438, "sampling/sampling_logp_difference/max": 0.09369659423828125, "sampling/sampling_logp_difference/mean": 0.005197607912123203, "step": 127, "step_time": 9.882357544000115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.16988810384646058, "epoch": 0.00128, "grad_norm": 0.16435672342777252, "kl": 0.40043333172798157, "learning_rate": 4.873906127455528e-05, "loss": 0.0005, "step": 128, "step_time": 4.889399568999806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 554.6875, "completions/mean_terminated_length": 554.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3650151174515486, "epoch": 0.00129, "frac_reward_zero_std": 0.25, "grad_norm": 0.4527609050273895, "kl": 1.52246630564332, "learning_rate": 4.872984523667473e-05, "loss": 0.0265, "num_tokens": 2685436.0, "reward": 0.8028676509857178, "reward_std": 0.42521458864212036, "rewards/rollout_reward_func/mean": 0.8028676509857178, "rewards/rollout_reward_func/std": 0.4866582155227661, "sampling/importance_sampling_ratio/max": 1.106960415840149, "sampling/importance_sampling_ratio/mean": 0.9689699411392212, "sampling/importance_sampling_ratio/min": 0.6218124628067017, "sampling/sampling_logp_difference/max": 0.2074112892150879, "sampling/sampling_logp_difference/mean": 0.01872650533914566, "step": 129, "step_time": 9.693799490999936 }, { "clip_ratio/high_max": 0.03645833395421505, "clip_ratio/high_mean": 0.018229166977107525, "clip_ratio/low_mean": 0.031387061811983585, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04961622878909111, "entropy": 0.39007267355918884, "epoch": 0.0013, "grad_norm": 0.1946258693933487, "kl": 0.9385641776025295, "learning_rate": 4.872047032334557e-05, "loss": 0.0202, "step": 130, "step_time": 5.106162215999802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1560.0, "completions/max_terminated_length": 1560.0, "completions/mean_length": 462.625, "completions/mean_terminated_length": 462.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22926532849669456, "epoch": 0.00131, "frac_reward_zero_std": 0.25, "grad_norm": 0.09993653744459152, "kl": 0.4886532723903656, "learning_rate": 4.871093660702971e-05, "loss": 0.0026, "num_tokens": 2729089.0, "reward": 0.7490109801292419, "reward_std": 0.5244057178497314, "rewards/rollout_reward_func/mean": 0.7490109801292419, "rewards/rollout_reward_func/std": 0.6161410212516785, "sampling/importance_sampling_ratio/max": 1.038031816482544, "sampling/importance_sampling_ratio/mean": 0.9962401390075684, "sampling/importance_sampling_ratio/min": 0.8687638640403748, "sampling/sampling_logp_difference/max": 0.13899946212768555, "sampling/sampling_logp_difference/mean": 0.005661217030137777, "step": 131, "step_time": 9.345564473999957 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018229166977107525, "entropy": 0.2614561803638935, "epoch": 0.00132, "grad_norm": 0.08193816244602203, "kl": 0.48142994195222855, "learning_rate": 4.870124416141651e-05, "loss": 0.0022, "step": 132, "step_time": 5.144033742000147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 409.53125, "completions/mean_terminated_length": 409.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3479697434231639, "epoch": 0.00133, "frac_reward_zero_std": 0.0, "grad_norm": 0.17612224817276, "kl": 0.41523138992488384, "learning_rate": 4.8691393061422215e-05, "loss": -0.0024, "num_tokens": 2770289.0, "reward": 0.6256675720214844, "reward_std": 0.7094040513038635, "rewards/rollout_reward_func/mean": 0.6256675720214844, "rewards/rollout_reward_func/std": 0.7453022003173828, "sampling/importance_sampling_ratio/max": 1.2762497663497925, "sampling/importance_sampling_ratio/mean": 1.004800796508789, "sampling/importance_sampling_ratio/min": 0.7684888243675232, "sampling/sampling_logp_difference/max": 0.26342272758483887, "sampling/sampling_logp_difference/mean": 0.015150820836424828, "step": 133, "step_time": 9.804589071999999 }, { "clip_ratio/high_max": 0.028125000186264515, "clip_ratio/high_mean": 0.014062500093132257, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014062500093132257, "entropy": 0.37649984657764435, "epoch": 0.00134, "grad_norm": 0.14569897949695587, "kl": 0.41635847836732864, "learning_rate": 4.868138338318935e-05, "loss": -0.0038, "step": 134, "step_time": 5.715183079999633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1524.0, "completions/max_terminated_length": 1524.0, "completions/mean_length": 332.6875, "completions/mean_terminated_length": 332.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.317515604197979, "epoch": 0.00135, "frac_reward_zero_std": 0.25, "grad_norm": 0.10398127883672714, "kl": 0.4793325364589691, "learning_rate": 4.8671215204086155e-05, "loss": 0.0091, "num_tokens": 2808697.0, "reward": 0.7700953483581543, "reward_std": 0.4125383794307709, "rewards/rollout_reward_func/mean": 0.7700953483581543, "rewards/rollout_reward_func/std": 0.5088144540786743, "sampling/importance_sampling_ratio/max": 1.0691511631011963, "sampling/importance_sampling_ratio/mean": 1.0069091320037842, "sampling/importance_sampling_ratio/min": 0.894787609577179, "sampling/sampling_logp_difference/max": 0.11113786697387695, "sampling/sampling_logp_difference/mean": 0.007825307548046112, "step": 135, "step_time": 9.045559742999785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3386420141905546, "epoch": 0.00136, "grad_norm": 0.09558766335248947, "kl": 0.4722200036048889, "learning_rate": 4.866088860270597e-05, "loss": 0.0088, "step": 136, "step_time": 5.00480254900026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1785.0, "completions/max_terminated_length": 1785.0, "completions/mean_length": 541.75, "completions/mean_terminated_length": 541.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4426201432943344, "epoch": 0.00137, "frac_reward_zero_std": 0.25, "grad_norm": 0.23036079108715057, "kl": 0.8018119409680367, "learning_rate": 4.865040365886663e-05, "loss": 0.0083, "num_tokens": 2854571.0, "reward": 0.8166435360908508, "reward_std": 0.518610417842865, "rewards/rollout_reward_func/mean": 0.8166435360908508, "rewards/rollout_reward_func/std": 0.5795265436172485, "sampling/importance_sampling_ratio/max": 1.2528612613677979, "sampling/importance_sampling_ratio/mean": 0.9962642788887024, "sampling/importance_sampling_ratio/min": 0.8541568517684937, "sampling/sampling_logp_difference/max": 0.19103074073791504, "sampling/sampling_logp_difference/mean": 0.011917680501937866, "step": 137, "step_time": 10.180492532000471 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.016666667070239782, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03229166707023978, "entropy": 0.45746365189552307, "epoch": 0.00138, "grad_norm": 0.11410095542669296, "kl": 0.7807982992380857, "learning_rate": 4.8639760453609886e-05, "loss": 0.0063, "step": 138, "step_time": 5.565757810000605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1479.0, "completions/max_terminated_length": 1479.0, "completions/mean_length": 501.65625, "completions/mean_terminated_length": 501.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.48148482851684093, "epoch": 0.00139, "frac_reward_zero_std": 0.5, "grad_norm": 0.2096453607082367, "kl": 0.338961198925972, "learning_rate": 4.862895906920071e-05, "loss": -0.0066, "num_tokens": 2899598.0, "reward": 0.7738548517227173, "reward_std": 0.3764772415161133, "rewards/rollout_reward_func/mean": 0.7738548517227173, "rewards/rollout_reward_func/std": 0.5586192607879639, "sampling/importance_sampling_ratio/max": 1.094862937927246, "sampling/importance_sampling_ratio/mean": 0.9907705783843994, "sampling/importance_sampling_ratio/min": 0.7631676197052002, "sampling/sampling_logp_difference/max": 0.2305774688720703, "sampling/sampling_logp_difference/mean": 0.01340179331600666, "step": 139, "step_time": 9.400911802999644 }, { "clip_ratio/high_max": 0.04062500037252903, "clip_ratio/high_mean": 0.020312500186264515, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03072916716337204, "entropy": 0.4655865654349327, "epoch": 0.0014, "grad_norm": 0.09763188660144806, "kl": 0.36102304607629776, "learning_rate": 4.8617999589126694e-05, "loss": -0.0071, "step": 140, "step_time": 5.809514046000231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1560.0, "completions/max_terminated_length": 1560.0, "completions/mean_length": 366.625, "completions/mean_terminated_length": 366.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.393144752131775, "epoch": 0.00141, "frac_reward_zero_std": 0.5, "grad_norm": 0.1970221847295761, "kl": 0.37640607729554176, "learning_rate": 4.8606882098097435e-05, "loss": 0.0023, "num_tokens": 2939057.0, "reward": 0.7572975158691406, "reward_std": 0.4207494854927063, "rewards/rollout_reward_func/mean": 0.7572975158691406, "rewards/rollout_reward_func/std": 0.6524859070777893, "sampling/importance_sampling_ratio/max": 1.0825660228729248, "sampling/importance_sampling_ratio/mean": 1.004056453704834, "sampling/importance_sampling_ratio/min": 0.8073540329933167, "sampling/sampling_logp_difference/max": 0.21437740325927734, "sampling/sampling_logp_difference/mean": 0.00918533280491829, "step": 141, "step_time": 9.174857143000281 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.031250000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.031250000931322575, "entropy": 0.3795996156986803, "epoch": 0.00142, "grad_norm": 0.10265341401100159, "kl": 0.39880412071943283, "learning_rate": 4.859560668204383e-05, "loss": 0.0013, "step": 142, "step_time": 5.074377998000045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1749.0, "completions/max_terminated_length": 1749.0, "completions/mean_length": 393.0625, "completions/mean_terminated_length": 393.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.433885058388114, "epoch": 0.00143, "frac_reward_zero_std": 0.25, "grad_norm": 0.1403685361146927, "kl": 0.3942210227251053, "learning_rate": 4.858417342811743e-05, "loss": 0.0041, "num_tokens": 2980545.0, "reward": 0.7099786996841431, "reward_std": 0.4996805787086487, "rewards/rollout_reward_func/mean": 0.7099786996841431, "rewards/rollout_reward_func/std": 0.5858648419380188, "sampling/importance_sampling_ratio/max": 1.0835371017456055, "sampling/importance_sampling_ratio/mean": 0.9849342107772827, "sampling/importance_sampling_ratio/min": 0.818988025188446, "sampling/sampling_logp_difference/max": 0.19974756240844727, "sampling/sampling_logp_difference/mean": 0.012078540399670601, "step": 143, "step_time": 9.953579866999917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.40710264071822166, "epoch": 0.00144, "grad_norm": 0.1197826936841011, "kl": 0.41946819983422756, "learning_rate": 4.8572582424689784e-05, "loss": 0.0043, "step": 144, "step_time": 5.5641453289999845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 465.96875, "completions/mean_terminated_length": 465.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.35633246414363384, "epoch": 0.00145, "frac_reward_zero_std": 0.25, "grad_norm": 0.17685899138450623, "kl": 0.5475931763648987, "learning_rate": 4.8560833761351724e-05, "loss": 0.0062, "num_tokens": 3023228.0, "reward": 0.7432956695556641, "reward_std": 0.4458003044128418, "rewards/rollout_reward_func/mean": 0.7432956695556641, "rewards/rollout_reward_func/std": 0.5734567046165466, "sampling/importance_sampling_ratio/max": 1.1325602531433105, "sampling/importance_sampling_ratio/mean": 0.997156023979187, "sampling/importance_sampling_ratio/min": 0.8032222986221313, "sampling/sampling_logp_difference/max": 0.27280616760253906, "sampling/sampling_logp_difference/mean": 0.013265181332826614, "step": 145, "step_time": 9.668969617999892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.014062500093132257, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014062500093132257, "entropy": 0.3296446055173874, "epoch": 0.00146, "grad_norm": 0.13739530742168427, "kl": 0.6345520913600922, "learning_rate": 4.854892752891271e-05, "loss": 0.0066, "step": 146, "step_time": 5.279382575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 894.0, "completions/max_terminated_length": 894.0, "completions/mean_length": 421.78125, "completions/mean_terminated_length": 421.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.40208013728260994, "epoch": 0.00147, "frac_reward_zero_std": 0.0, "grad_norm": 0.15122497081756592, "kl": 0.8154167085886002, "learning_rate": 4.853686381940009e-05, "loss": 0.0072, "num_tokens": 3066217.0, "reward": 0.7367901802062988, "reward_std": 0.5132282972335815, "rewards/rollout_reward_func/mean": 0.7367901802062988, "rewards/rollout_reward_func/std": 0.5244065523147583, "sampling/importance_sampling_ratio/max": 1.2193634510040283, "sampling/importance_sampling_ratio/mean": 0.9867029190063477, "sampling/importance_sampling_ratio/min": 0.7125111818313599, "sampling/sampling_logp_difference/max": 0.33915185928344727, "sampling/sampling_logp_difference/mean": 0.0159075316041708, "step": 147, "step_time": 8.977859376999731 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.026041666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03645833395421505, "entropy": 0.39937982708215714, "epoch": 0.00148, "grad_norm": 0.12062593549489975, "kl": 0.7941956147551537, "learning_rate": 4.852464272605841e-05, "loss": 0.0063, "step": 148, "step_time": 3.971411405000026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1409.0, "completions/max_terminated_length": 1409.0, "completions/mean_length": 424.25, "completions/mean_terminated_length": 424.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.41107476502656937, "epoch": 0.00149, "frac_reward_zero_std": 0.0, "grad_norm": 0.35113009810447693, "kl": 0.6088944748044014, "learning_rate": 4.8512264343348696e-05, "loss": 0.0087, "num_tokens": 3108232.0, "reward": 0.6687621474266052, "reward_std": 0.5755384564399719, "rewards/rollout_reward_func/mean": 0.6687621474266052, "rewards/rollout_reward_func/std": 0.5586510896682739, "sampling/importance_sampling_ratio/max": 1.1292555332183838, "sampling/importance_sampling_ratio/mean": 0.9897196292877197, "sampling/importance_sampling_ratio/min": 0.8032168745994568, "sampling/sampling_logp_difference/max": 0.18542873859405518, "sampling/sampling_logp_difference/mean": 0.011107724159955978, "step": 149, "step_time": 8.993635151999797 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.057291668839752674, "clip_ratio/low_min": 0.02083333395421505, "clip_ratio/region_mean": 0.06354166893288493, "entropy": 0.4597155600786209, "epoch": 0.0015, "grad_norm": 0.10752938687801361, "kl": 0.5289982054382563, "learning_rate": 4.849972876694771e-05, "loss": 0.0058, "step": 150, "step_time": 4.878595058999963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1328.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 452.6875, "completions/mean_terminated_length": 452.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5813268758356571, "epoch": 0.00151, "frac_reward_zero_std": 0.25, "grad_norm": 0.7432031631469727, "kl": 0.360255004838109, "learning_rate": 4.8487036093747225e-05, "loss": 0.1044, "num_tokens": 3152201.0, "reward": 0.8055681586265564, "reward_std": 0.400839239358902, "rewards/rollout_reward_func/mean": 0.8055681586265564, "rewards/rollout_reward_func/std": 0.47916683554649353, "sampling/importance_sampling_ratio/max": 1.7297760248184204, "sampling/importance_sampling_ratio/mean": 1.0251097679138184, "sampling/importance_sampling_ratio/min": 0.8095357418060303, "sampling/sampling_logp_difference/max": 0.2663055658340454, "sampling/sampling_logp_difference/mean": 0.01576552726328373, "step": 151, "step_time": 8.577120639000213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.050109649542719126, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.050109649542719126, "entropy": 0.617727056145668, "epoch": 0.00152, "grad_norm": 0.32782182097435, "kl": 0.33087453059852123, "learning_rate": 4.8474186421853266e-05, "loss": 0.09, "step": 152, "step_time": 4.732702530999859 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2380.0, "completions/max_terminated_length": 2380.0, "completions/mean_length": 657.625, "completions/mean_terminated_length": 657.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.594463899731636, "epoch": 0.00153, "frac_reward_zero_std": 0.5, "grad_norm": 0.12798050045967102, "kl": 0.34477805253118277, "learning_rate": 4.8461179850585355e-05, "loss": 0.0077, "num_tokens": 3202706.0, "reward": 0.8446115255355835, "reward_std": 0.3598591685295105, "rewards/rollout_reward_func/mean": 0.8446115255355835, "rewards/rollout_reward_func/std": 0.5111827254295349, "sampling/importance_sampling_ratio/max": 1.266123652458191, "sampling/importance_sampling_ratio/mean": 0.9917253851890564, "sampling/importance_sampling_ratio/min": 0.7768818736076355, "sampling/sampling_logp_difference/max": 0.13852453231811523, "sampling/sampling_logp_difference/mean": 0.014958497136831284, "step": 153, "step_time": 12.006085189000032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "entropy": 0.6353004053235054, "epoch": 0.00154, "grad_norm": 0.1304071843624115, "kl": 0.35100386291742325, "learning_rate": 4.844801648047573e-05, "loss": 0.0074, "step": 154, "step_time": 7.468078461000232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 490.1875, "completions/mean_terminated_length": 490.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.48487958684563637, "epoch": 0.00155, "frac_reward_zero_std": 0.5, "grad_norm": 0.12075866013765335, "kl": 0.30701472982764244, "learning_rate": 4.843469641326861e-05, "loss": -0.0038, "num_tokens": 3246786.0, "reward": 0.7731899619102478, "reward_std": 0.35527950525283813, "rewards/rollout_reward_func/mean": 0.7731899619102478, "rewards/rollout_reward_func/std": 0.5629463195800781, "sampling/importance_sampling_ratio/max": 1.1291555166244507, "sampling/importance_sampling_ratio/mean": 0.982441782951355, "sampling/importance_sampling_ratio/min": 0.795728325843811, "sampling/sampling_logp_difference/max": 0.19081544876098633, "sampling/sampling_logp_difference/mean": 0.012550966814160347, "step": 155, "step_time": 8.958872106999934 }, { "clip_ratio/high_max": 0.041666666977107525, "clip_ratio/high_mean": 0.020833333488553762, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.028645833488553762, "entropy": 0.5056030787527561, "epoch": 0.00156, "grad_norm": 0.06220289319753647, "kl": 0.28399867936968803, "learning_rate": 4.8421219751919355e-05, "loss": -0.0047, "step": 156, "step_time": 4.877732702999992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1650.0, "completions/max_terminated_length": 1650.0, "completions/mean_length": 572.4375, "completions/mean_terminated_length": 572.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5746085569262505, "epoch": 0.00157, "frac_reward_zero_std": 0.5, "grad_norm": 0.3881043493747711, "kl": 1.5709530357271433, "learning_rate": 4.840758660059371e-05, "loss": 0.003, "num_tokens": 3294683.0, "reward": 0.6881698966026306, "reward_std": 0.4501141309738159, "rewards/rollout_reward_func/mean": 0.6881698966026306, "rewards/rollout_reward_func/std": 0.6830360293388367, "sampling/importance_sampling_ratio/max": 1.1254938840866089, "sampling/importance_sampling_ratio/mean": 0.9901334047317505, "sampling/importance_sampling_ratio/min": 0.7888857126235962, "sampling/sampling_logp_difference/max": 0.21750736236572266, "sampling/sampling_logp_difference/mean": 0.01534991804510355, "step": 157, "step_time": 9.868310598999642 }, { "clip_ratio/high_max": 0.01785714365541935, "clip_ratio/high_mean": 0.008928571827709675, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015178571920841932, "entropy": 0.6012807227671146, "epoch": 0.00158, "grad_norm": 0.11642083525657654, "kl": 0.6243079975247383, "learning_rate": 4.8393797064666957e-05, "loss": -0.0025, "step": 158, "step_time": 5.356190552000044 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 435.34375, "completions/mean_terminated_length": 435.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5939459018409252, "epoch": 0.00159, "frac_reward_zero_std": 0.25, "grad_norm": 0.22879034280776978, "kl": 0.21005595847964287, "learning_rate": 4.837985125072316e-05, "loss": -0.0057, "num_tokens": 3336954.0, "reward": 0.6254698634147644, "reward_std": 0.6390626430511475, "rewards/rollout_reward_func/mean": 0.6254698634147644, "rewards/rollout_reward_func/std": 0.74527508020401, "sampling/importance_sampling_ratio/max": 1.1270534992218018, "sampling/importance_sampling_ratio/mean": 1.009194254875183, "sampling/importance_sampling_ratio/min": 0.81285560131073, "sampling/sampling_logp_difference/max": 0.11961746215820312, "sampling/sampling_logp_difference/mean": 0.016185004264116287, "step": 159, "step_time": 8.9701206310001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "entropy": 0.6029087677598, "epoch": 0.0016, "grad_norm": 0.08868234604597092, "kl": 0.2076372094452381, "learning_rate": 4.8365749266554313e-05, "loss": -0.0062, "step": 160, "step_time": 6.140663405000396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2189.0, "completions/max_terminated_length": 2189.0, "completions/mean_length": 589.375, "completions/mean_terminated_length": 592.1612548828125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7767472956329584, "epoch": 0.00161, "frac_reward_zero_std": 0.0, "grad_norm": 0.5091516971588135, "kl": 0.2814192734658718, "learning_rate": 4.8351491221159464e-05, "loss": -0.0166, "num_tokens": 3383626.0, "reward": 0.37949898838996887, "reward_std": 0.8880079984664917, "rewards/rollout_reward_func/mean": 0.37949898838996887, "rewards/rollout_reward_func/std": 0.8607695698738098, "sampling/importance_sampling_ratio/max": 1.3940626382827759, "sampling/importance_sampling_ratio/mean": 1.0044021606445312, "sampling/importance_sampling_ratio/min": 0.8184801340103149, "sampling/sampling_logp_difference/max": 0.1537923812866211, "sampling/sampling_logp_difference/mean": 0.02106192335486412, "step": 161, "step_time": 11.613055542999973 }, { "clip_ratio/high_max": 0.04732789937406778, "clip_ratio/high_mean": 0.02366394968703389, "clip_ratio/low_mean": 0.049218750558793545, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.07288269978016615, "entropy": 0.7618411965668201, "epoch": 0.00162, "grad_norm": 0.2087266743183136, "kl": 0.2768544424325228, "learning_rate": 4.8337077224743955e-05, "loss": -0.0217, "step": 162, "step_time": 6.369438966999951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1659.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 465.875, "completions/mean_terminated_length": 465.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7695397362112999, "epoch": 0.00163, "frac_reward_zero_std": 0.0, "grad_norm": 0.322905033826828, "kl": 0.23880905099213123, "learning_rate": 4.83225073887185e-05, "loss": -0.0085, "num_tokens": 3426765.0, "reward": 0.7100490927696228, "reward_std": 0.6500344276428223, "rewards/rollout_reward_func/mean": 0.7100490927696228, "rewards/rollout_reward_func/std": 0.6434489488601685, "sampling/importance_sampling_ratio/max": 1.077187418937683, "sampling/importance_sampling_ratio/mean": 0.9771279096603394, "sampling/importance_sampling_ratio/min": 0.49560290575027466, "sampling/sampling_logp_difference/max": 0.1651397943496704, "sampling/sampling_logp_difference/mean": 0.017371399328112602, "step": 163, "step_time": 9.8916101049997 }, { "clip_ratio/high_max": 0.018990929704159498, "clip_ratio/high_mean": 0.009495464852079749, "clip_ratio/low_mean": 0.03303571464493871, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.042531179497018456, "entropy": 0.7897908128798008, "epoch": 0.00164, "grad_norm": 0.18842202425003052, "kl": 0.23894082196056843, "learning_rate": 4.830778182569836e-05, "loss": -0.0099, "step": 164, "step_time": 5.334266403999436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1256.0, "completions/max_terminated_length": 1256.0, "completions/mean_length": 268.96875, "completions/mean_terminated_length": 268.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6762913502752781, "epoch": 0.00165, "frac_reward_zero_std": 0.0, "grad_norm": 0.8913102149963379, "kl": 0.5343112703412771, "learning_rate": 4.829290064950246e-05, "loss": 0.0487, "num_tokens": 3462148.0, "reward": 0.49046552181243896, "reward_std": 0.8337488174438477, "rewards/rollout_reward_func/mean": 0.49046552181243896, "rewards/rollout_reward_func/std": 0.8059068918228149, "sampling/importance_sampling_ratio/max": 1.7506853342056274, "sampling/importance_sampling_ratio/mean": 1.0172955989837646, "sampling/importance_sampling_ratio/min": 0.7688634395599365, "sampling/sampling_logp_difference/max": 0.321286678314209, "sampling/sampling_logp_difference/mean": 0.0189798716455698, "step": 165, "step_time": 8.342268313000204 }, { "clip_ratio/high_max": 0.030381944496184587, "clip_ratio/high_mean": 0.015190972248092294, "clip_ratio/low_mean": 0.027529762592166662, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04272073530592024, "entropy": 0.6915403958410025, "epoch": 0.00166, "grad_norm": 0.21865229308605194, "kl": 0.7077426202595234, "learning_rate": 4.8277863975152514e-05, "loss": 0.0452, "step": 166, "step_time": 4.477608841000347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2332.0, "completions/max_terminated_length": 2332.0, "completions/mean_length": 615.5625, "completions/mean_terminated_length": 615.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.884999256581068, "epoch": 0.00167, "frac_reward_zero_std": 0.5, "grad_norm": 0.5599318146705627, "kl": 0.3435104489326477, "learning_rate": 4.826267191887214e-05, "loss": 0.0525, "num_tokens": 3511496.0, "reward": 0.7796090245246887, "reward_std": 0.4180104732513428, "rewards/rollout_reward_func/mean": 0.7796090245246887, "rewards/rollout_reward_func/std": 0.6084023118019104, "sampling/importance_sampling_ratio/max": 1.2910112142562866, "sampling/importance_sampling_ratio/mean": 0.9898958206176758, "sampling/importance_sampling_ratio/min": 0.17416246235370636, "sampling/sampling_logp_difference/max": 0.2850189208984375, "sampling/sampling_logp_difference/mean": 0.0271103847771883, "step": 167, "step_time": 13.324311256000101 }, { "clip_ratio/high_max": 0.051875000819563866, "clip_ratio/high_mean": 0.025937500409781933, "clip_ratio/low_mean": 0.03872053977102041, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.06465804018080235, "entropy": 0.8418506011366844, "epoch": 0.00168, "grad_norm": 0.15048280358314514, "kl": 0.3380285557359457, "learning_rate": 4.824732459808595e-05, "loss": 0.0487, "step": 168, "step_time": 6.57819355200013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 459.59375, "completions/mean_terminated_length": 473.3870849609375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0026399306952953, "epoch": 0.00169, "frac_reward_zero_std": 0.25, "grad_norm": 0.41315239667892456, "kl": 0.3069103052839637, "learning_rate": 4.823182213141865e-05, "loss": 0.0642, "num_tokens": 3554723.0, "reward": 0.6897692084312439, "reward_std": 0.6075426340103149, "rewards/rollout_reward_func/mean": 0.6897692084312439, "rewards/rollout_reward_func/std": 0.7324679493904114, "sampling/importance_sampling_ratio/max": 1.1730722188949585, "sampling/importance_sampling_ratio/mean": 0.9930843114852905, "sampling/importance_sampling_ratio/min": 0.5825884938240051, "sampling/sampling_logp_difference/max": 0.21034526824951172, "sampling/sampling_logp_difference/mean": 0.022011365741491318, "step": 169, "step_time": 9.382694251999965 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.019967039115726948, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.027779539115726948, "entropy": 0.9642381817102432, "epoch": 0.0017, "grad_norm": 0.26722434163093567, "kl": 0.30268789269030094, "learning_rate": 4.821616463869411e-05, "loss": 0.057, "step": 170, "step_time": 4.961753912000859 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2432.0, "completions/max_terminated_length": 2432.0, "completions/mean_length": 462.5, "completions/mean_terminated_length": 462.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6799621358513832, "epoch": 0.00171, "frac_reward_zero_std": 0.0, "grad_norm": 0.1472187042236328, "kl": 0.33222547359764576, "learning_rate": 4.820035224093448e-05, "loss": -0.0057, "num_tokens": 3597541.0, "reward": 0.6227859854698181, "reward_std": 0.6877440214157104, "rewards/rollout_reward_func/mean": 0.6227859854698181, "rewards/rollout_reward_func/std": 0.7067031860351562, "sampling/importance_sampling_ratio/max": 1.1405889987945557, "sampling/importance_sampling_ratio/mean": 1.018579363822937, "sampling/importance_sampling_ratio/min": 0.9481542110443115, "sampling/sampling_logp_difference/max": 0.09863996505737305, "sampling/sampling_logp_difference/mean": 0.012207355350255966, "step": 171, "step_time": 11.821396142999674 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "entropy": 0.6598606780171394, "epoch": 0.00172, "grad_norm": 0.1376795619726181, "kl": 0.32371439412236214, "learning_rate": 4.818438506035919e-05, "loss": -0.0065, "step": 172, "step_time": 6.7516833789998145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1650.0, "completions/max_terminated_length": 1650.0, "completions/mean_length": 419.96875, "completions/mean_terminated_length": 419.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.623671643435955, "epoch": 0.00173, "frac_reward_zero_std": 0.5, "grad_norm": 0.13770316541194916, "kl": 0.19268214516341686, "learning_rate": 4.816826322038408e-05, "loss": 0.0036, "num_tokens": 3639409.0, "reward": 0.7224621772766113, "reward_std": 0.414700448513031, "rewards/rollout_reward_func/mean": 0.7224621772766113, "rewards/rollout_reward_func/std": 0.6702480316162109, "sampling/importance_sampling_ratio/max": 1.152626395225525, "sampling/importance_sampling_ratio/mean": 1.003547191619873, "sampling/importance_sampling_ratio/min": 0.8875309824943542, "sampling/sampling_logp_difference/max": 0.11939239501953125, "sampling/sampling_logp_difference/mean": 0.01295589841902256, "step": 173, "step_time": 9.632739770999933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.6103913262486458, "epoch": 0.00174, "grad_norm": 0.07612433284521103, "kl": 0.19353108573704958, "learning_rate": 4.8151986845620344e-05, "loss": 0.0025, "step": 174, "step_time": 6.518657057000382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2245.0, "completions/max_terminated_length": 2245.0, "completions/mean_length": 457.6875, "completions/mean_terminated_length": 457.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6248960383236408, "epoch": 0.00175, "frac_reward_zero_std": 0.25, "grad_norm": 0.15514691174030304, "kl": 0.40589806996285915, "learning_rate": 4.81355560618737e-05, "loss": -0.0027, "num_tokens": 3683420.0, "reward": 0.7159249782562256, "reward_std": 0.5358838438987732, "rewards/rollout_reward_func/mean": 0.7159249782562256, "rewards/rollout_reward_func/std": 0.635924220085144, "sampling/importance_sampling_ratio/max": 1.123133659362793, "sampling/importance_sampling_ratio/mean": 0.995838463306427, "sampling/importance_sampling_ratio/min": 0.7659582495689392, "sampling/sampling_logp_difference/max": 0.2856675386428833, "sampling/sampling_logp_difference/mean": 0.012542098760604858, "step": 175, "step_time": 11.382912408000266 }, { "clip_ratio/high_max": 0.030357143841683865, "clip_ratio/high_mean": 0.015178571920841932, "clip_ratio/low_mean": 0.0069444444961845875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02212301641702652, "entropy": 0.6150951348245144, "epoch": 0.00176, "grad_norm": 0.06915596127510071, "kl": 0.4101022109389305, "learning_rate": 4.811897099614328e-05, "loss": -0.0038, "step": 176, "step_time": 6.3431571209998765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 362.28125, "completions/mean_terminated_length": 362.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6323211584240198, "epoch": 0.00177, "frac_reward_zero_std": 0.0, "grad_norm": 0.22069989144802094, "kl": 0.60599793959409, "learning_rate": 4.8102231776620765e-05, "loss": 0.0053, "num_tokens": 3723089.0, "reward": 0.42975252866744995, "reward_std": 0.7659915685653687, "rewards/rollout_reward_func/mean": 0.42975252866744995, "rewards/rollout_reward_func/std": 0.797655463218689, "sampling/importance_sampling_ratio/max": 1.186799168586731, "sampling/importance_sampling_ratio/mean": 1.0080021619796753, "sampling/importance_sampling_ratio/min": 0.8315845131874084, "sampling/sampling_logp_difference/max": 0.14984416961669922, "sampling/sampling_logp_difference/mean": 0.014439372345805168, "step": 177, "step_time": 9.378283477999503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02083333395421505, "entropy": 0.6143402215093374, "epoch": 0.00178, "grad_norm": 0.1156022772192955, "kl": 0.5901605896651745, "learning_rate": 4.808533853268932e-05, "loss": 0.0045, "step": 178, "step_time": 5.2250640700010536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1484.0, "completions/max_terminated_length": 1373.0, "completions/mean_length": 417.09375, "completions/mean_terminated_length": 382.6773986816406, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5359524525702, "epoch": 0.00179, "frac_reward_zero_std": 0.0, "grad_norm": 1.1600059270858765, "kl": 3.003987642005086, "learning_rate": 4.8068291394922624e-05, "loss": -0.0021, "num_tokens": 3764034.0, "reward": 0.6502822637557983, "reward_std": 0.7060048580169678, "rewards/rollout_reward_func/mean": 0.6502822637557983, "rewards/rollout_reward_func/std": 0.7049581408500671, "sampling/importance_sampling_ratio/max": 1.131537914276123, "sampling/importance_sampling_ratio/mean": 0.9991644024848938, "sampling/importance_sampling_ratio/min": 0.8358486890792847, "sampling/sampling_logp_difference/max": 0.20786714553833008, "sampling/sampling_logp_difference/mean": 0.008907731622457504, "step": 179, "step_time": 9.217764144999819 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0173611119389534, "clip_ratio/low_min": 0.013888888992369175, "clip_ratio/region_mean": 0.0251736119389534, "entropy": 0.553648829460144, "epoch": 0.0018, "grad_norm": 1.9235937595367432, "kl": 0.45041596703231335, "learning_rate": 4.805109049508384e-05, "loss": 0.0048, "step": 180, "step_time": 5.640090291000433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1497.0, "completions/max_terminated_length": 1497.0, "completions/mean_length": 385.03125, "completions/mean_terminated_length": 385.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5762960463762283, "epoch": 0.00181, "frac_reward_zero_std": 0.0, "grad_norm": 0.12731482088565826, "kl": 0.25559690222144127, "learning_rate": 4.8033735966124644e-05, "loss": -0.0029, "num_tokens": 3804933.0, "reward": 0.6179435849189758, "reward_std": 0.6993986368179321, "rewards/rollout_reward_func/mean": 0.6179435849189758, "rewards/rollout_reward_func/std": 0.7051264643669128, "sampling/importance_sampling_ratio/max": 1.1396548748016357, "sampling/importance_sampling_ratio/mean": 1.0157512426376343, "sampling/importance_sampling_ratio/min": 0.8953897356987, "sampling/sampling_logp_difference/max": 0.14385604858398438, "sampling/sampling_logp_difference/mean": 0.01315722893923521, "step": 181, "step_time": 9.145753420999881 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.5630298778414726, "epoch": 0.00182, "grad_norm": 0.1113315224647522, "kl": 0.25649726390838623, "learning_rate": 4.801622794218412e-05, "loss": -0.0033, "step": 182, "step_time": 5.015544425000371 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2218.0, "completions/max_terminated_length": 2218.0, "completions/mean_length": 483.4375, "completions/mean_terminated_length": 483.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5618644468486309, "epoch": 0.00183, "frac_reward_zero_std": 0.0, "grad_norm": 0.1815018653869629, "kl": 0.24932047445327044, "learning_rate": 4.799856655858779e-05, "loss": -0.0031, "num_tokens": 3850149.0, "reward": 0.6451775431632996, "reward_std": 0.5740936398506165, "rewards/rollout_reward_func/mean": 0.6451775431632996, "rewards/rollout_reward_func/std": 0.6613548398017883, "sampling/importance_sampling_ratio/max": 1.1076174974441528, "sampling/importance_sampling_ratio/mean": 1.0050973892211914, "sampling/importance_sampling_ratio/min": 0.8895362615585327, "sampling/sampling_logp_difference/max": 0.1035306453704834, "sampling/sampling_logp_difference/mean": 0.011391302570700645, "step": 183, "step_time": 11.653426993999346 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.018229166977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02864583395421505, "entropy": 0.5605708733201027, "epoch": 0.00184, "grad_norm": 0.12596969306468964, "kl": 0.25564993917942047, "learning_rate": 4.7980751951846565e-05, "loss": -0.0044, "step": 184, "step_time": 6.469991183999809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 423.375, "completions/mean_terminated_length": 423.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4818386696279049, "epoch": 0.00185, "frac_reward_zero_std": 0.0, "grad_norm": 0.09972500056028366, "kl": 0.22554744686931372, "learning_rate": 4.7962784259655636e-05, "loss": -0.0064, "num_tokens": 3891830.0, "reward": 0.7116200923919678, "reward_std": 0.5962855815887451, "rewards/rollout_reward_func/mean": 0.7116200923919678, "rewards/rollout_reward_func/std": 0.6421465277671814, "sampling/importance_sampling_ratio/max": 1.136235237121582, "sampling/importance_sampling_ratio/mean": 1.0039501190185547, "sampling/importance_sampling_ratio/min": 0.8992865681648254, "sampling/sampling_logp_difference/max": 0.10134744644165039, "sampling/sampling_logp_difference/mean": 0.011287270113825798, "step": 185, "step_time": 9.140488284000185 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.016666667070239782, "entropy": 0.48374253883957863, "epoch": 0.00186, "grad_norm": 0.06369346380233765, "kl": 0.22868698742240667, "learning_rate": 4.794466362089346e-05, "loss": -0.0071, "step": 186, "step_time": 4.961671724000098 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 342.4375, "completions/mean_terminated_length": 342.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4787561874836683, "epoch": 0.00187, "frac_reward_zero_std": 0.0, "grad_norm": 0.18273542821407318, "kl": 0.30101922433823347, "learning_rate": 4.792639017562067e-05, "loss": 0.0047, "num_tokens": 3930027.0, "reward": 0.7157949209213257, "reward_std": 0.616649866104126, "rewards/rollout_reward_func/mean": 0.7157949209213257, "rewards/rollout_reward_func/std": 0.6308570504188538, "sampling/importance_sampling_ratio/max": 1.0971696376800537, "sampling/importance_sampling_ratio/mean": 0.9919716715812683, "sampling/importance_sampling_ratio/min": 0.8644018769264221, "sampling/sampling_logp_difference/max": 0.14320111274719238, "sampling/sampling_logp_difference/mean": 0.012988418340682983, "step": 187, "step_time": 9.43910078099998 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.03645833395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.046875000931322575, "entropy": 0.48749712482094765, "epoch": 0.00188, "grad_norm": 0.08923758566379547, "kl": 0.329339942894876, "learning_rate": 4.7907964065079e-05, "loss": 0.0026, "step": 188, "step_time": 4.858409575999758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 417.1875, "completions/mean_terminated_length": 417.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5569738745689392, "epoch": 0.00189, "frac_reward_zero_std": 0.0, "grad_norm": 0.1635407656431198, "kl": 0.45603544265031815, "learning_rate": 4.788938543169017e-05, "loss": 0.0096, "num_tokens": 3971776.0, "reward": 0.4082822799682617, "reward_std": 0.8333300352096558, "rewards/rollout_reward_func/mean": 0.4082822799682617, "rewards/rollout_reward_func/std": 0.8238674402236938, "sampling/importance_sampling_ratio/max": 1.1292253732681274, "sampling/importance_sampling_ratio/mean": 1.0123379230499268, "sampling/importance_sampling_ratio/min": 0.9084633588790894, "sampling/sampling_logp_difference/max": 0.10609018802642822, "sampling/sampling_logp_difference/mean": 0.012751254253089428, "step": 189, "step_time": 9.18576474199972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.041145834140479565, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.041145834140479565, "entropy": 0.5557082034647465, "epoch": 0.0019, "grad_norm": 0.10322694480419159, "kl": 0.4760703481733799, "learning_rate": 4.7870654419054795e-05, "loss": 0.009, "step": 190, "step_time": 5.015169163000337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 530.28125, "completions/mean_terminated_length": 530.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5976090952754021, "epoch": 0.00191, "frac_reward_zero_std": 0.0, "grad_norm": 0.2889545261859894, "kl": 0.6812055595219135, "learning_rate": 4.785177117195132e-05, "loss": 0.0112, "num_tokens": 4017799.0, "reward": 0.5373107194900513, "reward_std": 0.6339462995529175, "rewards/rollout_reward_func/mean": 0.5373107194900513, "rewards/rollout_reward_func/std": 0.7417227625846863, "sampling/importance_sampling_ratio/max": 1.1195497512817383, "sampling/importance_sampling_ratio/mean": 1.0012648105621338, "sampling/importance_sampling_ratio/min": 0.8292686343193054, "sampling/sampling_logp_difference/max": 0.15431642532348633, "sampling/sampling_logp_difference/mean": 0.014760110527276993, "step": 191, "step_time": 9.624201282999366 }, { "clip_ratio/high_max": 0.03645833395421505, "clip_ratio/high_mean": 0.018229166977107525, "clip_ratio/low_mean": 0.054687500931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0729166679084301, "entropy": 0.6089659631252289, "epoch": 0.00192, "grad_norm": 0.12189386039972305, "kl": 0.6567981243133545, "learning_rate": 4.783273583633483e-05, "loss": 0.0094, "step": 192, "step_time": 5.211877918000937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2326.0, "completions/max_terminated_length": 2326.0, "completions/mean_length": 529.15625, "completions/mean_terminated_length": 529.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5626246519386768, "epoch": 0.00193, "frac_reward_zero_std": 0.0, "grad_norm": 0.2609524428844452, "kl": 0.5942886024713516, "learning_rate": 4.781354855933596e-05, "loss": -0.0135, "num_tokens": 4063095.0, "reward": 0.6823707818984985, "reward_std": 0.6918222308158875, "rewards/rollout_reward_func/mean": 0.6823707818984985, "rewards/rollout_reward_func/std": 0.6955005526542664, "sampling/importance_sampling_ratio/max": 1.1200693845748901, "sampling/importance_sampling_ratio/mean": 0.9822509288787842, "sampling/importance_sampling_ratio/min": 0.8578800559043884, "sampling/sampling_logp_difference/max": 0.24247336387634277, "sampling/sampling_logp_difference/mean": 0.013423077762126923, "step": 193, "step_time": 11.708120971999506 }, { "clip_ratio/high_max": 0.02864583395421505, "clip_ratio/high_mean": 0.014322916977107525, "clip_ratio/low_mean": 0.022693452890962362, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03701636986806989, "entropy": 0.5904402360320091, "epoch": 0.00194, "grad_norm": 0.9050363898277283, "kl": 1.5750022884458303, "learning_rate": 4.779420948925976e-05, "loss": -0.0116, "step": 194, "step_time": 6.9862828370005445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1668.0, "completions/max_terminated_length": 1668.0, "completions/mean_length": 464.6875, "completions/mean_terminated_length": 464.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6378889828920364, "epoch": 0.00195, "frac_reward_zero_std": 0.25, "grad_norm": 0.35638150572776794, "kl": 0.9477353468537331, "learning_rate": 4.777471877558452e-05, "loss": -0.0061, "num_tokens": 4106876.0, "reward": 0.5947796106338501, "reward_std": 0.6541172862052917, "rewards/rollout_reward_func/mean": 0.5947796106338501, "rewards/rollout_reward_func/std": 0.7914800047874451, "sampling/importance_sampling_ratio/max": 1.1026030778884888, "sampling/importance_sampling_ratio/mean": 0.9911418557167053, "sampling/importance_sampling_ratio/min": 0.7932169437408447, "sampling/sampling_logp_difference/max": 0.2318286895751953, "sampling/sampling_logp_difference/mean": 0.013631407171487808, "step": 195, "step_time": 9.748222578999957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03229166707023978, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03229166707023978, "entropy": 0.5846089385449886, "epoch": 0.00196, "grad_norm": 2.613997459411621, "kl": 2.9340174421668053, "learning_rate": 4.775507656896068e-05, "loss": 0.0048, "step": 196, "step_time": 5.295512746999066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1499.0, "completions/max_terminated_length": 1499.0, "completions/mean_length": 372.59375, "completions/mean_terminated_length": 372.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6231891438364983, "epoch": 0.00197, "frac_reward_zero_std": 0.25, "grad_norm": 0.20252110064029694, "kl": 0.4539224337786436, "learning_rate": 4.7735283021209566e-05, "loss": -0.001, "num_tokens": 4147639.0, "reward": 0.650886595249176, "reward_std": 0.5293436050415039, "rewards/rollout_reward_func/mean": 0.650886595249176, "rewards/rollout_reward_func/std": 0.7035650610923767, "sampling/importance_sampling_ratio/max": 1.1159805059432983, "sampling/importance_sampling_ratio/mean": 0.9968987703323364, "sampling/importance_sampling_ratio/min": 0.8236980438232422, "sampling/sampling_logp_difference/max": 0.1238546371459961, "sampling/sampling_logp_difference/mean": 0.01561923697590828, "step": 197, "step_time": 8.941721927000344 }, { "clip_ratio/high_max": 0.0572916679084301, "clip_ratio/high_mean": 0.02864583395421505, "clip_ratio/low_mean": 0.02864583395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0572916679084301, "entropy": 0.5733043402433395, "epoch": 0.00198, "grad_norm": 0.26359421014785767, "kl": 0.45495041459798813, "learning_rate": 4.77153382853223e-05, "loss": -0.0013, "step": 198, "step_time": 4.927502147000723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1695.0, "completions/max_terminated_length": 1695.0, "completions/mean_length": 470.625, "completions/mean_terminated_length": 470.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5621666312217712, "epoch": 0.00199, "frac_reward_zero_std": 0.0, "grad_norm": 0.10419287532567978, "kl": 0.6903238035738468, "learning_rate": 4.76952425154586e-05, "loss": -0.0014, "num_tokens": 4191980.0, "reward": 0.5193423628807068, "reward_std": 0.6935361623764038, "rewards/rollout_reward_func/mean": 0.5193423628807068, "rewards/rollout_reward_func/std": 0.7191073298454285, "sampling/importance_sampling_ratio/max": 1.138864278793335, "sampling/importance_sampling_ratio/mean": 0.9905344843864441, "sampling/importance_sampling_ratio/min": 0.7152730226516724, "sampling/sampling_logp_difference/max": 0.335115909576416, "sampling/sampling_logp_difference/mean": 0.014784537255764008, "step": 199, "step_time": 9.961356987000272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.026041666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.026041666977107525, "entropy": 0.5476225093007088, "epoch": 0.002, "grad_norm": 0.17103320360183716, "kl": 0.9724948909133673, "learning_rate": 4.767499586694555e-05, "loss": -0.0012, "step": 200, "step_time": 5.4415785110004435 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1524.0, "completions/max_terminated_length": 1524.0, "completions/mean_length": 468.125, "completions/mean_terminated_length": 468.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5588520094752312, "epoch": 0.00201, "frac_reward_zero_std": 0.0, "grad_norm": 0.28308868408203125, "kl": 0.8937339968979359, "learning_rate": 4.7654598496276445e-05, "loss": 0.0046, "num_tokens": 4235301.0, "reward": 0.64350426197052, "reward_std": 0.5940038561820984, "rewards/rollout_reward_func/mean": 0.64350426197052, "rewards/rollout_reward_func/std": 0.6127797365188599, "sampling/importance_sampling_ratio/max": 1.3057353496551514, "sampling/importance_sampling_ratio/mean": 1.0097551345825195, "sampling/importance_sampling_ratio/min": 0.8070796728134155, "sampling/sampling_logp_difference/max": 0.21512937545776367, "sampling/sampling_logp_difference/mean": 0.01621514931321144, "step": 201, "step_time": 10.196866378000323 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.021875000093132257, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.029687500093132257, "entropy": 0.5693402625620365, "epoch": 0.00202, "grad_norm": 0.15850810706615448, "kl": 0.7235010452568531, "learning_rate": 4.7634050561109564e-05, "loss": 0.0021, "step": 202, "step_time": 4.996221654000237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 383.96875, "completions/mean_terminated_length": 383.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5092301201075315, "epoch": 0.00203, "frac_reward_zero_std": 0.25, "grad_norm": 0.17632420361042023, "kl": 0.3024257719516754, "learning_rate": 4.761335222026695e-05, "loss": -0.0022, "num_tokens": 4276094.0, "reward": 0.8073577880859375, "reward_std": 0.47695523500442505, "rewards/rollout_reward_func/mean": 0.8073577880859375, "rewards/rollout_reward_func/std": 0.5402930378913879, "sampling/importance_sampling_ratio/max": 1.1032660007476807, "sampling/importance_sampling_ratio/mean": 0.9885110855102539, "sampling/importance_sampling_ratio/min": 0.7745369076728821, "sampling/sampling_logp_difference/max": 0.22403812408447266, "sampling/sampling_logp_difference/mean": 0.013467339798808098, "step": 203, "step_time": 8.828689067000141 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.5254483371973038, "epoch": 0.00204, "grad_norm": 0.14493735134601593, "kl": 0.30556735396385193, "learning_rate": 4.759250363373315e-05, "loss": -0.003, "step": 204, "step_time": 4.854067960000066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2414.0, "completions/max_terminated_length": 2414.0, "completions/mean_length": 343.125, "completions/mean_terminated_length": 343.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.539176934864372, "epoch": 0.00205, "frac_reward_zero_std": 0.25, "grad_norm": 0.13168303668498993, "kl": 0.4261856209486723, "learning_rate": 4.7571504962654064e-05, "loss": 0.0008, "num_tokens": 4314923.0, "reward": 0.7208502888679504, "reward_std": 0.5428961515426636, "rewards/rollout_reward_func/mean": 0.7208502888679504, "rewards/rollout_reward_func/std": 0.6243513822555542, "sampling/importance_sampling_ratio/max": 1.1039471626281738, "sampling/importance_sampling_ratio/mean": 1.0034632682800293, "sampling/importance_sampling_ratio/min": 0.9290780425071716, "sampling/sampling_logp_difference/max": 0.09815335273742676, "sampling/sampling_logp_difference/mean": 0.00987362302839756, "step": 205, "step_time": 11.570836263000274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "entropy": 0.5281401845859364, "epoch": 0.00206, "grad_norm": 0.0704156681895256, "kl": 0.44628653302788734, "learning_rate": 4.755035636933562e-05, "loss": -0.0001, "step": 206, "step_time": 6.575076615000398 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1704.0, "completions/max_terminated_length": 1704.0, "completions/mean_length": 276.34375, "completions/mean_terminated_length": 276.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5329456492327154, "epoch": 0.00207, "frac_reward_zero_std": 0.0, "grad_norm": 0.18349239230155945, "kl": 0.9080077148973942, "learning_rate": 4.752905801724255e-05, "loss": 0.0159, "num_tokens": 4350988.0, "reward": 0.7021842002868652, "reward_std": 0.5376940965652466, "rewards/rollout_reward_func/mean": 0.7021842002868652, "rewards/rollout_reward_func/std": 0.5468345880508423, "sampling/importance_sampling_ratio/max": 1.066788911819458, "sampling/importance_sampling_ratio/mean": 0.9623873233795166, "sampling/importance_sampling_ratio/min": 2.2011106826091975e-10, "sampling/sampling_logp_difference/max": 22.094823837280273, "sampling/sampling_logp_difference/mean": 0.24077388644218445, "step": 207, "step_time": 9.737724583000272 }, { "clip_ratio/high_max": 0.0833333358168602, "clip_ratio/high_mean": 0.052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.052083334885537624, "entropy": 0.5816530399024487, "epoch": 0.00208, "grad_norm": 0.12389452010393143, "kl": 0.8035165313631296, "learning_rate": 4.750761007099712e-05, "loss": 0.0141, "step": 208, "step_time": 5.61415376099967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1578.0, "completions/max_terminated_length": 1578.0, "completions/mean_length": 432.0625, "completions/mean_terminated_length": 432.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7744302153587341, "epoch": 0.00209, "frac_reward_zero_std": 0.5, "grad_norm": 0.2093820571899414, "kl": 0.531598262488842, "learning_rate": 4.7486012696377856e-05, "loss": -0.0039, "num_tokens": 4393824.0, "reward": 0.778666615486145, "reward_std": 0.4200664162635803, "rewards/rollout_reward_func/mean": 0.778666615486145, "rewards/rollout_reward_func/std": 0.6118333339691162, "sampling/importance_sampling_ratio/max": 1.073084831237793, "sampling/importance_sampling_ratio/mean": 0.9929169416427612, "sampling/importance_sampling_ratio/min": 0.7999931573867798, "sampling/sampling_logp_difference/max": 0.17805159091949463, "sampling/sampling_logp_difference/mean": 0.010559668764472008, "step": 209, "step_time": 9.428390328000205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02864583395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02864583395421505, "entropy": 0.7822712697088718, "epoch": 0.0021, "grad_norm": 0.05686241015791893, "kl": 0.537765372544527, "learning_rate": 4.746426606031829e-05, "loss": -0.0052, "step": 210, "step_time": 5.1755727820000175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2504.0, "completions/max_terminated_length": 2504.0, "completions/mean_length": 477.09375, "completions/mean_terminated_length": 477.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7827214896678925, "epoch": 0.00211, "frac_reward_zero_std": 0.0, "grad_norm": 0.2876884937286377, "kl": 0.42221113480627537, "learning_rate": 4.744237033090561e-05, "loss": 0.0119, "num_tokens": 4437891.0, "reward": 0.5439050793647766, "reward_std": 0.8303225040435791, "rewards/rollout_reward_func/mean": 0.5439050793647766, "rewards/rollout_reward_func/std": 0.8194069266319275, "sampling/importance_sampling_ratio/max": 1.1812814474105835, "sampling/importance_sampling_ratio/mean": 0.9924805164337158, "sampling/importance_sampling_ratio/min": 0.8790174126625061, "sampling/sampling_logp_difference/max": 0.12207412719726562, "sampling/sampling_logp_difference/mean": 0.013341287150979042, "step": 211, "step_time": 11.65073931500001 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.027901786379516125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.038318454287946224, "entropy": 0.8001825511455536, "epoch": 0.00212, "grad_norm": 0.10812677443027496, "kl": 0.5679977042600513, "learning_rate": 4.742032567737943e-05, "loss": 0.0102, "step": 212, "step_time": 6.685860527999921 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 467.5625, "completions/mean_terminated_length": 467.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9303464442491531, "epoch": 0.00213, "frac_reward_zero_std": 0.0, "grad_norm": 0.745556116104126, "kl": 1.534761119633913, "learning_rate": 4.739813227013044e-05, "loss": 0.0383, "num_tokens": 4482067.0, "reward": 0.6815237998962402, "reward_std": 0.5294597744941711, "rewards/rollout_reward_func/mean": 0.6815237998962402, "rewards/rollout_reward_func/std": 0.646403968334198, "sampling/importance_sampling_ratio/max": 1.530448317527771, "sampling/importance_sampling_ratio/mean": 1.004296064376831, "sampling/importance_sampling_ratio/min": 0.8211678266525269, "sampling/sampling_logp_difference/max": 0.19701576232910156, "sampling/sampling_logp_difference/mean": 0.02160508558154106, "step": 213, "step_time": 9.534641385999748 }, { "clip_ratio/high_max": 0.0357142873108387, "clip_ratio/high_mean": 0.01785714365541935, "clip_ratio/low_mean": 0.02521306835114956, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04307021200656891, "entropy": 0.9074513018131256, "epoch": 0.00214, "grad_norm": 0.28323450684547424, "kl": 1.0391889587044716, "learning_rate": 4.737579028069909e-05, "loss": 0.0322, "step": 214, "step_time": 5.774909163000757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 419.8125, "completions/mean_terminated_length": 419.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8479634933173656, "epoch": 0.00215, "frac_reward_zero_std": 0.25, "grad_norm": 0.2136605679988861, "kl": 0.707822110503912, "learning_rate": 4.7353299881774285e-05, "loss": -0.0034, "num_tokens": 4523998.0, "reward": 0.6553901433944702, "reward_std": 0.6480010747909546, "rewards/rollout_reward_func/mean": 0.6553901433944702, "rewards/rollout_reward_func/std": 0.7433361411094666, "sampling/importance_sampling_ratio/max": 1.1178195476531982, "sampling/importance_sampling_ratio/mean": 1.0068755149841309, "sampling/importance_sampling_ratio/min": 0.6811454892158508, "sampling/sampling_logp_difference/max": 0.14165925979614258, "sampling/sampling_logp_difference/mean": 0.015301631763577461, "step": 215, "step_time": 9.44890788399971 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.026041666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.041666666977107525, "entropy": 0.8385222963988781, "epoch": 0.00216, "grad_norm": 0.08674466609954834, "kl": 0.7004627324640751, "learning_rate": 4.733066124719202e-05, "loss": -0.0055, "step": 216, "step_time": 5.193420164000145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 481.40625, "completions/mean_terminated_length": 449.1612854003906, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8701792024075985, "epoch": 0.00217, "frac_reward_zero_std": 0.0, "grad_norm": 0.25828832387924194, "kl": 0.48145049437880516, "learning_rate": 4.7307874551934085e-05, "loss": -0.042, "num_tokens": 4569500.0, "reward": 0.6745848655700684, "reward_std": 0.6229050755500793, "rewards/rollout_reward_func/mean": 0.6745848655700684, "rewards/rollout_reward_func/std": 0.6034284830093384, "sampling/importance_sampling_ratio/max": 1.427747368812561, "sampling/importance_sampling_ratio/mean": 1.0127016305923462, "sampling/importance_sampling_ratio/min": 0.8500921130180359, "sampling/sampling_logp_difference/max": 0.1919926404953003, "sampling/sampling_logp_difference/mean": 0.02312799170613289, "step": 217, "step_time": 9.652117010999973 }, { "clip_ratio/high_max": 0.013888888992369175, "clip_ratio/high_mean": 0.0069444444961845875, "clip_ratio/low_mean": 0.0182291679084301, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.025173612404614687, "entropy": 0.8780004158616066, "epoch": 0.00218, "grad_norm": 0.20735907554626465, "kl": 0.46068728528916836, "learning_rate": 4.728493997212665e-05, "loss": -0.0454, "step": 218, "step_time": 5.024762762000137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 312.6875, "completions/mean_terminated_length": 312.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8711941167712212, "epoch": 0.00219, "frac_reward_zero_std": 0.0, "grad_norm": 0.14039258658885956, "kl": 0.46139184944331646, "learning_rate": 4.7261857685038944e-05, "loss": 0.0005, "num_tokens": 4607774.0, "reward": 0.4658811092376709, "reward_std": 0.7860012650489807, "rewards/rollout_reward_func/mean": 0.4658811092376709, "rewards/rollout_reward_func/std": 0.8421306014060974, "sampling/importance_sampling_ratio/max": 1.110649824142456, "sampling/importance_sampling_ratio/mean": 1.013901948928833, "sampling/importance_sampling_ratio/min": 0.912300705909729, "sampling/sampling_logp_difference/max": 0.10361528396606445, "sampling/sampling_logp_difference/mean": 0.010247668251395226, "step": 219, "step_time": 8.934264805001021 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.016666667070239782, "entropy": 0.8980246037244797, "epoch": 0.0022, "grad_norm": 0.11270999163389206, "kl": 0.46421261224895716, "learning_rate": 4.72386278690819e-05, "loss": -0.0006, "step": 220, "step_time": 4.996568869000384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1497.0, "completions/max_terminated_length": 1497.0, "completions/mean_length": 402.9375, "completions/mean_terminated_length": 402.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9574736580252647, "epoch": 0.00221, "frac_reward_zero_std": 0.0, "grad_norm": 0.21521338820457458, "kl": 0.6774773634970188, "learning_rate": 4.7215250703806715e-05, "loss": -0.0029, "num_tokens": 4649593.0, "reward": 0.548141360282898, "reward_std": 0.717764139175415, "rewards/rollout_reward_func/mean": 0.548141360282898, "rewards/rollout_reward_func/std": 0.7271764278411865, "sampling/importance_sampling_ratio/max": 1.1276310682296753, "sampling/importance_sampling_ratio/mean": 0.9955645799636841, "sampling/importance_sampling_ratio/min": 0.8384537696838379, "sampling/sampling_logp_difference/max": 0.1762242317199707, "sampling/sampling_logp_difference/mean": 0.01620984449982643, "step": 221, "step_time": 9.45280324399937 }, { "clip_ratio/high_max": 0.0843253992497921, "clip_ratio/high_mean": 0.04216269962489605, "clip_ratio/low_mean": 0.033854166977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.07601686660200357, "entropy": 0.9986613169312477, "epoch": 0.00222, "grad_norm": 0.11206323653459549, "kl": 0.679485734552145, "learning_rate": 4.719172636990352e-05, "loss": -0.0054, "step": 222, "step_time": 4.897579329000109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1355.0, "completions/max_terminated_length": 1355.0, "completions/mean_length": 314.21875, "completions/mean_terminated_length": 314.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9617488384246826, "epoch": 0.00223, "frac_reward_zero_std": 0.0, "grad_norm": 0.5107397437095642, "kl": 0.6292787529528141, "learning_rate": 4.716805504919996e-05, "loss": 0.0168, "num_tokens": 4687409.0, "reward": 0.6849800944328308, "reward_std": 0.6867934465408325, "rewards/rollout_reward_func/mean": 0.6849800944328308, "rewards/rollout_reward_func/std": 0.6923316121101379, "sampling/importance_sampling_ratio/max": 1.274857759475708, "sampling/importance_sampling_ratio/mean": 1.0032122135162354, "sampling/importance_sampling_ratio/min": 0.782121479511261, "sampling/sampling_logp_difference/max": 0.1316438913345337, "sampling/sampling_logp_difference/mean": 0.01741165667772293, "step": 223, "step_time": 8.381223063000107 }, { "clip_ratio/high_max": 0.14092262275516987, "clip_ratio/high_mean": 0.09345238283276558, "clip_ratio/low_mean": 0.04110577004030347, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1345581505447626, "entropy": 0.8820876926183701, "epoch": 0.00224, "grad_norm": 0.13508740067481995, "kl": 0.6058110650628805, "learning_rate": 4.71442369246598e-05, "loss": 0.0125, "step": 224, "step_time": 4.652309453000271 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1364.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 316.03125, "completions/mean_terminated_length": 292.23333740234375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8740872638300061, "epoch": 0.00225, "frac_reward_zero_std": 0.0, "grad_norm": 0.7127480506896973, "kl": 0.5173007193952799, "learning_rate": 4.712027218038149e-05, "loss": 0.1065, "num_tokens": 4725193.0, "reward": 0.49878501892089844, "reward_std": 0.8473473191261292, "rewards/rollout_reward_func/mean": 0.49878501892089844, "rewards/rollout_reward_func/std": 0.8401981592178345, "sampling/importance_sampling_ratio/max": 1.1449825763702393, "sampling/importance_sampling_ratio/mean": 0.9742230176925659, "sampling/importance_sampling_ratio/min": 0.6034879684448242, "sampling/sampling_logp_difference/max": 0.24596452713012695, "sampling/sampling_logp_difference/mean": 0.02369750663638115, "step": 225, "step_time": 8.814782986000864 }, { "clip_ratio/high_max": 0.0036764706019312143, "clip_ratio/high_mean": 0.0018382353009656072, "clip_ratio/low_mean": 0.09238246735185385, "clip_ratio/low_min": 0.018382353708148003, "clip_ratio/region_mean": 0.09422070265281945, "entropy": 0.7979674492962658, "epoch": 0.00226, "grad_norm": 0.36563077569007874, "kl": 0.49361571855843067, "learning_rate": 4.709616100159674e-05, "loss": 0.0948, "step": 226, "step_time": 4.664960772000541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 462.125, "completions/mean_terminated_length": 462.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5870153941214085, "epoch": 0.00227, "frac_reward_zero_std": 0.25, "grad_norm": 0.05440787225961685, "kl": 0.5618906505405903, "learning_rate": 4.707190357466915e-05, "loss": -0.0106, "num_tokens": 4768569.0, "reward": 0.6994985342025757, "reward_std": 0.45797199010849, "rewards/rollout_reward_func/mean": 0.6994985342025757, "rewards/rollout_reward_func/std": 0.548666775226593, "sampling/importance_sampling_ratio/max": 1.1304652690887451, "sampling/importance_sampling_ratio/mean": 0.9908905029296875, "sampling/importance_sampling_ratio/min": 0.8524116277694702, "sampling/sampling_logp_difference/max": 0.15967750549316406, "sampling/sampling_logp_difference/mean": 0.014225386083126068, "step": 227, "step_time": 8.979643670000769 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "entropy": 0.5445671640336514, "epoch": 0.00228, "grad_norm": 0.05684860795736313, "kl": 0.5741170048713684, "learning_rate": 4.704750008709267e-05, "loss": -0.0105, "step": 228, "step_time": 5.975484795999819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1578.0, "completions/max_terminated_length": 1578.0, "completions/mean_length": 431.71875, "completions/mean_terminated_length": 431.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.512842234224081, "epoch": 0.00229, "frac_reward_zero_std": 0.25, "grad_norm": 0.09396354109048843, "kl": 0.3560453411191702, "learning_rate": 4.7022950727490235e-05, "loss": 0.0018, "num_tokens": 4810984.0, "reward": 0.7390810251235962, "reward_std": 0.42601633071899414, "rewards/rollout_reward_func/mean": 0.7390810251235962, "rewards/rollout_reward_func/std": 0.5251220464706421, "sampling/importance_sampling_ratio/max": 1.0860648155212402, "sampling/importance_sampling_ratio/mean": 0.9924042224884033, "sampling/importance_sampling_ratio/min": 0.8922594785690308, "sampling/sampling_logp_difference/max": 0.11399626731872559, "sampling/sampling_logp_difference/mean": 0.01150705199688673, "step": 229, "step_time": 9.187682237000445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.4838455244898796, "epoch": 0.0023, "grad_norm": 0.059025838971138, "kl": 0.35284582152962685, "learning_rate": 4.6998255685612276e-05, "loss": 0.0014, "step": 230, "step_time": 5.14538049300063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1632.0, "completions/max_terminated_length": 1632.0, "completions/mean_length": 302.15625, "completions/mean_terminated_length": 302.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4470636583864689, "epoch": 0.00231, "frac_reward_zero_std": 0.25, "grad_norm": 0.13428346812725067, "kl": 0.3944466933608055, "learning_rate": 4.697341515233522e-05, "loss": -0.0046, "num_tokens": 4847257.0, "reward": 0.7130756378173828, "reward_std": 0.5295749306678772, "rewards/rollout_reward_func/mean": 0.7130756378173828, "rewards/rollout_reward_func/std": 0.636299192905426, "sampling/importance_sampling_ratio/max": 1.121934413909912, "sampling/importance_sampling_ratio/mean": 0.9991604089736938, "sampling/importance_sampling_ratio/min": 0.8994333148002625, "sampling/sampling_logp_difference/max": 0.13970518112182617, "sampling/sampling_logp_difference/mean": 0.010904578492045403, "step": 231, "step_time": 8.922649107000325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02083333395421505, "entropy": 0.4276432916522026, "epoch": 0.00232, "grad_norm": 0.0641188770532608, "kl": 0.40859245136380196, "learning_rate": 4.694842931966007e-05, "loss": -0.005, "step": 232, "step_time": 5.0475258619999295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1695.0, "completions/max_terminated_length": 1695.0, "completions/mean_length": 401.09375, "completions/mean_terminated_length": 401.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4509815964847803, "epoch": 0.00233, "frac_reward_zero_std": 0.0, "grad_norm": 0.11854226887226105, "kl": 0.500079489313066, "learning_rate": 4.6923298380710904e-05, "loss": 0.0015, "num_tokens": 4888381.0, "reward": 0.7670515775680542, "reward_std": 0.5069683790206909, "rewards/rollout_reward_func/mean": 0.7670515775680542, "rewards/rollout_reward_func/std": 0.5122117400169373, "sampling/importance_sampling_ratio/max": 1.120550513267517, "sampling/importance_sampling_ratio/mean": 1.010507345199585, "sampling/importance_sampling_ratio/min": 0.9343709349632263, "sampling/sampling_logp_difference/max": 0.11318802833557129, "sampling/sampling_logp_difference/mean": 0.007818413898348808, "step": 233, "step_time": 9.29259822199947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.018229166977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018229166977107525, "entropy": 0.4500732533633709, "epoch": 0.00234, "grad_norm": 0.0907149538397789, "kl": 0.547541594132781, "learning_rate": 4.689802252973338e-05, "loss": 0.0007, "step": 234, "step_time": 5.649630017001073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 353.4375, "completions/mean_terminated_length": 353.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.42825985699892044, "epoch": 0.00235, "frac_reward_zero_std": 0.0, "grad_norm": 0.10128292441368103, "kl": 0.40911856666207314, "learning_rate": 4.6872601962093197e-05, "loss": -0.0092, "num_tokens": 4928008.0, "reward": 0.7401810884475708, "reward_std": 0.5689855813980103, "rewards/rollout_reward_func/mean": 0.7401810884475708, "rewards/rollout_reward_func/std": 0.5787392258644104, "sampling/importance_sampling_ratio/max": 1.1189417839050293, "sampling/importance_sampling_ratio/mean": 1.008695125579834, "sampling/importance_sampling_ratio/min": 0.9204097986221313, "sampling/sampling_logp_difference/max": 0.11145162582397461, "sampling/sampling_logp_difference/mean": 0.008665577508509159, "step": 235, "step_time": 9.915355473999625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4322589337825775, "epoch": 0.00236, "grad_norm": 0.11492940038442612, "kl": 0.39569808170199394, "learning_rate": 4.684703687427465e-05, "loss": -0.0089, "step": 236, "step_time": 5.105614787999457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 858.0, "completions/max_terminated_length": 858.0, "completions/mean_length": 294.75, "completions/mean_terminated_length": 294.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4457790069282055, "epoch": 0.00237, "frac_reward_zero_std": 0.25, "grad_norm": 0.08967066556215286, "kl": 0.29646194260567427, "learning_rate": 4.682132746387908e-05, "loss": 0.0039, "num_tokens": 4964955.0, "reward": 0.6116321086883545, "reward_std": 0.5647243857383728, "rewards/rollout_reward_func/mean": 0.6116321086883545, "rewards/rollout_reward_func/std": 0.6676426529884338, "sampling/importance_sampling_ratio/max": 1.096215844154358, "sampling/importance_sampling_ratio/mean": 0.9931606650352478, "sampling/importance_sampling_ratio/min": 0.902008056640625, "sampling/sampling_logp_difference/max": 0.10328149795532227, "sampling/sampling_logp_difference/mean": 0.008864129893481731, "step": 237, "step_time": 6.874635707999914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.44566701259464025, "epoch": 0.00238, "grad_norm": 0.08527178317308426, "kl": 0.298743087798357, "learning_rate": 4.679547392962333e-05, "loss": 0.0033, "step": 238, "step_time": 3.7099793109996426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 495.875, "completions/mean_terminated_length": 495.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5385508127510548, "epoch": 0.00239, "frac_reward_zero_std": 0.0, "grad_norm": 0.15333430469036102, "kl": 0.39105674624443054, "learning_rate": 4.676947647133822e-05, "loss": 0.0045, "num_tokens": 5010365.0, "reward": 0.7098828554153442, "reward_std": 0.5843148827552795, "rewards/rollout_reward_func/mean": 0.7098828554153442, "rewards/rollout_reward_func/std": 0.5894032716751099, "sampling/importance_sampling_ratio/max": 1.1499911546707153, "sampling/importance_sampling_ratio/mean": 1.0110396146774292, "sampling/importance_sampling_ratio/min": 0.9186562299728394, "sampling/sampling_logp_difference/max": 0.13936233520507812, "sampling/sampling_logp_difference/mean": 0.006863999646157026, "step": 239, "step_time": 9.395657881999796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5402183048427105, "epoch": 0.0024, "grad_norm": 0.13026228547096252, "kl": 0.399281769990921, "learning_rate": 4.6743335289967045e-05, "loss": 0.0035, "step": 240, "step_time": 5.219846763000078 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1569.0, "completions/max_terminated_length": 1569.0, "completions/mean_length": 461.0, "completions/mean_terminated_length": 461.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.43049828708171844, "epoch": 0.00241, "frac_reward_zero_std": 0.25, "grad_norm": 0.04698249697685242, "kl": 0.2767790015786886, "learning_rate": 4.671705058756395e-05, "loss": -0.0023, "num_tokens": 5053615.0, "reward": 0.772064208984375, "reward_std": 0.43700194358825684, "rewards/rollout_reward_func/mean": 0.772064208984375, "rewards/rollout_reward_func/std": 0.5060839056968689, "sampling/importance_sampling_ratio/max": 1.055083990097046, "sampling/importance_sampling_ratio/mean": 0.9893848896026611, "sampling/importance_sampling_ratio/min": 0.8228607773780823, "sampling/sampling_logp_difference/max": 0.11169910430908203, "sampling/sampling_logp_difference/mean": 0.010001150891184807, "step": 241, "step_time": 10.43392852099987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4345607105642557, "epoch": 0.00242, "grad_norm": 0.046413108706474304, "kl": 0.27664210088551044, "learning_rate": 4.669062256729242e-05, "loss": -0.0026, "step": 242, "step_time": 5.13591005699891 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1686.0, "completions/max_terminated_length": 1686.0, "completions/mean_length": 409.3125, "completions/mean_terminated_length": 409.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5074815489351749, "epoch": 0.00243, "frac_reward_zero_std": 0.25, "grad_norm": 0.09523482620716095, "kl": 0.8063109926879406, "learning_rate": 4.6664051433423675e-05, "loss": 0.0014, "num_tokens": 5095103.0, "reward": 0.739631175994873, "reward_std": 0.473169207572937, "rewards/rollout_reward_func/mean": 0.739631175994873, "rewards/rollout_reward_func/std": 0.5827919244766235, "sampling/importance_sampling_ratio/max": 1.0641525983810425, "sampling/importance_sampling_ratio/mean": 1.003458023071289, "sampling/importance_sampling_ratio/min": 0.9180117845535278, "sampling/sampling_logp_difference/max": 0.08617877960205078, "sampling/sampling_logp_difference/mean": 0.0065727559849619865, "step": 243, "step_time": 9.319670496999606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5136723592877388, "epoch": 0.00244, "grad_norm": 0.09509000182151794, "kl": 0.8860200643539429, "learning_rate": 4.663733739133513e-05, "loss": 0.0007, "step": 244, "step_time": 5.194913168998937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1731.0, "completions/max_terminated_length": 1731.0, "completions/mean_length": 474.15625, "completions/mean_terminated_length": 474.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4562520273029804, "epoch": 0.00245, "frac_reward_zero_std": 0.5, "grad_norm": 0.14502817392349243, "kl": 0.4016511309891939, "learning_rate": 4.661048064750878e-05, "loss": -0.0035, "num_tokens": 5138781.0, "reward": 0.8299047946929932, "reward_std": 0.26671382784843445, "rewards/rollout_reward_func/mean": 0.8299047946929932, "rewards/rollout_reward_func/std": 0.4017057716846466, "sampling/importance_sampling_ratio/max": 1.0743623971939087, "sampling/importance_sampling_ratio/mean": 0.991772472858429, "sampling/importance_sampling_ratio/min": 0.8253337740898132, "sampling/sampling_logp_difference/max": 0.2611570358276367, "sampling/sampling_logp_difference/mean": 0.01197755429893732, "step": 245, "step_time": 10.236091187000056 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.46383960358798504, "epoch": 0.00246, "grad_norm": 0.03660755977034569, "kl": 0.3925664871931076, "learning_rate": 4.658348140952962e-05, "loss": -0.0043, "step": 246, "step_time": 5.439909248000731 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 439.09375, "completions/mean_terminated_length": 439.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5718227103352547, "epoch": 0.00247, "frac_reward_zero_std": 0.0, "grad_norm": 0.27879267930984497, "kl": 0.40529870986938477, "learning_rate": 4.6556339886084e-05, "loss": 0.0027, "num_tokens": 5181333.0, "reward": 0.468522846698761, "reward_std": 0.8341646194458008, "rewards/rollout_reward_func/mean": 0.468522846698761, "rewards/rollout_reward_func/std": 0.8321065902709961, "sampling/importance_sampling_ratio/max": 1.1074260473251343, "sampling/importance_sampling_ratio/mean": 0.9983157515525818, "sampling/importance_sampling_ratio/min": 0.8854577541351318, "sampling/sampling_logp_difference/max": 0.21505069732666016, "sampling/sampling_logp_difference/mean": 0.014317598193883896, "step": 247, "step_time": 9.03258765999999 }, { "clip_ratio/high_max": 0.01923076994717121, "clip_ratio/high_mean": 0.009615384973585606, "clip_ratio/low_mean": 0.028245192021131516, "clip_ratio/low_min": 0.009615384973585606, "clip_ratio/region_mean": 0.03786057699471712, "entropy": 0.5589376799762249, "epoch": 0.00248, "grad_norm": 0.15786102414131165, "kl": 0.395873686298728, "learning_rate": 4.652905628695807e-05, "loss": 0.0009, "step": 248, "step_time": 6.008915283000533 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 370.75, "completions/mean_terminated_length": 370.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.46409141831099987, "epoch": 0.00249, "frac_reward_zero_std": 0.0, "grad_norm": 2.6302359104156494, "kl": 7.232265366241336, "learning_rate": 4.650163082303613e-05, "loss": 0.0186, "num_tokens": 5221540.0, "reward": 0.6772809028625488, "reward_std": 0.5764788389205933, "rewards/rollout_reward_func/mean": 0.6772809028625488, "rewards/rollout_reward_func/std": 0.6543562412261963, "sampling/importance_sampling_ratio/max": 1.117995262145996, "sampling/importance_sampling_ratio/mean": 1.0115413665771484, "sampling/importance_sampling_ratio/min": 0.780290424823761, "sampling/sampling_logp_difference/max": 0.2491774559020996, "sampling/sampling_logp_difference/mean": 0.012481102719902992, "step": 249, "step_time": 8.890195160000076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.4621431604027748, "epoch": 0.0025, "grad_norm": 0.8449077606201172, "kl": 0.8346614465117455, "learning_rate": 4.6474063706298994e-05, "loss": 0.0071, "step": 250, "step_time": 4.823332104999736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1659.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 497.5625, "completions/mean_terminated_length": 497.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5618453994393349, "epoch": 0.00251, "frac_reward_zero_std": 0.0, "grad_norm": 0.09182306379079819, "kl": 0.5596221853047609, "learning_rate": 4.644635514982236e-05, "loss": 0.0058, "num_tokens": 5266312.0, "reward": 0.5833603143692017, "reward_std": 0.6316594481468201, "rewards/rollout_reward_func/mean": 0.5833603143692017, "rewards/rollout_reward_func/std": 0.6691997647285461, "sampling/importance_sampling_ratio/max": 1.1064316034317017, "sampling/importance_sampling_ratio/mean": 0.9987003803253174, "sampling/importance_sampling_ratio/min": 0.8867242336273193, "sampling/sampling_logp_difference/max": 0.108795166015625, "sampling/sampling_logp_difference/mean": 0.012930603697896004, "step": 251, "step_time": 9.539515418000065 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.5504823438823223, "epoch": 0.00252, "grad_norm": 0.08048254996538162, "kl": 0.5792170539498329, "learning_rate": 4.641850536777517e-05, "loss": 0.0054, "step": 252, "step_time": 5.319774664000761 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1578.0, "completions/max_terminated_length": 1578.0, "completions/mean_length": 419.125, "completions/mean_terminated_length": 419.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6466461271047592, "epoch": 0.00253, "frac_reward_zero_std": 0.0, "grad_norm": 0.20063315331935883, "kl": 0.31554569862782955, "learning_rate": 4.6390514575417935e-05, "loss": -0.0248, "num_tokens": 5308554.0, "reward": 0.6058715581893921, "reward_std": 0.6444660425186157, "rewards/rollout_reward_func/mean": 0.6058715581893921, "rewards/rollout_reward_func/std": 0.628871738910675, "sampling/importance_sampling_ratio/max": 1.4532262086868286, "sampling/importance_sampling_ratio/mean": 1.0220646858215332, "sampling/importance_sampling_ratio/min": 0.8921107053756714, "sampling/sampling_logp_difference/max": 0.17629361152648926, "sampling/sampling_logp_difference/mean": 0.01834419183433056, "step": 253, "step_time": 9.125277038999684 }, { "clip_ratio/high_max": 0.02812499925494194, "clip_ratio/high_mean": 0.01406249962747097, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01406249962747097, "entropy": 0.6409415975213051, "epoch": 0.00254, "grad_norm": 0.17043881118297577, "kl": 0.3140127342194319, "learning_rate": 4.6362382989101094e-05, "loss": -0.0281, "step": 254, "step_time": 5.044436589998895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 559.4375, "completions/mean_terminated_length": 559.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5288480501621962, "epoch": 0.00255, "frac_reward_zero_std": 0.0, "grad_norm": 0.20160996913909912, "kl": 0.40095819160342216, "learning_rate": 4.633411082626332e-05, "loss": 0.0037, "num_tokens": 5355882.0, "reward": 0.5641722679138184, "reward_std": 0.7542788982391357, "rewards/rollout_reward_func/mean": 0.5641722679138184, "rewards/rollout_reward_func/std": 0.743893027305603, "sampling/importance_sampling_ratio/max": 1.064274549484253, "sampling/importance_sampling_ratio/mean": 0.9901213645935059, "sampling/importance_sampling_ratio/min": 0.8447436690330505, "sampling/sampling_logp_difference/max": 0.11402058601379395, "sampling/sampling_logp_difference/mean": 0.008722800761461258, "step": 255, "step_time": 10.588104030999602 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.026041666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.033854166977107525, "entropy": 0.513609129935503, "epoch": 0.00256, "grad_norm": 0.1050688624382019, "kl": 0.39711805060505867, "learning_rate": 4.630569830542987e-05, "loss": 0.0015, "step": 256, "step_time": 5.276573082000141 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2182.0, "completions/max_terminated_length": 2182.0, "completions/mean_length": 502.5625, "completions/mean_terminated_length": 502.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4449512157589197, "epoch": 0.00257, "frac_reward_zero_std": 0.25, "grad_norm": 0.14327721297740936, "kl": 0.4927577506750822, "learning_rate": 4.627714564621085e-05, "loss": 0.0001, "num_tokens": 5400434.0, "reward": 0.8373973369598389, "reward_std": 0.3940616846084595, "rewards/rollout_reward_func/mean": 0.8373973369598389, "rewards/rollout_reward_func/std": 0.45604878664016724, "sampling/importance_sampling_ratio/max": 1.1974989175796509, "sampling/importance_sampling_ratio/mean": 1.012618064880371, "sampling/importance_sampling_ratio/min": 0.897311270236969, "sampling/sampling_logp_difference/max": 0.18032753467559814, "sampling/sampling_logp_difference/mean": 0.011064953170716763, "step": 257, "step_time": 10.876640112999667 }, { "clip_ratio/high_max": 0.026041666977107525, "clip_ratio/high_mean": 0.013020833488553762, "clip_ratio/low_mean": 0.018229166977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125000139698386, "entropy": 0.43514760583639145, "epoch": 0.00258, "grad_norm": 0.07431601732969284, "kl": 0.514441255480051, "learning_rate": 4.6248453069299554e-05, "loss": -0.0015, "step": 258, "step_time": 6.103161416000148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1560.0, "completions/max_terminated_length": 1560.0, "completions/mean_length": 451.03125, "completions/mean_terminated_length": 451.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.45003731921315193, "epoch": 0.00259, "frac_reward_zero_std": 0.0, "grad_norm": 0.1358211785554886, "kl": 0.3478456288576126, "learning_rate": 4.6219620796470754e-05, "loss": 0.0027, "num_tokens": 5442477.0, "reward": 0.6215956211090088, "reward_std": 0.6796096563339233, "rewards/rollout_reward_func/mean": 0.6215956211090088, "rewards/rollout_reward_func/std": 0.7024564743041992, "sampling/importance_sampling_ratio/max": 1.1608662605285645, "sampling/importance_sampling_ratio/mean": 0.9883631467819214, "sampling/importance_sampling_ratio/min": 0.8024227619171143, "sampling/sampling_logp_difference/max": 0.18895268440246582, "sampling/sampling_logp_difference/mean": 0.012660171836614609, "step": 259, "step_time": 9.101745852000931 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.04427083395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.05052083404734731, "entropy": 0.4342627227306366, "epoch": 0.0026, "grad_norm": 0.0962696298956871, "kl": 0.3492920957505703, "learning_rate": 4.619064905057898e-05, "loss": 0.0016, "step": 260, "step_time": 5.069653709001159 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 598.65625, "completions/mean_terminated_length": 598.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.45593131333589554, "epoch": 0.00261, "frac_reward_zero_std": 0.25, "grad_norm": 0.11956219375133514, "kl": 0.48760965652763844, "learning_rate": 4.616153805555679e-05, "loss": -0.0006, "num_tokens": 5491708.0, "reward": 0.6900585889816284, "reward_std": 0.5271799564361572, "rewards/rollout_reward_func/mean": 0.6900585889816284, "rewards/rollout_reward_func/std": 0.6350724697113037, "sampling/importance_sampling_ratio/max": 1.1057791709899902, "sampling/importance_sampling_ratio/mean": 1.005411148071289, "sampling/importance_sampling_ratio/min": 0.9136157631874084, "sampling/sampling_logp_difference/max": 0.09968811273574829, "sampling/sampling_logp_difference/mean": 0.008957109414041042, "step": 261, "step_time": 10.128088620998824 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.02864583395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03489583404734731, "entropy": 0.4515322409570217, "epoch": 0.00262, "grad_norm": 0.1334623098373413, "kl": 0.6038501784205437, "learning_rate": 4.613228803641304e-05, "loss": -0.0007, "step": 262, "step_time": 5.787150440000005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 458.375, "completions/mean_terminated_length": 458.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5134959332644939, "epoch": 0.00263, "frac_reward_zero_std": 0.0, "grad_norm": 0.1679469347000122, "kl": 0.35127237625420094, "learning_rate": 4.610289921923117e-05, "loss": -0.0032, "num_tokens": 5535009.0, "reward": 0.5606997013092041, "reward_std": 0.7380523681640625, "rewards/rollout_reward_func/mean": 0.5606997013092041, "rewards/rollout_reward_func/std": 0.7516773343086243, "sampling/importance_sampling_ratio/max": 1.1859005689620972, "sampling/importance_sampling_ratio/mean": 0.9964484572410583, "sampling/importance_sampling_ratio/min": 0.8973520994186401, "sampling/sampling_logp_difference/max": 0.11385703086853027, "sampling/sampling_logp_difference/mean": 0.012835300527513027, "step": 263, "step_time": 8.925168330000815 }, { "clip_ratio/high_max": 0.033333334140479565, "clip_ratio/high_mean": 0.016666667070239782, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04010416707023978, "entropy": 0.5239488631486893, "epoch": 0.00264, "grad_norm": 0.12841551005840302, "kl": 0.35788978077471256, "learning_rate": 4.607337183116742e-05, "loss": -0.005, "step": 264, "step_time": 4.864382579999528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1373.0, "completions/max_terminated_length": 1373.0, "completions/mean_length": 409.09375, "completions/mean_terminated_length": 409.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5477988794445992, "epoch": 0.00265, "frac_reward_zero_std": 0.25, "grad_norm": 0.5233628153800964, "kl": 0.4426777008920908, "learning_rate": 4.604370610044911e-05, "loss": 0.0769, "num_tokens": 5576434.0, "reward": 0.8661518096923828, "reward_std": 0.3136124610900879, "rewards/rollout_reward_func/mean": 0.8661518096923828, "rewards/rollout_reward_func/std": 0.35991787910461426, "sampling/importance_sampling_ratio/max": 1.129164218902588, "sampling/importance_sampling_ratio/mean": 0.998163104057312, "sampling/importance_sampling_ratio/min": 0.8338829874992371, "sampling/sampling_logp_difference/max": 0.14407873153686523, "sampling/sampling_logp_difference/mean": 0.016218166798353195, "step": 265, "step_time": 8.916938153999581 }, { "clip_ratio/high_max": 0.027931950287893414, "clip_ratio/high_mean": 0.013965975143946707, "clip_ratio/low_mean": 0.0219594594091177, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03592543536797166, "entropy": 0.5585471782833338, "epoch": 0.00266, "grad_norm": 0.23581017553806305, "kl": 0.43899644538760185, "learning_rate": 4.601390225637285e-05, "loss": 0.0666, "step": 266, "step_time": 4.7531341030003205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 345.0625, "completions/mean_terminated_length": 345.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4320762939751148, "epoch": 0.00267, "frac_reward_zero_std": 0.0, "grad_norm": 0.15289507806301117, "kl": 0.4063198138028383, "learning_rate": 4.598396052930276e-05, "loss": -0.0006, "num_tokens": 5616178.0, "reward": 0.5933696031570435, "reward_std": 0.7619976997375488, "rewards/rollout_reward_func/mean": 0.5933696031570435, "rewards/rollout_reward_func/std": 0.7467468976974487, "sampling/importance_sampling_ratio/max": 1.0662494897842407, "sampling/importance_sampling_ratio/mean": 0.9911150932312012, "sampling/importance_sampling_ratio/min": 0.8243468999862671, "sampling/sampling_logp_difference/max": 0.19394445419311523, "sampling/sampling_logp_difference/mean": 0.010379282757639885, "step": 267, "step_time": 8.54757357500057 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.44938434660434723, "epoch": 0.00268, "grad_norm": 0.13867853581905365, "kl": 0.40606894344091415, "learning_rate": 4.595388115066871e-05, "loss": -0.002, "step": 268, "step_time": 5.146224900000107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1569.0, "completions/max_terminated_length": 1569.0, "completions/mean_length": 610.96875, "completions/mean_terminated_length": 610.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5979040935635567, "epoch": 0.00269, "frac_reward_zero_std": 0.25, "grad_norm": 0.23586289584636688, "kl": 0.5660366714000702, "learning_rate": 4.5923664352964555e-05, "loss": -0.0031, "num_tokens": 5664708.0, "reward": 0.7539829015731812, "reward_std": 0.5749520659446716, "rewards/rollout_reward_func/mean": 0.7539829015731812, "rewards/rollout_reward_func/std": 0.6614054441452026, "sampling/importance_sampling_ratio/max": 1.1295238733291626, "sampling/importance_sampling_ratio/mean": 0.9933105111122131, "sampling/importance_sampling_ratio/min": 0.7360506057739258, "sampling/sampling_logp_difference/max": 0.24549007415771484, "sampling/sampling_logp_difference/mean": 0.01858064904808998, "step": 269, "step_time": 10.456081997999718 }, { "clip_ratio/high_max": 0.01854395680129528, "clip_ratio/high_mean": 0.00927197840064764, "clip_ratio/low_mean": 0.026562500279396772, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03583447868004441, "entropy": 0.6078807711601257, "epoch": 0.0027, "grad_norm": 0.1269243359565735, "kl": 0.5599331632256508, "learning_rate": 4.5893310369746284e-05, "loss": -0.0056, "step": 270, "step_time": 5.229564466999818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1373.0, "completions/max_terminated_length": 1373.0, "completions/mean_length": 412.71875, "completions/mean_terminated_length": 404.2903137207031, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.45268896594643593, "epoch": 0.00271, "frac_reward_zero_std": 0.5, "grad_norm": 0.06035642325878143, "kl": 0.40988776087760925, "learning_rate": 4.586281943563023e-05, "loss": -0.0045, "num_tokens": 5705895.0, "reward": 0.8714056015014648, "reward_std": 0.2808187007904053, "rewards/rollout_reward_func/mean": 0.8714056015014648, "rewards/rollout_reward_func/std": 0.42505624890327454, "sampling/importance_sampling_ratio/max": 1.3193758726119995, "sampling/importance_sampling_ratio/mean": 1.0121073722839355, "sampling/importance_sampling_ratio/min": 0.8788411617279053, "sampling/sampling_logp_difference/max": 0.15764331817626953, "sampling/sampling_logp_difference/mean": 0.012815006077289581, "step": 271, "step_time": 8.691749501000231 }, { "clip_ratio/high_max": 0.040277778171002865, "clip_ratio/high_mean": 0.020138889085501432, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020138889085501432, "entropy": 0.4757050797343254, "epoch": 0.00272, "grad_norm": 0.03880469501018524, "kl": 0.41231659054756165, "learning_rate": 4.583219178629129e-05, "loss": -0.0047, "step": 272, "step_time": 4.720651124000142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 404.3125, "completions/mean_terminated_length": 404.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6455893199890852, "epoch": 0.00273, "frac_reward_zero_std": 0.0, "grad_norm": 0.41780519485473633, "kl": 0.4682430028915405, "learning_rate": 4.580142765846108e-05, "loss": 0.0348, "num_tokens": 5746677.0, "reward": 0.7454465627670288, "reward_std": 0.555956244468689, "rewards/rollout_reward_func/mean": 0.7454465627670288, "rewards/rollout_reward_func/std": 0.5638887286186218, "sampling/importance_sampling_ratio/max": 1.3882437944412231, "sampling/importance_sampling_ratio/mean": 1.0360255241394043, "sampling/importance_sampling_ratio/min": 0.9700131416320801, "sampling/sampling_logp_difference/max": 0.30130815505981445, "sampling/sampling_logp_difference/mean": 0.017024826258420944, "step": 273, "step_time": 9.057460500999696 }, { "clip_ratio/high_max": 0.06458333507180214, "clip_ratio/high_mean": 0.03229166753590107, "clip_ratio/low_mean": 0.026041666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.058333336375653744, "entropy": 0.6740639340132475, "epoch": 0.00274, "grad_norm": 0.19504813849925995, "kl": 0.4780670590698719, "learning_rate": 4.577052728992608e-05, "loss": 0.0283, "step": 274, "step_time": 5.035858394000115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 1654.0, "completions/max_terminated_length": 1654.0, "completions/mean_length": 431.8125, "completions/mean_terminated_length": 391.1071472167969, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0002595707774162, "epoch": 0.00275, "frac_reward_zero_std": 0.25, "grad_norm": 0.5027241110801697, "kl": 0.43502186611294746, "learning_rate": 4.5739490919525846e-05, "loss": 0.0557, "num_tokens": 5788592.0, "reward": 0.6837561726570129, "reward_std": 0.5060193538665771, "rewards/rollout_reward_func/mean": 0.6837561726570129, "rewards/rollout_reward_func/std": 0.6370497941970825, "sampling/importance_sampling_ratio/max": 1.2333165407180786, "sampling/importance_sampling_ratio/mean": 0.9662628173828125, "sampling/importance_sampling_ratio/min": 1.040597469303961e-11, "sampling/sampling_logp_difference/max": 14.895196914672852, "sampling/sampling_logp_difference/mean": 0.0837336853146553, "step": 275, "step_time": 11.294832799000687 }, { "clip_ratio/high_max": 0.021875000093132257, "clip_ratio/high_mean": 0.013252315111458302, "clip_ratio/low_mean": 0.022478070808574557, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03573038592003286, "entropy": 1.0042749214917421, "epoch": 0.00276, "grad_norm": 0.3103949725627899, "kl": 0.4326898157596588, "learning_rate": 4.570831878715111e-05, "loss": 0.0461, "step": 276, "step_time": 5.249264700999902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1611.0, "completions/max_terminated_length": 1611.0, "completions/mean_length": 520.15625, "completions/mean_terminated_length": 520.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5876888073980808, "epoch": 0.00277, "frac_reward_zero_std": 0.25, "grad_norm": 0.15495175123214722, "kl": 0.4981069630011916, "learning_rate": 4.5677011133742e-05, "loss": 0.0151, "num_tokens": 5833820.0, "reward": 0.7538532614707947, "reward_std": 0.49632173776626587, "rewards/rollout_reward_func/mean": 0.7538532614707947, "rewards/rollout_reward_func/std": 0.6072624325752258, "sampling/importance_sampling_ratio/max": 1.1024460792541504, "sampling/importance_sampling_ratio/mean": 0.9850590825080872, "sampling/importance_sampling_ratio/min": 0.7723720073699951, "sampling/sampling_logp_difference/max": 0.12529325485229492, "sampling/sampling_logp_difference/mean": 0.014574235305190086, "step": 277, "step_time": 9.545201554000414 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.008928571827709675, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015178571920841932, "entropy": 0.5631454326212406, "epoch": 0.00278, "grad_norm": 0.10039346665143967, "kl": 0.5150489835068583, "learning_rate": 4.56455682012861e-05, "loss": 0.0143, "step": 278, "step_time": 5.257844178999676 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 380.15625, "completions/mean_terminated_length": 380.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5018457788974047, "epoch": 0.00279, "frac_reward_zero_std": 0.25, "grad_norm": 0.15161053836345673, "kl": 0.2959432601928711, "learning_rate": 4.56139902328166e-05, "loss": -0.0015, "num_tokens": 5874030.0, "reward": 0.7021114826202393, "reward_std": 0.39904871582984924, "rewards/rollout_reward_func/mean": 0.7021114826202393, "rewards/rollout_reward_func/std": 0.5431117415428162, "sampling/importance_sampling_ratio/max": 1.0722570419311523, "sampling/importance_sampling_ratio/mean": 0.9889722466468811, "sampling/importance_sampling_ratio/min": 0.814717173576355, "sampling/sampling_logp_difference/max": 0.2128458023071289, "sampling/sampling_logp_difference/mean": 0.011549701914191246, "step": 279, "step_time": 9.136273194001205 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.01875000074505806, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.029166667722165585, "entropy": 0.4799728225916624, "epoch": 0.0028, "grad_norm": 0.050775326788425446, "kl": 0.2916928259655833, "learning_rate": 4.558227747241048e-05, "loss": -0.0025, "step": 280, "step_time": 4.95238757099969 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1578.0, "completions/max_terminated_length": 1578.0, "completions/mean_length": 573.0625, "completions/mean_terminated_length": 573.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5415681861341, "epoch": 0.00281, "frac_reward_zero_std": 0.25, "grad_norm": 0.18968923389911652, "kl": 0.6925543341785669, "learning_rate": 4.555043016518652e-05, "loss": -0.0041, "num_tokens": 5922414.0, "reward": 0.8103024959564209, "reward_std": 0.4529076814651489, "rewards/rollout_reward_func/mean": 0.8103024959564209, "rewards/rollout_reward_func/std": 0.5349337458610535, "sampling/importance_sampling_ratio/max": 1.1195849180221558, "sampling/importance_sampling_ratio/mean": 1.0035316944122314, "sampling/importance_sampling_ratio/min": 0.8484418988227844, "sampling/sampling_logp_difference/max": 0.16444730758666992, "sampling/sampling_logp_difference/mean": 0.0103178471326828, "step": 281, "step_time": 9.944212945999425 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0193452388048172, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0271577388048172, "entropy": 0.5379639118909836, "epoch": 0.00282, "grad_norm": 0.14247554540634155, "kl": 0.6493175663053989, "learning_rate": 4.551844855730349e-05, "loss": -0.0061, "step": 282, "step_time": 6.189799448000031 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 485.125, "completions/mean_terminated_length": 464.3000183105469, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7656975015997887, "epoch": 0.00283, "frac_reward_zero_std": 0.0, "grad_norm": 0.5077521800994873, "kl": 0.5478998571634293, "learning_rate": 4.548633289595819e-05, "loss": 0.1407, "num_tokens": 5966737.0, "reward": 0.6097269058227539, "reward_std": 0.6411639451980591, "rewards/rollout_reward_func/mean": 0.6097269058227539, "rewards/rollout_reward_func/std": 0.6708220839500427, "sampling/importance_sampling_ratio/max": 1.094997525215149, "sampling/importance_sampling_ratio/mean": 0.9910489320755005, "sampling/importance_sampling_ratio/min": 0.6009795665740967, "sampling/sampling_logp_difference/max": 0.20102906227111816, "sampling/sampling_logp_difference/mean": 0.02164580300450325, "step": 283, "step_time": 9.49672468200015 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.016906194388866425, "clip_ratio/low_min": 0.009191176854074001, "clip_ratio/region_mean": 0.024718694388866425, "entropy": 0.768932182341814, "epoch": 0.00284, "grad_norm": 0.36947599053382874, "kl": 0.5413878206163645, "learning_rate": 4.545408342938361e-05, "loss": 0.1248, "step": 284, "step_time": 5.028021588000229 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2508.0, "completions/max_terminated_length": 2508.0, "completions/mean_length": 526.96875, "completions/mean_terminated_length": 517.0645141601562, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7332190200686455, "epoch": 0.00285, "frac_reward_zero_std": 0.25, "grad_norm": 0.3584897220134735, "kl": 0.5001751109957695, "learning_rate": 4.542170040684691e-05, "loss": 0.0452, "num_tokens": 6011632.0, "reward": 0.6926295161247253, "reward_std": 0.5928170680999756, "rewards/rollout_reward_func/mean": 0.6926295161247253, "rewards/rollout_reward_func/std": 0.680054247379303, "sampling/importance_sampling_ratio/max": 1.1836739778518677, "sampling/importance_sampling_ratio/mean": 0.9954276084899902, "sampling/importance_sampling_ratio/min": 0.6839469075202942, "sampling/sampling_logp_difference/max": 0.27391719818115234, "sampling/sampling_logp_difference/mean": 0.018306491896510124, "step": 285, "step_time": 12.204277717999958 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.049122808035463095, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0647478080354631, "entropy": 0.7545193135738373, "epoch": 0.00286, "grad_norm": 0.21448323130607605, "kl": 0.4999484531581402, "learning_rate": 4.53891840786476e-05, "loss": 0.0383, "step": 286, "step_time": 6.723258041000918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1564.0, "completions/max_terminated_length": 1564.0, "completions/mean_length": 385.96875, "completions/mean_terminated_length": 374.1290283203125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7191807776689529, "epoch": 0.00287, "frac_reward_zero_std": 0.0, "grad_norm": 0.8347065448760986, "kl": 0.5202568285167217, "learning_rate": 4.535653469611551e-05, "loss": 0.0901, "num_tokens": 6052843.0, "reward": 0.6134296655654907, "reward_std": 0.6349297761917114, "rewards/rollout_reward_func/mean": 0.6134296655654907, "rewards/rollout_reward_func/std": 0.6674875020980835, "sampling/importance_sampling_ratio/max": 1.2705374956130981, "sampling/importance_sampling_ratio/mean": 1.0263875722885132, "sampling/importance_sampling_ratio/min": 0.9541559815406799, "sampling/sampling_logp_difference/max": 0.20713496208190918, "sampling/sampling_logp_difference/mean": 0.01966901868581772, "step": 287, "step_time": 9.765636305999578 }, { "clip_ratio/high_max": 0.026041666977107525, "clip_ratio/high_mean": 0.016310307197272778, "clip_ratio/low_mean": 0.03856907971203327, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.05487938690930605, "entropy": 0.7059215120971203, "epoch": 0.00288, "grad_norm": 0.32748985290527344, "kl": 0.5152970254421234, "learning_rate": 4.5323752511608934e-05, "loss": 0.0784, "step": 288, "step_time": 5.629080540999894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1587.0, "completions/max_terminated_length": 1587.0, "completions/mean_length": 522.78125, "completions/mean_terminated_length": 518.774169921875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8213042393326759, "epoch": 0.00289, "frac_reward_zero_std": 0.25, "grad_norm": 0.2752106189727783, "kl": 0.5243588965386152, "learning_rate": 4.5290837778512615e-05, "loss": -0.0304, "num_tokens": 6099360.0, "reward": 0.7450450658798218, "reward_std": 0.5142843723297119, "rewards/rollout_reward_func/mean": 0.7450450658798218, "rewards/rollout_reward_func/std": 0.6250448822975159, "sampling/importance_sampling_ratio/max": 1.3312735557556152, "sampling/importance_sampling_ratio/mean": 1.036529541015625, "sampling/importance_sampling_ratio/min": 0.8398435115814209, "sampling/sampling_logp_difference/max": 0.282257080078125, "sampling/sampling_logp_difference/mean": 0.02253495156764984, "step": 289, "step_time": 10.264007212000251 }, { "clip_ratio/high_max": 0.017500000074505806, "clip_ratio/high_mean": 0.008750000037252903, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.016562500037252903, "entropy": 0.833214171230793, "epoch": 0.0029, "grad_norm": 0.19707588851451874, "kl": 0.6021217219531536, "learning_rate": 4.525779075123581e-05, "loss": -0.032, "step": 290, "step_time": 5.176173851000385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1519.0, "completions/max_terminated_length": 1519.0, "completions/mean_length": 363.6875, "completions/mean_terminated_length": 363.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8487923964858055, "epoch": 0.00291, "frac_reward_zero_std": 0.25, "grad_norm": 0.518794059753418, "kl": 2.2360727954655886, "learning_rate": 4.522461168521033e-05, "loss": 0.0108, "num_tokens": 6139274.0, "reward": 0.5389575958251953, "reward_std": 0.6705641150474548, "rewards/rollout_reward_func/mean": 0.5389575958251953, "rewards/rollout_reward_func/std": 0.8253065347671509, "sampling/importance_sampling_ratio/max": 1.108742594718933, "sampling/importance_sampling_ratio/mean": 0.9906811714172363, "sampling/importance_sampling_ratio/min": 0.8542822003364563, "sampling/sampling_logp_difference/max": 0.19647598266601562, "sampling/sampling_logp_difference/mean": 0.020885083824396133, "step": 291, "step_time": 9.121281900998838 }, { "clip_ratio/high_max": 0.0193452388048172, "clip_ratio/high_mean": 0.0096726194024086, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0096726194024086, "entropy": 0.9041766077280045, "epoch": 0.00292, "grad_norm": 0.25620409846305847, "kl": 1.0078645180910826, "learning_rate": 4.519130083688856e-05, "loss": 0.0057, "step": 292, "step_time": 4.979576817000179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1657.0, "completions/max_terminated_length": 1657.0, "completions/mean_length": 522.96875, "completions/mean_terminated_length": 522.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6728114783763885, "epoch": 0.00293, "frac_reward_zero_std": 0.25, "grad_norm": 0.1694919764995575, "kl": 0.5785791669040918, "learning_rate": 4.5157858463741454e-05, "loss": -0.0032, "num_tokens": 6185034.0, "reward": 0.7029841542243958, "reward_std": 0.42289137840270996, "rewards/rollout_reward_func/mean": 0.7029841542243958, "rewards/rollout_reward_func/std": 0.5439256429672241, "sampling/importance_sampling_ratio/max": 1.1570500135421753, "sampling/importance_sampling_ratio/mean": 1.0027540922164917, "sampling/importance_sampling_ratio/min": 0.6920920014381409, "sampling/sampling_logp_difference/max": 0.12988805770874023, "sampling/sampling_logp_difference/mean": 0.01829216256737709, "step": 293, "step_time": 9.70539023900028 }, { "clip_ratio/high_max": 0.014583333861082792, "clip_ratio/high_mean": 0.007291666930541396, "clip_ratio/low_mean": 0.007352941203862429, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014644608134403825, "entropy": 0.6954755783081055, "epoch": 0.00294, "grad_norm": 0.13905836641788483, "kl": 0.5799965485930443, "learning_rate": 4.512428482425661e-05, "loss": -0.0041, "step": 294, "step_time": 5.246595915001308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1525.0, "completions/max_terminated_length": 1525.0, "completions/mean_length": 289.28125, "completions/mean_terminated_length": 289.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7056399211287498, "epoch": 0.00295, "frac_reward_zero_std": 0.0, "grad_norm": 0.12171868979930878, "kl": 0.6939934138208628, "learning_rate": 4.509058017793618e-05, "loss": -0.0068, "num_tokens": 6222071.0, "reward": 0.588118314743042, "reward_std": 0.743166446685791, "rewards/rollout_reward_func/mean": 0.588118314743042, "rewards/rollout_reward_func/std": 0.7569315433502197, "sampling/importance_sampling_ratio/max": 1.1981372833251953, "sampling/importance_sampling_ratio/mean": 0.9727566242218018, "sampling/importance_sampling_ratio/min": 1.1709278295232717e-12, "sampling/sampling_logp_difference/max": 21.462915420532227, "sampling/sampling_logp_difference/mean": 0.17768025398254395, "step": 295, "step_time": 9.514571281999906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.70181455463171, "epoch": 0.00296, "grad_norm": 0.09441104531288147, "kl": 0.7725566420704126, "learning_rate": 4.505674478529495e-05, "loss": -0.0076, "step": 296, "step_time": 5.375524901000972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1474.0, "completions/max_terminated_length": 1435.0, "completions/mean_length": 367.09375, "completions/mean_terminated_length": 340.70001220703125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9992959871888161, "epoch": 0.00297, "frac_reward_zero_std": 0.0, "grad_norm": 0.4074653685092926, "kl": 0.5481140147894621, "learning_rate": 4.502277890785827e-05, "loss": -0.0625, "num_tokens": 6262872.0, "reward": 0.615750253200531, "reward_std": 0.6933867931365967, "rewards/rollout_reward_func/mean": 0.615750253200531, "rewards/rollout_reward_func/std": 0.7122234106063843, "sampling/importance_sampling_ratio/max": 2.2187490463256836, "sampling/importance_sampling_ratio/mean": 1.0154848098754883, "sampling/importance_sampling_ratio/min": 8.898251840984856e-17, "sampling/sampling_logp_difference/max": 21.487424850463867, "sampling/sampling_logp_difference/mean": 0.1464133858680725, "step": 297, "step_time": 9.734486204999484 }, { "clip_ratio/high_max": 0.030624999664723873, "clip_ratio/high_mean": 0.015312499832361937, "clip_ratio/low_mean": 0.02651515183970332, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04182765167206526, "entropy": 0.9698964171111584, "epoch": 0.00298, "grad_norm": 0.2800791263580322, "kl": 0.5328615121543407, "learning_rate": 4.498868280816006e-05, "loss": -0.0705, "step": 298, "step_time": 4.930944414999431 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 522.59375, "completions/mean_terminated_length": 522.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7434393689036369, "epoch": 0.00299, "frac_reward_zero_std": 0.0, "grad_norm": 0.2291867882013321, "kl": 0.6484497208148241, "learning_rate": 4.4954456749740755e-05, "loss": 0.0043, "num_tokens": 6308788.0, "reward": 0.745918869972229, "reward_std": 0.5966472029685974, "rewards/rollout_reward_func/mean": 0.745918869972229, "rewards/rollout_reward_func/std": 0.6273282170295715, "sampling/importance_sampling_ratio/max": 1.1688741445541382, "sampling/importance_sampling_ratio/mean": 0.9977250099182129, "sampling/importance_sampling_ratio/min": 0.7492424845695496, "sampling/sampling_logp_difference/max": 0.34111595153808594, "sampling/sampling_logp_difference/mean": 0.014035840518772602, "step": 299, "step_time": 9.668118367000261 }, { "clip_ratio/high_max": 0.03485576994717121, "clip_ratio/high_mean": 0.017427884973585606, "clip_ratio/low_mean": 0.01875000074505806, "clip_ratio/low_min": 0.012500000186264515, "clip_ratio/region_mean": 0.036177885718643665, "entropy": 0.7314649298787117, "epoch": 0.003, "grad_norm": 0.1335267722606659, "kl": 0.6273622065782547, "learning_rate": 4.492010099714531e-05, "loss": 0.0009, "step": 300, "step_time": 5.207989926999744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 807.0, "completions/max_terminated_length": 805.0, "completions/mean_length": 362.625, "completions/mean_terminated_length": 348.2903137207031, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7907173372805119, "epoch": 0.00301, "frac_reward_zero_std": 0.0, "grad_norm": 0.5027024149894714, "kl": 0.2940632067620754, "learning_rate": 4.488561581592112e-05, "loss": 0.078, "num_tokens": 6348762.0, "reward": 0.6890425682067871, "reward_std": 0.6936211585998535, "rewards/rollout_reward_func/mean": 0.6890425682067871, "rewards/rollout_reward_func/std": 0.6820225715637207, "sampling/importance_sampling_ratio/max": 1.1701868772506714, "sampling/importance_sampling_ratio/mean": 1.0112857818603516, "sampling/importance_sampling_ratio/min": 0.8749542832374573, "sampling/sampling_logp_difference/max": 0.14203548431396484, "sampling/sampling_logp_difference/mean": 0.01606678031384945, "step": 301, "step_time": 7.164834020999933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.05468749953433871, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.05468749953433871, "entropy": 0.7863772287964821, "epoch": 0.00302, "grad_norm": 0.2205897718667984, "kl": 0.2944489661604166, "learning_rate": 4.485100147261597e-05, "loss": 0.0663, "step": 302, "step_time": 4.715097755000443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1628.0, "completions/max_terminated_length": 1628.0, "completions/mean_length": 401.40625, "completions/mean_terminated_length": 401.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5824746638536453, "epoch": 0.00303, "frac_reward_zero_std": 0.25, "grad_norm": 0.45276790857315063, "kl": 0.3323972783982754, "learning_rate": 4.4816258234776e-05, "loss": 0.0385, "num_tokens": 6389400.0, "reward": 0.7533024549484253, "reward_std": 0.5776093602180481, "rewards/rollout_reward_func/mean": 0.7533024549484253, "rewards/rollout_reward_func/std": 0.6632077097892761, "sampling/importance_sampling_ratio/max": 1.1131436824798584, "sampling/importance_sampling_ratio/mean": 1.007781744003296, "sampling/importance_sampling_ratio/min": 0.9000315070152283, "sampling/sampling_logp_difference/max": 0.10544145107269287, "sampling/sampling_logp_difference/mean": 0.011867282912135124, "step": 303, "step_time": 9.563283856000453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.019176136702299118, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.019176136702299118, "entropy": 0.5635005943477154, "epoch": 0.00304, "grad_norm": 0.14182913303375244, "kl": 0.3252980634570122, "learning_rate": 4.478138637094361e-05, "loss": 0.0351, "step": 304, "step_time": 5.244413482999789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 454.59375, "completions/mean_terminated_length": 454.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.541319016367197, "epoch": 0.00305, "frac_reward_zero_std": 0.25, "grad_norm": 0.11333584040403366, "kl": 0.6929347645491362, "learning_rate": 4.4746386150655385e-05, "loss": -0.0066, "num_tokens": 6432895.0, "reward": 0.7471193671226501, "reward_std": 0.5115798115730286, "rewards/rollout_reward_func/mean": 0.7471193671226501, "rewards/rollout_reward_func/std": 0.6227315068244934, "sampling/importance_sampling_ratio/max": 1.2940081357955933, "sampling/importance_sampling_ratio/mean": 1.0168004035949707, "sampling/importance_sampling_ratio/min": 0.8967656493186951, "sampling/sampling_logp_difference/max": 0.12302732467651367, "sampling/sampling_logp_difference/mean": 0.010269518941640854, "step": 305, "step_time": 9.047144084000138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.008928571827709675, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008928571827709675, "entropy": 0.5287333130836487, "epoch": 0.00306, "grad_norm": 0.11751564592123032, "kl": 0.7926004361361265, "learning_rate": 4.471125784444006e-05, "loss": -0.0075, "step": 306, "step_time": 4.829468421000456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2407.0, "completions/max_terminated_length": 2407.0, "completions/mean_length": 635.28125, "completions/mean_terminated_length": 635.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7112803310155869, "epoch": 0.00307, "frac_reward_zero_std": 0.5, "grad_norm": 0.12746217846870422, "kl": 0.37505339086055756, "learning_rate": 4.4676001723816325e-05, "loss": -0.003, "num_tokens": 6482767.0, "reward": 0.6332030892372131, "reward_std": 0.5064598321914673, "rewards/rollout_reward_func/mean": 0.6332030892372131, "rewards/rollout_reward_func/std": 0.77605140209198, "sampling/importance_sampling_ratio/max": 1.094273567199707, "sampling/importance_sampling_ratio/mean": 0.9715784788131714, "sampling/importance_sampling_ratio/min": 0.5425114631652832, "sampling/sampling_logp_difference/max": 0.17838037014007568, "sampling/sampling_logp_difference/mean": 0.016987932845950127, "step": 307, "step_time": 11.9611234099998 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01927083358168602, "entropy": 0.6991673931479454, "epoch": 0.00308, "grad_norm": 0.10570370405912399, "kl": 0.3696310054510832, "learning_rate": 4.464061806129086e-05, "loss": -0.004, "step": 308, "step_time": 7.223985697999979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2342.0, "completions/max_terminated_length": 2342.0, "completions/mean_length": 399.0, "completions/mean_terminated_length": 399.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.41318673081696033, "epoch": 0.00309, "frac_reward_zero_std": 0.25, "grad_norm": 0.1294134259223938, "kl": 0.357820812612772, "learning_rate": 4.4605107130356104e-05, "loss": 0.0152, "num_tokens": 6522420.0, "reward": 0.7854530811309814, "reward_std": 0.48731622099876404, "rewards/rollout_reward_func/mean": 0.7854530811309814, "rewards/rollout_reward_func/std": 0.5964456796646118, "sampling/importance_sampling_ratio/max": 1.0842169523239136, "sampling/importance_sampling_ratio/mean": 0.9966585636138916, "sampling/importance_sampling_ratio/min": 0.9138302206993103, "sampling/sampling_logp_difference/max": 0.09535861015319824, "sampling/sampling_logp_difference/mean": 0.007002371363341808, "step": 309, "step_time": 11.993241744999978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "entropy": 0.40490835532546043, "epoch": 0.0031, "grad_norm": 0.09968464821577072, "kl": 0.36372797936201096, "learning_rate": 4.456946920548824e-05, "loss": 0.0139, "step": 310, "step_time": 6.4753548569992745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 564.5625, "completions/mean_terminated_length": 564.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7845202796161175, "epoch": 0.00311, "frac_reward_zero_std": 0.25, "grad_norm": 0.4539591372013092, "kl": 0.43084052950143814, "learning_rate": 4.4533704562145e-05, "loss": 0.021, "num_tokens": 6570078.0, "reward": 0.6017996072769165, "reward_std": 0.6354162693023682, "rewards/rollout_reward_func/mean": 0.6017996072769165, "rewards/rollout_reward_func/std": 0.7768802642822266, "sampling/importance_sampling_ratio/max": 1.0820945501327515, "sampling/importance_sampling_ratio/mean": 1.0013065338134766, "sampling/importance_sampling_ratio/min": 0.8766872882843018, "sampling/sampling_logp_difference/max": 0.11629819869995117, "sampling/sampling_logp_difference/mean": 0.008732199668884277, "step": 311, "step_time": 10.01848548399903 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.05260416865348816, "clip_ratio/low_min": 0.03125, "clip_ratio/region_mean": 0.05260416865348816, "entropy": 0.7786072269082069, "epoch": 0.00312, "grad_norm": 0.11688404530286789, "kl": 0.39949757792055607, "learning_rate": 4.449781347676359e-05, "loss": 0.0154, "step": 312, "step_time": 5.297986879000291 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2180.0, "completions/max_terminated_length": 2180.0, "completions/mean_length": 440.9375, "completions/mean_terminated_length": 430.58062744140625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7224212363362312, "epoch": 0.00313, "frac_reward_zero_std": 0.0, "grad_norm": 0.17831946909427643, "kl": 0.41622407734394073, "learning_rate": 4.4461796226758497e-05, "loss": 0.01, "num_tokens": 6612446.0, "reward": 0.5201669931411743, "reward_std": 0.7319791316986084, "rewards/rollout_reward_func/mean": 0.5201669931411743, "rewards/rollout_reward_func/std": 0.7245771288871765, "sampling/importance_sampling_ratio/max": 1.1589261293411255, "sampling/importance_sampling_ratio/mean": 1.0066885948181152, "sampling/importance_sampling_ratio/min": 0.8722841143608093, "sampling/sampling_logp_difference/max": 0.17632293701171875, "sampling/sampling_logp_difference/mean": 0.015209799632430077, "step": 313, "step_time": 11.160888751000584 }, { "clip_ratio/high_max": 0.007352941203862429, "clip_ratio/high_mean": 0.0036764706019312143, "clip_ratio/low_mean": 0.013888889225199819, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017565359827131033, "entropy": 0.7095127739012241, "epoch": 0.00314, "grad_norm": 0.12322857975959778, "kl": 0.41342626605182886, "learning_rate": 4.4425653090519425e-05, "loss": 0.0083, "step": 314, "step_time": 6.165506213001208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1641.0, "completions/max_terminated_length": 1641.0, "completions/mean_length": 284.9375, "completions/mean_terminated_length": 284.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.38461907394230366, "epoch": 0.00315, "frac_reward_zero_std": 0.0, "grad_norm": 0.11162667721509933, "kl": 0.37202737107872963, "learning_rate": 4.4389384347409054e-05, "loss": 0.0004, "num_tokens": 6647047.0, "reward": 0.7752677202224731, "reward_std": 0.5512118339538574, "rewards/rollout_reward_func/mean": 0.7752677202224731, "rewards/rollout_reward_func/std": 0.5560808181762695, "sampling/importance_sampling_ratio/max": 1.0711816549301147, "sampling/importance_sampling_ratio/mean": 0.995093822479248, "sampling/importance_sampling_ratio/min": 0.9064978957176208, "sampling/sampling_logp_difference/max": 0.08151423931121826, "sampling/sampling_logp_difference/mean": 0.007781929802149534, "step": 315, "step_time": 10.083844112999486 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "entropy": 0.37531473487615585, "epoch": 0.00316, "grad_norm": 0.06282185018062592, "kl": 0.37205037102103233, "learning_rate": 4.435299027776095e-05, "loss": -0.0003, "step": 316, "step_time": 5.783402190999368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1659.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 434.46875, "completions/mean_terminated_length": 434.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5024554263800383, "epoch": 0.00317, "frac_reward_zero_std": 0.25, "grad_norm": 0.27558282017707825, "kl": 0.41037931852042675, "learning_rate": 4.431647116287734e-05, "loss": 0.0167, "num_tokens": 6688079.0, "reward": 0.778258740901947, "reward_std": 0.4821239411830902, "rewards/rollout_reward_func/mean": 0.778258740901947, "rewards/rollout_reward_func/std": 0.5531739592552185, "sampling/importance_sampling_ratio/max": 1.1183350086212158, "sampling/importance_sampling_ratio/mean": 0.9918532371520996, "sampling/importance_sampling_ratio/min": 0.8024612069129944, "sampling/sampling_logp_difference/max": 0.11259913444519043, "sampling/sampling_logp_difference/mean": 0.011209284886717796, "step": 317, "step_time": 9.649502608000148 }, { "clip_ratio/high_max": 0.03645833395421505, "clip_ratio/high_mean": 0.018229166977107525, "clip_ratio/low_mean": 0.024038461968302727, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04226762894541025, "entropy": 0.5036080507561564, "epoch": 0.00318, "grad_norm": 0.0528140664100647, "kl": 0.41927542351186275, "learning_rate": 4.427982728502702e-05, "loss": 0.0146, "step": 318, "step_time": 5.285732294000354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2263.0, "completions/max_terminated_length": 2263.0, "completions/mean_length": 469.1875, "completions/mean_terminated_length": 469.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5967639274895191, "epoch": 0.00319, "frac_reward_zero_std": 0.25, "grad_norm": 0.17559343576431274, "kl": 0.3516575004905462, "learning_rate": 4.424305892744306e-05, "loss": -0.0074, "num_tokens": 6732031.0, "reward": 0.622019350528717, "reward_std": 0.5884510278701782, "rewards/rollout_reward_func/mean": 0.622019350528717, "rewards/rollout_reward_func/std": 0.7022910714149475, "sampling/importance_sampling_ratio/max": 1.133292317390442, "sampling/importance_sampling_ratio/mean": 1.0116145610809326, "sampling/importance_sampling_ratio/min": 0.901370644569397, "sampling/sampling_logp_difference/max": 0.10460090637207031, "sampling/sampling_logp_difference/mean": 0.01014580950140953, "step": 319, "step_time": 11.408957316999931 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.031250000931322575, "entropy": 0.5847398675978184, "epoch": 0.0032, "grad_norm": 0.10380716621875763, "kl": 0.3473154529929161, "learning_rate": 4.4206166374320716e-05, "loss": -0.009, "step": 320, "step_time": 6.4395947419998265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1543.0, "completions/max_terminated_length": 1543.0, "completions/mean_length": 426.125, "completions/mean_terminated_length": 426.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5297391042113304, "epoch": 0.00321, "frac_reward_zero_std": 0.0, "grad_norm": 0.20516107976436615, "kl": 0.5184890786185861, "learning_rate": 4.416914991081519e-05, "loss": 0.0219, "num_tokens": 6773746.0, "reward": 0.5908182859420776, "reward_std": 0.7165402173995972, "rewards/rollout_reward_func/mean": 0.5908182859420776, "rewards/rollout_reward_func/std": 0.7528811097145081, "sampling/importance_sampling_ratio/max": 1.1634849309921265, "sampling/importance_sampling_ratio/mean": 1.0019457340240479, "sampling/importance_sampling_ratio/min": 0.7366762161254883, "sampling/sampling_logp_difference/max": 0.2402329444885254, "sampling/sampling_logp_difference/mean": 0.00863843783736229, "step": 321, "step_time": 9.317762918999051 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03802083432674408, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03802083432674408, "entropy": 0.504865450784564, "epoch": 0.00322, "grad_norm": 0.1776113659143448, "kl": 0.5896044978871942, "learning_rate": 4.413200982303942e-05, "loss": 0.0203, "step": 322, "step_time": 6.158531075000155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 840.0, "completions/max_terminated_length": 840.0, "completions/mean_length": 316.4375, "completions/mean_terminated_length": 316.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4279155097901821, "epoch": 0.00323, "frac_reward_zero_std": 0.25, "grad_norm": 0.1441330909729004, "kl": 0.22342588007450104, "learning_rate": 4.409474639806189e-05, "loss": -0.007, "num_tokens": 6812275.0, "reward": 0.8004679679870605, "reward_std": 0.4134165942668915, "rewards/rollout_reward_func/mean": 0.8004679679870605, "rewards/rollout_reward_func/std": 0.491279661655426, "sampling/importance_sampling_ratio/max": 1.0685906410217285, "sampling/importance_sampling_ratio/mean": 0.9955387115478516, "sampling/importance_sampling_ratio/min": 0.9046796560287476, "sampling/sampling_logp_difference/max": 0.07766342163085938, "sampling/sampling_logp_difference/mean": 0.008928889408707619, "step": 323, "step_time": 7.226350719000038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.41249705106019974, "epoch": 0.00324, "grad_norm": 0.1436152309179306, "kl": 0.22658172622323036, "learning_rate": 4.405735992390437e-05, "loss": -0.0073, "step": 324, "step_time": 3.777885019999303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 370.03125, "completions/mean_terminated_length": 370.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4533200953155756, "epoch": 0.00325, "frac_reward_zero_std": 0.0, "grad_norm": 0.11588450521230698, "kl": 0.9735795967280865, "learning_rate": 4.401985068953976e-05, "loss": 0.0018, "num_tokens": 6852064.0, "reward": 0.6430283784866333, "reward_std": 0.6642811298370361, "rewards/rollout_reward_func/mean": 0.6430283784866333, "rewards/rollout_reward_func/std": 0.6660192012786865, "sampling/importance_sampling_ratio/max": 1.12325918674469, "sampling/importance_sampling_ratio/mean": 1.0042169094085693, "sampling/importance_sampling_ratio/min": 0.8821572661399841, "sampling/sampling_logp_difference/max": 0.11640691757202148, "sampling/sampling_logp_difference/mean": 0.011201855726540089, "step": 325, "step_time": 9.181848489000458 }, { "clip_ratio/high_max": 0.01315789483487606, "clip_ratio/high_mean": 0.00657894741743803, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00657894741743803, "entropy": 0.4408071041107178, "epoch": 0.00326, "grad_norm": 0.10622195899486542, "kl": 0.9226335063576698, "learning_rate": 4.398221898488979e-05, "loss": 0.0009, "step": 326, "step_time": 4.974154009999438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1623.0, "completions/max_terminated_length": 1623.0, "completions/mean_length": 426.03125, "completions/mean_terminated_length": 426.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4290744625031948, "epoch": 0.00327, "frac_reward_zero_std": 0.25, "grad_norm": 0.06570156663656235, "kl": 0.47995617613196373, "learning_rate": 4.394446510082282e-05, "loss": -0.0061, "num_tokens": 6894019.0, "reward": 0.6477166414260864, "reward_std": 0.5665237903594971, "rewards/rollout_reward_func/mean": 0.6477166414260864, "rewards/rollout_reward_func/std": 0.657835066318512, "sampling/importance_sampling_ratio/max": 1.1148790121078491, "sampling/importance_sampling_ratio/mean": 1.0059852600097656, "sampling/importance_sampling_ratio/min": 0.8026244044303894, "sampling/sampling_logp_difference/max": 0.08556842803955078, "sampling/sampling_logp_difference/mean": 0.009131618775427341, "step": 327, "step_time": 9.665096868999626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4199998416006565, "epoch": 0.00328, "grad_norm": 0.06379567086696625, "kl": 0.48836710304021835, "learning_rate": 4.390658932915157e-05, "loss": -0.0061, "step": 328, "step_time": 5.289178627000183 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 371.90625, "completions/mean_terminated_length": 354.9677429199219, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4568175971508026, "epoch": 0.00329, "frac_reward_zero_std": 0.25, "grad_norm": 0.0985732302069664, "kl": 0.8904061950743198, "learning_rate": 4.3868591962630865e-05, "loss": -0.0035, "num_tokens": 6933962.0, "reward": 0.7818727493286133, "reward_std": 0.5350722074508667, "rewards/rollout_reward_func/mean": 0.7818727493286133, "rewards/rollout_reward_func/std": 0.6036068797111511, "sampling/importance_sampling_ratio/max": 1.1136161088943481, "sampling/importance_sampling_ratio/mean": 1.0004661083221436, "sampling/importance_sampling_ratio/min": 0.9000470638275146, "sampling/sampling_logp_difference/max": 0.10525703430175781, "sampling/sampling_logp_difference/mean": 0.010184608399868011, "step": 329, "step_time": 10.215901761000168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4581134468317032, "epoch": 0.0033, "grad_norm": 0.09349752962589264, "kl": 0.8615279644727707, "learning_rate": 4.383047329495537e-05, "loss": -0.0038, "step": 330, "step_time": 4.872646908000206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2400.0, "completions/max_terminated_length": 2400.0, "completions/mean_length": 480.40625, "completions/mean_terminated_length": 480.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6148389708250761, "epoch": 0.00331, "frac_reward_zero_std": 0.0, "grad_norm": 0.3909803628921509, "kl": 0.4138144366443157, "learning_rate": 4.379223362075735e-05, "loss": 0.0491, "num_tokens": 6977475.0, "reward": 0.7459344863891602, "reward_std": 0.5692851543426514, "rewards/rollout_reward_func/mean": 0.7459344863891602, "rewards/rollout_reward_func/std": 0.5706638097763062, "sampling/importance_sampling_ratio/max": 1.1049734354019165, "sampling/importance_sampling_ratio/mean": 0.9873597025871277, "sampling/importance_sampling_ratio/min": 0.6775913834571838, "sampling/sampling_logp_difference/max": 0.11443018913269043, "sampling/sampling_logp_difference/mean": 0.012057820335030556, "step": 331, "step_time": 11.690917157000968 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.02656250074505806, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03046875074505806, "entropy": 0.630740599706769, "epoch": 0.00332, "grad_norm": 0.18915148079395294, "kl": 0.4111648090183735, "learning_rate": 4.375387323560438e-05, "loss": 0.0417, "step": 332, "step_time": 6.462803823000286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2407.0, "completions/max_terminated_length": 2407.0, "completions/mean_length": 501.8125, "completions/mean_terminated_length": 501.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6284094080328941, "epoch": 0.00333, "frac_reward_zero_std": 0.0, "grad_norm": 0.1475677788257599, "kl": 0.3976634591817856, "learning_rate": 4.3715392435997e-05, "loss": -0.0164, "num_tokens": 7022206.0, "reward": 0.6595606207847595, "reward_std": 0.7611272931098938, "rewards/rollout_reward_func/mean": 0.6595606207847595, "rewards/rollout_reward_func/std": 0.7347927689552307, "sampling/importance_sampling_ratio/max": 1.118947148323059, "sampling/importance_sampling_ratio/mean": 0.9902980923652649, "sampling/importance_sampling_ratio/min": 0.6698177456855774, "sampling/sampling_logp_difference/max": 0.42984867095947266, "sampling/sampling_logp_difference/mean": 0.019818557426333427, "step": 333, "step_time": 11.970598870000686 }, { "clip_ratio/high_max": 0.01896551763638854, "clip_ratio/high_mean": 0.00948275881819427, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00948275881819427, "entropy": 0.6459088809788227, "epoch": 0.00334, "grad_norm": 0.1362995058298111, "kl": 0.38426172360777855, "learning_rate": 4.3676791519366536e-05, "loss": -0.0174, "step": 334, "step_time": 6.587629043001016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1623.0, "completions/max_terminated_length": 1623.0, "completions/mean_length": 568.5625, "completions/mean_terminated_length": 559.7096557617188, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.653051245957613, "epoch": 0.00335, "frac_reward_zero_std": 0.0, "grad_norm": 0.5831186771392822, "kl": 0.4962775930762291, "learning_rate": 4.36380707840727e-05, "loss": 0.0963, "num_tokens": 7070204.0, "reward": 0.6456350088119507, "reward_std": 0.6289982795715332, "rewards/rollout_reward_func/mean": 0.6456350088119507, "rewards/rollout_reward_func/std": 0.6107676029205322, "sampling/importance_sampling_ratio/max": 1.2012298107147217, "sampling/importance_sampling_ratio/mean": 1.0067440271377563, "sampling/importance_sampling_ratio/min": 0.9019725322723389, "sampling/sampling_logp_difference/max": 0.1849684715270996, "sampling/sampling_logp_difference/mean": 0.014390562660992146, "step": 335, "step_time": 10.48189593800089 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.027960526756942272, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.027960526756942272, "entropy": 0.650990430265665, "epoch": 0.00336, "grad_norm": 0.2632696032524109, "kl": 0.5018901694566011, "learning_rate": 4.359923052940134e-05, "loss": 0.0849, "step": 336, "step_time": 5.689964484000029 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2307.0, "completions/max_terminated_length": 2307.0, "completions/mean_length": 516.40625, "completions/mean_terminated_length": 516.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6113283932209015, "epoch": 0.00337, "frac_reward_zero_std": 0.0, "grad_norm": 0.19432878494262695, "kl": 0.41812789253890514, "learning_rate": 4.35602710555621e-05, "loss": -0.0311, "num_tokens": 7116078.0, "reward": 0.6539928913116455, "reward_std": 0.690733790397644, "rewards/rollout_reward_func/mean": 0.6539928913116455, "rewards/rollout_reward_func/std": 0.6946818232536316, "sampling/importance_sampling_ratio/max": 1.2923800945281982, "sampling/importance_sampling_ratio/mean": 1.0088220834732056, "sampling/importance_sampling_ratio/min": 0.9255794286727905, "sampling/sampling_logp_difference/max": 0.12116861343383789, "sampling/sampling_logp_difference/mean": 0.01440480351448059, "step": 337, "step_time": 11.433579786999417 }, { "clip_ratio/high_max": 0.007978723384439945, "clip_ratio/high_mean": 0.003989361692219973, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.024822695646435022, "entropy": 0.609207721427083, "epoch": 0.00338, "grad_norm": 0.16116748750209808, "kl": 0.42114031966775656, "learning_rate": 4.352119266368612e-05, "loss": -0.0338, "step": 338, "step_time": 6.429993428999751 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1686.0, "completions/max_terminated_length": 1686.0, "completions/mean_length": 350.8125, "completions/mean_terminated_length": 350.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4637354239821434, "epoch": 0.00339, "frac_reward_zero_std": 0.0, "grad_norm": 0.1945277899503708, "kl": 0.3928692638874054, "learning_rate": 4.348199565582369e-05, "loss": 0.0142, "num_tokens": 7154723.0, "reward": 0.552872896194458, "reward_std": 0.7266799211502075, "rewards/rollout_reward_func/mean": 0.552872896194458, "rewards/rollout_reward_func/std": 0.718950092792511, "sampling/importance_sampling_ratio/max": 1.3066285848617554, "sampling/importance_sampling_ratio/mean": 0.9986491203308105, "sampling/importance_sampling_ratio/min": 0.9068791270256042, "sampling/sampling_logp_difference/max": 0.1867438554763794, "sampling/sampling_logp_difference/mean": 0.009362439624965191, "step": 339, "step_time": 9.759940305000782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.018229166977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018229166977107525, "entropy": 0.46615155786275864, "epoch": 0.0034, "grad_norm": 0.14118124544620514, "kl": 0.3966478072106838, "learning_rate": 4.344268033494193e-05, "loss": 0.0129, "step": 340, "step_time": 5.388011914999879 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1641.0, "completions/max_terminated_length": 1641.0, "completions/mean_length": 474.3125, "completions/mean_terminated_length": 460.6773986816406, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.632196981459856, "epoch": 0.00341, "frac_reward_zero_std": 0.0, "grad_norm": 0.5825992226600647, "kl": 0.6789249051362276, "learning_rate": 4.3403247004922434e-05, "loss": 0.0882, "num_tokens": 7198555.0, "reward": 0.732464075088501, "reward_std": 0.48086434602737427, "rewards/rollout_reward_func/mean": 0.732464075088501, "rewards/rollout_reward_func/std": 0.4712190330028534, "sampling/importance_sampling_ratio/max": 1.1093218326568604, "sampling/importance_sampling_ratio/mean": 0.9946275949478149, "sampling/importance_sampling_ratio/min": 0.8799177408218384, "sampling/sampling_logp_difference/max": 0.1281604766845703, "sampling/sampling_logp_difference/mean": 0.01588056981563568, "step": 341, "step_time": 9.983962875000088 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.032291666604578495, "clip_ratio/low_min": 0.012500000186264515, "clip_ratio/region_mean": 0.032291666604578495, "entropy": 0.6400797329843044, "epoch": 0.00342, "grad_norm": 0.45089128613471985, "kl": 0.817920533940196, "learning_rate": 4.3363695970558946e-05, "loss": 0.0799, "step": 342, "step_time": 5.950543429999925 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1650.0, "completions/max_terminated_length": 1650.0, "completions/mean_length": 560.25, "completions/mean_terminated_length": 550.258056640625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7788029368966818, "epoch": 0.00343, "frac_reward_zero_std": 0.0, "grad_norm": 0.5923560261726379, "kl": 0.4897633232176304, "learning_rate": 4.332402753755497e-05, "loss": 0.0735, "num_tokens": 7246133.0, "reward": 0.803714394569397, "reward_std": 0.4895680248737335, "rewards/rollout_reward_func/mean": 0.803714394569397, "rewards/rollout_reward_func/std": 0.48519209027290344, "sampling/importance_sampling_ratio/max": 1.2022066116333008, "sampling/importance_sampling_ratio/mean": 1.005216360092163, "sampling/importance_sampling_ratio/min": 0.8489552140235901, "sampling/sampling_logp_difference/max": 0.20374548435211182, "sampling/sampling_logp_difference/mean": 0.017059212550520897, "step": 343, "step_time": 11.164905831999931 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.029276315588504076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.029276315588504076, "entropy": 0.7908334154635668, "epoch": 0.00344, "grad_norm": 0.32026180624961853, "kl": 0.528102558106184, "learning_rate": 4.328424201252141e-05, "loss": 0.0634, "step": 344, "step_time": 5.37520256399921 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2155.0, "completions/max_terminated_length": 2155.0, "completions/mean_length": 503.25, "completions/mean_terminated_length": 504.70965576171875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6613400653004646, "epoch": 0.00345, "frac_reward_zero_std": 0.0, "grad_norm": 3.0360212326049805, "kl": 9.649091655388474, "learning_rate": 4.324433970297427e-05, "loss": 0.0399, "num_tokens": 7290770.0, "reward": 0.523584246635437, "reward_std": 0.7526698708534241, "rewards/rollout_reward_func/mean": 0.523584246635437, "rewards/rollout_reward_func/std": 0.7618927359580994, "sampling/importance_sampling_ratio/max": 1.093595027923584, "sampling/importance_sampling_ratio/mean": 0.9919476509094238, "sampling/importance_sampling_ratio/min": 0.6683241128921509, "sampling/sampling_logp_difference/max": 0.295074462890625, "sampling/sampling_logp_difference/mean": 0.016062866896390915, "step": 345, "step_time": 11.092279714999677 }, { "clip_ratio/high_max": 0.01597222243435681, "clip_ratio/high_mean": 0.007986111217178404, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.031423611217178404, "entropy": 0.6953675169497728, "epoch": 0.00346, "grad_norm": 1.0117907524108887, "kl": 1.5080317314714193, "learning_rate": 4.3204320917332166e-05, "loss": 0.0188, "step": 346, "step_time": 6.11939109199966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 466.71875, "completions/mean_terminated_length": 466.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.701933030039072, "epoch": 0.00347, "frac_reward_zero_std": 0.75, "grad_norm": 0.14672230184078217, "kl": 0.3798035439103842, "learning_rate": 4.3164185964914016e-05, "loss": -0.0035, "num_tokens": 7334207.0, "reward": 0.8762474060058594, "reward_std": 0.22915199398994446, "rewards/rollout_reward_func/mean": 0.8762474060058594, "rewards/rollout_reward_func/std": 0.48697227239608765, "sampling/importance_sampling_ratio/max": 1.143469214439392, "sampling/importance_sampling_ratio/mean": 1.002418875694275, "sampling/importance_sampling_ratio/min": 0.8431497812271118, "sampling/sampling_logp_difference/max": 0.12730979919433594, "sampling/sampling_logp_difference/mean": 0.01506867166608572, "step": 347, "step_time": 9.78435548500056 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "entropy": 0.6955453082919121, "epoch": 0.00348, "grad_norm": 0.06734003871679306, "kl": 0.370736924931407, "learning_rate": 4.3123935155936636e-05, "loss": -0.0046, "step": 348, "step_time": 4.952898312999878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1641.0, "completions/max_terminated_length": 1641.0, "completions/mean_length": 487.625, "completions/mean_terminated_length": 487.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6069292388856411, "epoch": 0.00349, "frac_reward_zero_std": 0.25, "grad_norm": 0.10803303122520447, "kl": 0.26759606413543224, "learning_rate": 4.308356880151235e-05, "loss": -0.0082, "num_tokens": 7378110.0, "reward": 0.8090420961380005, "reward_std": 0.45688360929489136, "rewards/rollout_reward_func/mean": 0.8090420961380005, "rewards/rollout_reward_func/std": 0.5384293794631958, "sampling/importance_sampling_ratio/max": 1.120185375213623, "sampling/importance_sampling_ratio/mean": 1.0048651695251465, "sampling/importance_sampling_ratio/min": 0.9301095604896545, "sampling/sampling_logp_difference/max": 0.09678077697753906, "sampling/sampling_logp_difference/mean": 0.009677057154476643, "step": 349, "step_time": 10.250850643000376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02083333395421505, "entropy": 0.605390477925539, "epoch": 0.0035, "grad_norm": 0.0544484481215477, "kl": 0.26622752845287323, "learning_rate": 4.3043087213646516e-05, "loss": -0.0093, "step": 350, "step_time": 5.742831162000584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1337.0, "completions/max_terminated_length": 1337.0, "completions/mean_length": 382.9375, "completions/mean_terminated_length": 382.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6164091154932976, "epoch": 0.00351, "frac_reward_zero_std": 0.25, "grad_norm": 0.11408384889364243, "kl": 0.6749834679067135, "learning_rate": 4.3002490705235254e-05, "loss": -0.0061, "num_tokens": 7418159.0, "reward": 0.6795245409011841, "reward_std": 0.5551409721374512, "rewards/rollout_reward_func/mean": 0.6795245409011841, "rewards/rollout_reward_func/std": 0.6506083607673645, "sampling/importance_sampling_ratio/max": 1.2475093603134155, "sampling/importance_sampling_ratio/mean": 1.0095927715301514, "sampling/importance_sampling_ratio/min": 0.9041814804077148, "sampling/sampling_logp_difference/max": 0.12455606460571289, "sampling/sampling_logp_difference/mean": 0.012481662444770336, "step": 351, "step_time": 8.542516594000517 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018229166977107525, "entropy": 0.6070123575627804, "epoch": 0.00352, "grad_norm": 0.08731453865766525, "kl": 0.6903661005198956, "learning_rate": 4.296177959006287e-05, "loss": -0.0072, "step": 352, "step_time": 4.568527246999565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2263.0, "completions/max_terminated_length": 2263.0, "completions/mean_length": 502.3125, "completions/mean_terminated_length": 502.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8343955129384995, "epoch": 0.00353, "frac_reward_zero_std": 0.0, "grad_norm": 0.2725044786930084, "kl": 1.4547460451722145, "learning_rate": 4.2920954182799555e-05, "loss": -0.0012, "num_tokens": 7462995.0, "reward": 0.5895282030105591, "reward_std": 0.7681360244750977, "rewards/rollout_reward_func/mean": 0.5895282030105591, "rewards/rollout_reward_func/std": 0.7552579045295715, "sampling/importance_sampling_ratio/max": 1.1805355548858643, "sampling/importance_sampling_ratio/mean": 0.9914667010307312, "sampling/importance_sampling_ratio/min": 0.8613514304161072, "sampling/sampling_logp_difference/max": 0.23897933959960938, "sampling/sampling_logp_difference/mean": 0.023064112290740013, "step": 353, "step_time": 11.531813257000067 }, { "clip_ratio/high_max": 0.018939394503831863, "clip_ratio/high_mean": 0.009469697251915932, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01571969734504819, "entropy": 0.8259334042668343, "epoch": 0.00354, "grad_norm": 0.2079463005065918, "kl": 1.3386908024549484, "learning_rate": 4.2880014798998875e-05, "loss": -0.0044, "step": 354, "step_time": 6.325278668000465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1515.0, "completions/max_terminated_length": 1515.0, "completions/mean_length": 495.6875, "completions/mean_terminated_length": 495.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5468737296760082, "epoch": 0.00355, "frac_reward_zero_std": 0.0, "grad_norm": 0.10151279717683792, "kl": 0.4225687989965081, "learning_rate": 4.2838961755095375e-05, "loss": -0.0103, "num_tokens": 7507804.0, "reward": 0.7050447463989258, "reward_std": 0.5891166925430298, "rewards/rollout_reward_func/mean": 0.7050447463989258, "rewards/rollout_reward_func/std": 0.5943097472190857, "sampling/importance_sampling_ratio/max": 1.1402053833007812, "sampling/importance_sampling_ratio/mean": 1.0047792196273804, "sampling/importance_sampling_ratio/min": 0.9034812450408936, "sampling/sampling_logp_difference/max": 0.10482025146484375, "sampling/sampling_logp_difference/mean": 0.010407581925392151, "step": 355, "step_time": 9.490019487999234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5319183468818665, "epoch": 0.00356, "grad_norm": 0.10277921706438065, "kl": 0.420918975956738, "learning_rate": 4.27977953684021e-05, "loss": -0.0107, "step": 356, "step_time": 5.888208553000368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 521.125, "completions/mean_terminated_length": 521.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.48934782668948174, "epoch": 0.00357, "frac_reward_zero_std": 0.0, "grad_norm": 0.10051943361759186, "kl": 0.2608627127483487, "learning_rate": 4.2756515957108176e-05, "loss": -0.0012, "num_tokens": 7553248.0, "reward": 0.7110182046890259, "reward_std": 0.6078996658325195, "rewards/rollout_reward_func/mean": 0.7110182046890259, "rewards/rollout_reward_func/std": 0.5866532921791077, "sampling/importance_sampling_ratio/max": 1.1535699367523193, "sampling/importance_sampling_ratio/mean": 1.0095512866973877, "sampling/importance_sampling_ratio/min": 0.8939035534858704, "sampling/sampling_logp_difference/max": 0.1240243911743164, "sampling/sampling_logp_difference/mean": 0.010991412214934826, "step": 357, "step_time": 9.569336844000645 }, { "clip_ratio/high_max": 0.026041666977107525, "clip_ratio/high_mean": 0.013020833488553762, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013020833488553762, "entropy": 0.4776310473680496, "epoch": 0.00358, "grad_norm": 0.1023847684264183, "kl": 0.26106308959424496, "learning_rate": 4.271512384027634e-05, "loss": -0.002, "step": 358, "step_time": 5.159366729999874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 423.9375, "completions/mean_terminated_length": 423.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5032856054604053, "epoch": 0.00359, "frac_reward_zero_std": 0.0, "grad_norm": 0.08306502550840378, "kl": 0.5988969393074512, "learning_rate": 4.2673619337840456e-05, "loss": 0.0001, "num_tokens": 7595857.0, "reward": 0.7046263217926025, "reward_std": 0.57477867603302, "rewards/rollout_reward_func/mean": 0.7046263217926025, "rewards/rollout_reward_func/std": 0.5984350442886353, "sampling/importance_sampling_ratio/max": 1.1430495977401733, "sampling/importance_sampling_ratio/mean": 0.9764826893806458, "sampling/importance_sampling_ratio/min": 6.468362512279668e-10, "sampling/sampling_logp_difference/max": 20.58070945739746, "sampling/sampling_logp_difference/mean": 0.20884090662002563, "step": 359, "step_time": 9.158228839000458 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.4971914179623127, "epoch": 0.0036, "grad_norm": 0.055561769753694534, "kl": 0.6172671392560005, "learning_rate": 4.263200277060305e-05, "loss": -0.0006, "step": 360, "step_time": 4.955845019999288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1578.0, "completions/max_terminated_length": 1578.0, "completions/mean_length": 464.28125, "completions/mean_terminated_length": 417.5333557128906, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7958513526245952, "epoch": 0.00361, "frac_reward_zero_std": 0.25, "grad_norm": 0.34588760137557983, "kl": 0.4966821763664484, "learning_rate": 4.259027446023285e-05, "loss": 0.0058, "num_tokens": 7640198.0, "reward": 0.4895184636116028, "reward_std": 0.6230394840240479, "rewards/rollout_reward_func/mean": 0.4895184636116028, "rewards/rollout_reward_func/std": 0.7616648077964783, "sampling/importance_sampling_ratio/max": 1.139635443687439, "sampling/importance_sampling_ratio/mean": 0.9909803867340088, "sampling/importance_sampling_ratio/min": 0.48783132433891296, "sampling/sampling_logp_difference/max": 0.2935328483581543, "sampling/sampling_logp_difference/mean": 0.018762625753879547, "step": 361, "step_time": 9.989780618999703 }, { "clip_ratio/high_max": 0.0053571430034935474, "clip_ratio/high_mean": 0.0026785715017467737, "clip_ratio/low_mean": 0.07506396248936653, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.0777425339911133, "entropy": 0.7818032493814826, "epoch": 0.00362, "grad_norm": 0.18052653968334198, "kl": 0.5682998839765787, "learning_rate": 4.254843472926228e-05, "loss": 0.0003, "step": 362, "step_time": 5.91787168799965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 450.71875, "completions/mean_terminated_length": 450.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4615401830524206, "epoch": 0.00363, "frac_reward_zero_std": 0.25, "grad_norm": 0.17417453229427338, "kl": 0.5772524029016495, "learning_rate": 4.250648390108496e-05, "loss": 0.0009, "num_tokens": 7682933.0, "reward": 0.652418851852417, "reward_std": 0.6015955209732056, "rewards/rollout_reward_func/mean": 0.652418851852417, "rewards/rollout_reward_func/std": 0.7005115747451782, "sampling/importance_sampling_ratio/max": 1.1305938959121704, "sampling/importance_sampling_ratio/mean": 1.0007251501083374, "sampling/importance_sampling_ratio/min": 0.9111924767494202, "sampling/sampling_logp_difference/max": 0.12165307998657227, "sampling/sampling_logp_difference/mean": 0.011711962521076202, "step": 363, "step_time": 10.203346076998969 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.012276785913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012276785913854837, "entropy": 0.44634336791932583, "epoch": 0.00364, "grad_norm": 0.10604748874902725, "kl": 0.6175209991633892, "learning_rate": 4.2464422299953245e-05, "loss": -0.0004, "step": 364, "step_time": 5.252411510000002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 454.59375, "completions/mean_terminated_length": 454.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4026250969618559, "epoch": 0.00365, "frac_reward_zero_std": 0.25, "grad_norm": 0.06739354878664017, "kl": 0.3823420759290457, "learning_rate": 4.242225025097567e-05, "loss": 0.0017, "num_tokens": 7726837.0, "reward": 0.6604292392730713, "reward_std": 0.5875426530838013, "rewards/rollout_reward_func/mean": 0.6604292392730713, "rewards/rollout_reward_func/std": 0.6849721670150757, "sampling/importance_sampling_ratio/max": 1.1312222480773926, "sampling/importance_sampling_ratio/mean": 0.9977399110794067, "sampling/importance_sampling_ratio/min": 0.8942009806632996, "sampling/sampling_logp_difference/max": 0.12320566177368164, "sampling/sampling_logp_difference/mean": 0.007191185839474201, "step": 365, "step_time": 9.829569907000405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.38619719073176384, "epoch": 0.00366, "grad_norm": 0.06801632046699524, "kl": 0.42088502272963524, "learning_rate": 4.237996808011445e-05, "loss": 0.0015, "step": 366, "step_time": 5.253398706000098 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 2297.0, "completions/max_terminated_length": 2297.0, "completions/mean_length": 565.0, "completions/mean_terminated_length": 559.300048828125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8809302225708961, "epoch": 0.00367, "frac_reward_zero_std": 0.25, "grad_norm": 0.4834566116333008, "kl": 1.0112892109900713, "learning_rate": 4.233757611418301e-05, "loss": 0.0722, "num_tokens": 7773965.0, "reward": 0.5905121564865112, "reward_std": 0.6229978799819946, "rewards/rollout_reward_func/mean": 0.5905121564865112, "rewards/rollout_reward_func/std": 0.7537458539009094, "sampling/importance_sampling_ratio/max": 1.1823276281356812, "sampling/importance_sampling_ratio/mean": 0.9586366415023804, "sampling/importance_sampling_ratio/min": 5.065359515567334e-15, "sampling/sampling_logp_difference/max": 18.029905319213867, "sampling/sampling_logp_difference/mean": 0.13137422502040863, "step": 367, "step_time": 12.23194153499935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.04141212534159422, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04141212534159422, "entropy": 0.8761169165372849, "epoch": 0.00368, "grad_norm": 0.24676255881786346, "kl": 0.7297209650278091, "learning_rate": 4.229507468084338e-05, "loss": 0.0632, "step": 368, "step_time": 6.486794290999569 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 414.78125, "completions/mean_terminated_length": 400.3870849609375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.45528820902109146, "epoch": 0.00369, "frac_reward_zero_std": 0.0, "grad_norm": 0.29828330874443054, "kl": 1.1324495263397694, "learning_rate": 4.225246410860372e-05, "loss": 0.0279, "num_tokens": 7815839.0, "reward": 0.5906941294670105, "reward_std": 0.782216489315033, "rewards/rollout_reward_func/mean": 0.5906941294670105, "rewards/rollout_reward_func/std": 0.7541914582252502, "sampling/importance_sampling_ratio/max": 1.1149812936782837, "sampling/importance_sampling_ratio/mean": 0.9882969856262207, "sampling/importance_sampling_ratio/min": 0.5618729591369629, "sampling/sampling_logp_difference/max": 0.2529163360595703, "sampling/sampling_logp_difference/mean": 0.012171346694231033, "step": 369, "step_time": 10.19879210199997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.018457602243870497, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018457602243870497, "entropy": 0.4437125362455845, "epoch": 0.0037, "grad_norm": 0.18155910074710846, "kl": 1.216697745025158, "learning_rate": 4.2209744726815754e-05, "loss": 0.0243, "step": 370, "step_time": 5.63390126000013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1577.0, "completions/max_terminated_length": 1577.0, "completions/mean_length": 360.28125, "completions/mean_terminated_length": 360.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3528400235809386, "epoch": 0.00371, "frac_reward_zero_std": 0.25, "grad_norm": 0.08418449014425278, "kl": 0.4194776304066181, "learning_rate": 4.216691686567224e-05, "loss": 0.0, "num_tokens": 7855788.0, "reward": 0.650834321975708, "reward_std": 0.5992856621742249, "rewards/rollout_reward_func/mean": 0.650834321975708, "rewards/rollout_reward_func/std": 0.6987522840499878, "sampling/importance_sampling_ratio/max": 1.1263130903244019, "sampling/importance_sampling_ratio/mean": 1.002410888671875, "sampling/importance_sampling_ratio/min": 0.9115346670150757, "sampling/sampling_logp_difference/max": 0.11818218231201172, "sampling/sampling_logp_difference/mean": 0.00809308048337698, "step": 371, "step_time": 9.592356272000416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3352625537663698, "epoch": 0.00372, "grad_norm": 0.07132259011268616, "kl": 0.4576851427555084, "learning_rate": 4.21239808562044e-05, "loss": -0.0011, "step": 372, "step_time": 5.206090285000755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 463.0625, "completions/mean_terminated_length": 463.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29397106915712357, "epoch": 0.00373, "frac_reward_zero_std": 0.0, "grad_norm": 0.08391261100769043, "kl": 0.45322751998901367, "learning_rate": 4.208093703027938e-05, "loss": 0.0047, "num_tokens": 7898649.0, "reward": 0.745989203453064, "reward_std": 0.5865702629089355, "rewards/rollout_reward_func/mean": 0.745989203453064, "rewards/rollout_reward_func/std": 0.563930094242096, "sampling/importance_sampling_ratio/max": 1.03854238986969, "sampling/importance_sampling_ratio/mean": 0.9928315877914429, "sampling/importance_sampling_ratio/min": 0.8882851600646973, "sampling/sampling_logp_difference/max": 0.11975312232971191, "sampling/sampling_logp_difference/mean": 0.006819797679781914, "step": 373, "step_time": 9.522878769000272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.2794056236743927, "epoch": 0.00374, "grad_norm": 0.09144020080566406, "kl": 0.4578029550611973, "learning_rate": 4.203778572059768e-05, "loss": 0.0044, "step": 374, "step_time": 5.070840125000359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1722.0, "completions/max_terminated_length": 1722.0, "completions/mean_length": 346.125, "completions/mean_terminated_length": 346.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.25485622067935765, "epoch": 0.00375, "frac_reward_zero_std": 0.0, "grad_norm": 0.1620047241449356, "kl": 1.0151331275701523, "learning_rate": 4.199452726069055e-05, "loss": 0.0097, "num_tokens": 7936878.0, "reward": 0.5626060962677002, "reward_std": 0.7264589071273804, "rewards/rollout_reward_func/mean": 0.5626060962677002, "rewards/rollout_reward_func/std": 0.7461064457893372, "sampling/importance_sampling_ratio/max": 1.0896738767623901, "sampling/importance_sampling_ratio/mean": 0.9897398948669434, "sampling/importance_sampling_ratio/min": 0.6794376969337463, "sampling/sampling_logp_difference/max": 0.47337818145751953, "sampling/sampling_logp_difference/mean": 0.010339752770960331, "step": 375, "step_time": 9.58669288299916 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018229166977107525, "entropy": 0.25335289421491325, "epoch": 0.00376, "grad_norm": 0.076473668217659, "kl": 0.7679913304746151, "learning_rate": 4.1951161984917496e-05, "loss": 0.0085, "step": 376, "step_time": 5.8245141859993055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2371.0, "completions/max_terminated_length": 2371.0, "completions/mean_length": 481.0625, "completions/mean_terminated_length": 481.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24128277320414782, "epoch": 0.00377, "frac_reward_zero_std": 0.0, "grad_norm": 0.04783071577548981, "kl": 0.338759608566761, "learning_rate": 4.1907690228463574e-05, "loss": -0.0036, "num_tokens": 7979860.0, "reward": 0.7747324705123901, "reward_std": 0.5710453987121582, "rewards/rollout_reward_func/mean": 0.7747324705123901, "rewards/rollout_reward_func/std": 0.5596310496330261, "sampling/importance_sampling_ratio/max": 1.0599762201309204, "sampling/importance_sampling_ratio/mean": 1.000121831893921, "sampling/importance_sampling_ratio/min": 0.9314819574356079, "sampling/sampling_logp_difference/max": 0.07123589515686035, "sampling/sampling_logp_difference/mean": 0.0032973105553537607, "step": 377, "step_time": 12.066876174999834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.24364001210778952, "epoch": 0.00378, "grad_norm": 0.05094004049897194, "kl": 0.3395290728658438, "learning_rate": 4.186411232733692e-05, "loss": -0.0037, "step": 378, "step_time": 6.43166366999958 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2218.0, "completions/max_terminated_length": 2218.0, "completions/mean_length": 501.375, "completions/mean_terminated_length": 501.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2648435812443495, "epoch": 0.00379, "frac_reward_zero_std": 0.25, "grad_norm": 0.09919844567775726, "kl": 0.33282697200775146, "learning_rate": 4.1820428618366076e-05, "loss": 0.0057, "num_tokens": 8024933.0, "reward": 0.8429239392280579, "reward_std": 0.4442782700061798, "rewards/rollout_reward_func/mean": 0.8429239392280579, "rewards/rollout_reward_func/std": 0.5136594772338867, "sampling/importance_sampling_ratio/max": 1.1308516263961792, "sampling/importance_sampling_ratio/mean": 1.005608320236206, "sampling/importance_sampling_ratio/min": 0.9021822214126587, "sampling/sampling_logp_difference/max": 0.12277078628540039, "sampling/sampling_logp_difference/mean": 0.007960977032780647, "step": 379, "step_time": 12.037136122000902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.26625112630426884, "epoch": 0.0038, "grad_norm": 0.0937490239739418, "kl": 0.33420818485319614, "learning_rate": 4.1776639439197414e-05, "loss": 0.0055, "step": 380, "step_time": 6.484801124999194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3285.0, "completions/max_terminated_length": 3285.0, "completions/mean_length": 386.46875, "completions/mean_terminated_length": 386.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2596753127872944, "epoch": 0.00381, "frac_reward_zero_std": 0.25, "grad_norm": 0.24410833418369293, "kl": 0.4364766776561737, "learning_rate": 4.173274512829251e-05, "loss": 0.0174, "num_tokens": 8063613.0, "reward": 0.6653199195861816, "reward_std": 0.5788025259971619, "rewards/rollout_reward_func/mean": 0.6653199195861816, "rewards/rollout_reward_func/std": 0.7229302525520325, "sampling/importance_sampling_ratio/max": 1.1545072793960571, "sampling/importance_sampling_ratio/mean": 1.0035887956619263, "sampling/importance_sampling_ratio/min": 0.9284628033638, "sampling/sampling_logp_difference/max": 0.10945010185241699, "sampling/sampling_logp_difference/mean": 0.005616612732410431, "step": 381, "step_time": 14.24305955600039 }, { "clip_ratio/high_max": 0.008928571827709675, "clip_ratio/high_mean": 0.004464285913854837, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0174851194024086, "entropy": 0.25632583629339933, "epoch": 0.00382, "grad_norm": 0.1417224407196045, "kl": 0.44833339378237724, "learning_rate": 4.168874602492554e-05, "loss": 0.015, "step": 382, "step_time": 8.263422037000055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 563.59375, "completions/mean_terminated_length": 563.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.323261059820652, "epoch": 0.00383, "frac_reward_zero_std": 0.25, "grad_norm": 0.1879626214504242, "kl": 0.77327511459589, "learning_rate": 4.16446424691807e-05, "loss": 0.0002, "num_tokens": 8111060.0, "reward": 0.8067282438278198, "reward_std": 0.4181429147720337, "rewards/rollout_reward_func/mean": 0.8067282438278198, "rewards/rollout_reward_func/std": 0.4793858826160431, "sampling/importance_sampling_ratio/max": 1.705285668373108, "sampling/importance_sampling_ratio/mean": 1.0152080059051514, "sampling/importance_sampling_ratio/min": 0.8923817873001099, "sampling/sampling_logp_difference/max": 0.3749675750732422, "sampling/sampling_logp_difference/mean": 0.00929285492748022, "step": 383, "step_time": 10.328516400999888 }, { "clip_ratio/high_max": 0.05937500111758709, "clip_ratio/high_mean": 0.0359375006519258, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0359375006519258, "entropy": 0.33151754178106785, "epoch": 0.00384, "grad_norm": 0.07678636163473129, "kl": 0.7366516441106796, "learning_rate": 4.1600434801949445e-05, "loss": -0.0007, "step": 384, "step_time": 5.707346123000207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1440.0, "completions/max_terminated_length": 1440.0, "completions/mean_length": 417.625, "completions/mean_terminated_length": 417.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5351597033441067, "epoch": 0.00385, "frac_reward_zero_std": 0.5, "grad_norm": 0.28843820095062256, "kl": 1.414673324674368, "learning_rate": 4.155612336492804e-05, "loss": -0.0274, "num_tokens": 8153313.0, "reward": 0.5952019095420837, "reward_std": 0.500468909740448, "rewards/rollout_reward_func/mean": 0.5952019095420837, "rewards/rollout_reward_func/std": 0.7901350259780884, "sampling/importance_sampling_ratio/max": 1.3714433908462524, "sampling/importance_sampling_ratio/mean": 0.9942135214805603, "sampling/importance_sampling_ratio/min": 0.8369652032852173, "sampling/sampling_logp_difference/max": 0.17622709274291992, "sampling/sampling_logp_difference/mean": 0.014460677281022072, "step": 385, "step_time": 9.051290320000135 }, { "clip_ratio/high_max": 0.048684210516512394, "clip_ratio/high_mean": 0.024342105258256197, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.024342105258256197, "entropy": 0.5511863082647324, "epoch": 0.00386, "grad_norm": 0.18414027988910675, "kl": 1.107634324580431, "learning_rate": 4.151170850061476e-05, "loss": -0.0311, "step": 386, "step_time": 4.810029213000689 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2254.0, "completions/max_terminated_length": 2254.0, "completions/mean_length": 646.4375, "completions/mean_terminated_length": 646.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4160290639847517, "epoch": 0.00387, "frac_reward_zero_std": 0.5, "grad_norm": 0.1863703429698944, "kl": 0.5414737984538078, "learning_rate": 4.146719055230733e-05, "loss": 0.0114, "num_tokens": 8202651.0, "reward": 0.7737464904785156, "reward_std": 0.3250160813331604, "rewards/rollout_reward_func/mean": 0.7737464904785156, "rewards/rollout_reward_func/std": 0.49762073159217834, "sampling/importance_sampling_ratio/max": 1.0935289859771729, "sampling/importance_sampling_ratio/mean": 1.0091736316680908, "sampling/importance_sampling_ratio/min": 0.9430432915687561, "sampling/sampling_logp_difference/max": 0.09171652793884277, "sampling/sampling_logp_difference/mean": 0.008708924055099487, "step": 387, "step_time": 11.776834918000532 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.012500000186264515, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018750000279396772, "entropy": 0.4375996496528387, "epoch": 0.00388, "grad_norm": 0.07239069044589996, "kl": 0.5298512019217014, "learning_rate": 4.142256986410024e-05, "loss": 0.0101, "step": 388, "step_time": 6.4338605620000635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2173.0, "completions/max_terminated_length": 2173.0, "completions/mean_length": 386.875, "completions/mean_terminated_length": 386.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3413195312023163, "epoch": 0.00389, "frac_reward_zero_std": 0.0, "grad_norm": 0.1715715527534485, "kl": 0.5165232140570879, "learning_rate": 4.137784678088211e-05, "loss": -0.0092, "num_tokens": 8243846.0, "reward": 0.6253501176834106, "reward_std": 0.5158167481422424, "rewards/rollout_reward_func/mean": 0.6253501176834106, "rewards/rollout_reward_func/std": 0.5261673927307129, "sampling/importance_sampling_ratio/max": 1.1272265911102295, "sampling/importance_sampling_ratio/mean": 0.9994260668754578, "sampling/importance_sampling_ratio/min": 0.8974222540855408, "sampling/sampling_logp_difference/max": 0.11031603813171387, "sampling/sampling_logp_difference/mean": 0.007148211821913719, "step": 389, "step_time": 11.676560680000875 }, { "clip_ratio/high_max": 0.0416666679084301, "clip_ratio/high_mean": 0.02083333395421505, "clip_ratio/low_mean": 0.026041666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.046875000931322575, "entropy": 0.3704727999866009, "epoch": 0.0039, "grad_norm": 0.06453483551740646, "kl": 0.4906091634184122, "learning_rate": 4.133302164833296e-05, "loss": -0.0104, "step": 390, "step_time": 6.280174066998825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1686.0, "completions/max_terminated_length": 1686.0, "completions/mean_length": 459.75, "completions/mean_terminated_length": 459.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5039037503302097, "epoch": 0.00391, "frac_reward_zero_std": 0.0, "grad_norm": 0.31755173206329346, "kl": 1.0116702541708946, "learning_rate": 4.128809481292163e-05, "loss": 0.0162, "num_tokens": 8287082.0, "reward": 0.7075576186180115, "reward_std": 0.592479407787323, "rewards/rollout_reward_func/mean": 0.7075576186180115, "rewards/rollout_reward_func/std": 0.5909351706504822, "sampling/importance_sampling_ratio/max": 1.115526556968689, "sampling/importance_sampling_ratio/mean": 0.9945210218429565, "sampling/importance_sampling_ratio/min": 0.7259218096733093, "sampling/sampling_logp_difference/max": 0.24725914001464844, "sampling/sampling_logp_difference/mean": 0.012152740731835365, "step": 391, "step_time": 9.608037112000602 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.026988637633621693, "clip_ratio/low_min": 0.010416666977107525, "clip_ratio/region_mean": 0.03480113763362169, "entropy": 0.5166999790817499, "epoch": 0.00392, "grad_norm": 0.17852900922298431, "kl": 1.01947782933712, "learning_rate": 4.124306662190303e-05, "loss": 0.0149, "step": 392, "step_time": 5.278935193000052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2182.0, "completions/max_terminated_length": 2182.0, "completions/mean_length": 547.59375, "completions/mean_terminated_length": 547.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.517743282020092, "epoch": 0.00393, "frac_reward_zero_std": 0.0, "grad_norm": 0.37221887707710266, "kl": 0.7111997380852699, "learning_rate": 4.119793742331549e-05, "loss": 0.0204, "num_tokens": 8334274.0, "reward": 0.6798945069313049, "reward_std": 0.6645604372024536, "rewards/rollout_reward_func/mean": 0.6798945069313049, "rewards/rollout_reward_func/std": 0.6481858491897583, "sampling/importance_sampling_ratio/max": 1.1463518142700195, "sampling/importance_sampling_ratio/mean": 1.0055054426193237, "sampling/importance_sampling_ratio/min": 0.7726274728775024, "sampling/sampling_logp_difference/max": 0.2084038257598877, "sampling/sampling_logp_difference/mean": 0.012264474295079708, "step": 393, "step_time": 11.004280850999749 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.034226191230118275, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.042038691230118275, "entropy": 0.5238918662071228, "epoch": 0.00394, "grad_norm": 0.1026676669716835, "kl": 0.6502073369920254, "learning_rate": 4.115270756597808e-05, "loss": 0.0161, "step": 394, "step_time": 6.1182020230007765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 505.40625, "completions/mean_terminated_length": 505.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.456380944699049, "epoch": 0.00395, "frac_reward_zero_std": 0.0, "grad_norm": 0.21470192074775696, "kl": 0.3950885348021984, "learning_rate": 4.110737739948788e-05, "loss": 0.001, "num_tokens": 8378746.0, "reward": 0.7015014886856079, "reward_std": 0.566827654838562, "rewards/rollout_reward_func/mean": 0.7015014886856079, "rewards/rollout_reward_func/std": 0.5479796528816223, "sampling/importance_sampling_ratio/max": 1.0746514797210693, "sampling/importance_sampling_ratio/mean": 0.9947313070297241, "sampling/importance_sampling_ratio/min": 0.9040399789810181, "sampling/sampling_logp_difference/max": 0.1033167839050293, "sampling/sampling_logp_difference/mean": 0.007041980046778917, "step": 395, "step_time": 9.575984476999565 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.0338541679084301, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.04010416753590107, "entropy": 0.4637001659721136, "epoch": 0.00396, "grad_norm": 0.12741832435131073, "kl": 0.39260758832097054, "learning_rate": 4.106194727421728e-05, "loss": -0.001, "step": 396, "step_time": 5.7358733780001785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1355.0, "completions/max_terminated_length": 1355.0, "completions/mean_length": 432.625, "completions/mean_terminated_length": 432.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5147895142436028, "epoch": 0.00397, "frac_reward_zero_std": 0.0, "grad_norm": 0.12085860222578049, "kl": 0.4576429072767496, "learning_rate": 4.10164175413113e-05, "loss": -0.0022, "num_tokens": 8421943.0, "reward": 0.5248082876205444, "reward_std": 0.7158059477806091, "rewards/rollout_reward_func/mean": 0.5248082876205444, "rewards/rollout_reward_func/std": 0.7581151723861694, "sampling/importance_sampling_ratio/max": 1.2673338651657104, "sampling/importance_sampling_ratio/mean": 1.0087099075317383, "sampling/importance_sampling_ratio/min": 0.8257744908332825, "sampling/sampling_logp_difference/max": 0.22989845275878906, "sampling/sampling_logp_difference/mean": 0.011920465156435966, "step": 397, "step_time": 9.439630241999566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.5221869461238384, "epoch": 0.00398, "grad_norm": 0.10854934900999069, "kl": 0.44265278056263924, "learning_rate": 4.097078855268486e-05, "loss": -0.0032, "step": 398, "step_time": 4.703306373000032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 529.53125, "completions/mean_terminated_length": 529.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4612951911985874, "epoch": 0.00399, "frac_reward_zero_std": 0.25, "grad_norm": 0.048969365656375885, "kl": 0.33515724167227745, "learning_rate": 4.092506066102009e-05, "loss": -0.0008, "num_tokens": 8468448.0, "reward": 0.7423572540283203, "reward_std": 0.44537490606307983, "rewards/rollout_reward_func/mean": 0.7423572540283203, "rewards/rollout_reward_func/std": 0.5770049691200256, "sampling/importance_sampling_ratio/max": 1.064781665802002, "sampling/importance_sampling_ratio/mean": 0.986196756362915, "sampling/importance_sampling_ratio/min": 0.8893927335739136, "sampling/sampling_logp_difference/max": 0.1034696102142334, "sampling/sampling_logp_difference/mean": 0.009344853460788727, "step": 399, "step_time": 9.88572034300023 }, { "clip_ratio/high_max": 0.033333334140479565, "clip_ratio/high_mean": 0.016666667070239782, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.016666667070239782, "entropy": 0.47243113070726395, "epoch": 0.004, "grad_norm": 0.05256303399801254, "kl": 0.32873488031327724, "learning_rate": 4.087923421976351e-05, "loss": -0.0012, "step": 400, "step_time": 5.268684800000301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 440.0625, "completions/mean_terminated_length": 440.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.46761813201010227, "epoch": 0.00401, "frac_reward_zero_std": 0.0, "grad_norm": 0.16245989501476288, "kl": 0.4201507940888405, "learning_rate": 4.0833309583123426e-05, "loss": 0.0034, "num_tokens": 8511225.0, "reward": 0.712603747844696, "reward_std": 0.6362206339836121, "rewards/rollout_reward_func/mean": 0.712603747844696, "rewards/rollout_reward_func/std": 0.638583779335022, "sampling/importance_sampling_ratio/max": 1.107816457748413, "sampling/importance_sampling_ratio/mean": 0.9923955202102661, "sampling/importance_sampling_ratio/min": 0.8734209537506104, "sampling/sampling_logp_difference/max": 0.13483738899230957, "sampling/sampling_logp_difference/mean": 0.010167689993977547, "step": 401, "step_time": 9.603951930999301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.018229166977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018229166977107525, "entropy": 0.47620913200080395, "epoch": 0.00402, "grad_norm": 0.1279161274433136, "kl": 0.4176255315542221, "learning_rate": 4.07872871060671e-05, "loss": 0.0015, "step": 402, "step_time": 5.215042407000055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2239.0, "completions/max_terminated_length": 2239.0, "completions/mean_length": 445.1875, "completions/mean_terminated_length": 445.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4571065381169319, "epoch": 0.00403, "frac_reward_zero_std": 0.25, "grad_norm": 0.12207268178462982, "kl": 1.0014806343242526, "learning_rate": 4.074116714431803e-05, "loss": -0.0011, "num_tokens": 8553807.0, "reward": 0.6250513195991516, "reward_std": 0.6222379207611084, "rewards/rollout_reward_func/mean": 0.6250513195991516, "rewards/rollout_reward_func/std": 0.7476365566253662, "sampling/importance_sampling_ratio/max": 1.0739009380340576, "sampling/importance_sampling_ratio/mean": 1.0037544965744019, "sampling/importance_sampling_ratio/min": 0.835010826587677, "sampling/sampling_logp_difference/max": 0.1347123384475708, "sampling/sampling_logp_difference/mean": 0.009652459062635899, "step": 403, "step_time": 11.727427130000251 }, { "clip_ratio/high_max": 0.0052083334885537624, "clip_ratio/high_mean": 0.0026041667442768812, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026041667442768812, "entropy": 0.45736548118293285, "epoch": 0.00404, "grad_norm": 0.1653209775686264, "kl": 1.1533217877149582, "learning_rate": 4.0694950054353236e-05, "loss": -0.0017, "step": 404, "step_time": 6.935380875999272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2387.0, "completions/max_terminated_length": 2387.0, "completions/mean_length": 543.40625, "completions/mean_terminated_length": 531.4193115234375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6331015368923545, "epoch": 0.00405, "frac_reward_zero_std": 0.25, "grad_norm": 0.25636428594589233, "kl": 0.3299940191209316, "learning_rate": 4.0648636193400445e-05, "loss": 0.0334, "num_tokens": 8599731.0, "reward": 0.694064736366272, "reward_std": 0.5731330513954163, "rewards/rollout_reward_func/mean": 0.694064736366272, "rewards/rollout_reward_func/std": 0.6707468628883362, "sampling/importance_sampling_ratio/max": 1.1734092235565186, "sampling/importance_sampling_ratio/mean": 1.004514217376709, "sampling/importance_sampling_ratio/min": 0.52948397397995, "sampling/sampling_logp_difference/max": 0.23834514617919922, "sampling/sampling_logp_difference/mean": 0.017091620713472366, "step": 405, "step_time": 12.453502114999537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.01315789483487606, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01315789483487606, "entropy": 0.6357182250358164, "epoch": 0.00406, "grad_norm": 0.19149278104305267, "kl": 0.33816346153616905, "learning_rate": 4.060222591943538e-05, "loss": 0.0312, "step": 406, "step_time": 6.6912948500012135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 843.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 366.8125, "completions/mean_terminated_length": 351.45159912109375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.711659386754036, "epoch": 0.00407, "frac_reward_zero_std": 0.25, "grad_norm": 0.32513636350631714, "kl": 0.3112999424338341, "learning_rate": 4.055571959117897e-05, "loss": 0.0491, "num_tokens": 8639948.0, "reward": 0.7195809483528137, "reward_std": 0.5886447429656982, "rewards/rollout_reward_func/mean": 0.7195809483528137, "rewards/rollout_reward_func/std": 0.6776710748672485, "sampling/importance_sampling_ratio/max": 1.2261414527893066, "sampling/importance_sampling_ratio/mean": 0.9927012920379639, "sampling/importance_sampling_ratio/min": 0.7633904218673706, "sampling/sampling_logp_difference/max": 0.43830323219299316, "sampling/sampling_logp_difference/mean": 0.02067318744957447, "step": 407, "step_time": 7.441402289000507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.034722222946584225, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.034722222946584225, "entropy": 0.7031948789954185, "epoch": 0.00408, "grad_norm": 0.2013346552848816, "kl": 0.3099148701876402, "learning_rate": 4.050911756809457e-05, "loss": 0.0438, "step": 408, "step_time": 3.6765331789997617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2191.0, "completions/max_terminated_length": 2191.0, "completions/mean_length": 457.96875, "completions/mean_terminated_length": 457.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.44232158549129963, "epoch": 0.00409, "frac_reward_zero_std": 0.25, "grad_norm": 0.04407849907875061, "kl": 0.3175376821309328, "learning_rate": 4.04624202103852e-05, "loss": -0.0078, "num_tokens": 8683081.0, "reward": 0.8687988519668579, "reward_std": 0.371092826128006, "rewards/rollout_reward_func/mean": 0.8687988519668579, "rewards/rollout_reward_func/std": 0.43501797318458557, "sampling/importance_sampling_ratio/max": 1.0446006059646606, "sampling/importance_sampling_ratio/mean": 0.9922645092010498, "sampling/importance_sampling_ratio/min": 0.8950773477554321, "sampling/sampling_logp_difference/max": 0.11289417743682861, "sampling/sampling_logp_difference/mean": 0.008806375786662102, "step": 409, "step_time": 10.961928822000118 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.43451330438256264, "epoch": 0.0041, "grad_norm": 0.047486502677202225, "kl": 0.32394919730722904, "learning_rate": 4.041562787899075e-05, "loss": -0.0075, "step": 410, "step_time": 6.590814913999111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 496.46875, "completions/mean_terminated_length": 496.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5611784607172012, "epoch": 0.00411, "frac_reward_zero_std": 0.5, "grad_norm": 0.1031927764415741, "kl": 0.34185378812253475, "learning_rate": 4.0368740935585195e-05, "loss": -0.0064, "num_tokens": 8727754.0, "reward": 0.8437134027481079, "reward_std": 0.35873275995254517, "rewards/rollout_reward_func/mean": 0.8437134027481079, "rewards/rollout_reward_func/std": 0.5103022456169128, "sampling/importance_sampling_ratio/max": 1.1122218370437622, "sampling/importance_sampling_ratio/mean": 0.9924900531768799, "sampling/importance_sampling_ratio/min": 0.844445526599884, "sampling/sampling_logp_difference/max": 0.09919309616088867, "sampling/sampling_logp_difference/mean": 0.00881202332675457, "step": 411, "step_time": 10.168036535999818 }, { "clip_ratio/high_max": 0.019078947603702545, "clip_ratio/high_mean": 0.009539473801851273, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009539473801851273, "entropy": 0.5625748112797737, "epoch": 0.00412, "grad_norm": 0.07820095121860504, "kl": 0.33979424461722374, "learning_rate": 4.0321759742573814e-05, "loss": -0.0072, "step": 412, "step_time": 5.241471681000348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1535.0, "completions/max_terminated_length": 1535.0, "completions/mean_length": 427.9375, "completions/mean_terminated_length": 427.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5316348746418953, "epoch": 0.00413, "frac_reward_zero_std": 0.25, "grad_norm": 0.18993131816387177, "kl": 0.4083297476172447, "learning_rate": 4.0274684663090356e-05, "loss": -0.0254, "num_tokens": 8769693.0, "reward": 0.7749878764152527, "reward_std": 0.48553675413131714, "rewards/rollout_reward_func/mean": 0.7749878764152527, "rewards/rollout_reward_func/std": 0.5585291981697083, "sampling/importance_sampling_ratio/max": 1.20109224319458, "sampling/importance_sampling_ratio/mean": 0.9986929893493652, "sampling/importance_sampling_ratio/min": 0.8557447195053101, "sampling/sampling_logp_difference/max": 0.25325632095336914, "sampling/sampling_logp_difference/mean": 0.017935721203684807, "step": 413, "step_time": 9.464754399999947 }, { "clip_ratio/high_max": 0.008928571827709675, "clip_ratio/high_mean": 0.004464285913854837, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.02083333395421505, "clip_ratio/region_mean": 0.025297619868069887, "entropy": 0.5399261899292469, "epoch": 0.00414, "grad_norm": 0.13499051332473755, "kl": 0.4004916362464428, "learning_rate": 4.022751606099426e-05, "loss": -0.0273, "step": 414, "step_time": 5.0181391740002255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 471.125, "completions/mean_terminated_length": 471.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4130521323531866, "epoch": 0.00415, "frac_reward_zero_std": 0.25, "grad_norm": 0.06028808280825615, "kl": 0.2843144107609987, "learning_rate": 4.0180254300867824e-05, "loss": -0.0009, "num_tokens": 8812681.0, "reward": 0.8455390334129333, "reward_std": 0.4368816018104553, "rewards/rollout_reward_func/mean": 0.8455390334129333, "rewards/rollout_reward_func/std": 0.5037761926651001, "sampling/importance_sampling_ratio/max": 1.0959382057189941, "sampling/importance_sampling_ratio/mean": 1.0021746158599854, "sampling/importance_sampling_ratio/min": 0.9194827675819397, "sampling/sampling_logp_difference/max": 0.09062647819519043, "sampling/sampling_logp_difference/mean": 0.006760142743587494, "step": 415, "step_time": 9.040656114000285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.41858328878879547, "epoch": 0.00416, "grad_norm": 0.061630185693502426, "kl": 0.2859442289918661, "learning_rate": 4.01328997480134e-05, "loss": -0.0009, "step": 416, "step_time": 5.41066134599987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2371.0, "completions/max_terminated_length": 2371.0, "completions/mean_length": 621.9375, "completions/mean_terminated_length": 621.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4885246679186821, "epoch": 0.00417, "frac_reward_zero_std": 0.25, "grad_norm": 0.08329865336418152, "kl": 0.3028695024549961, "learning_rate": 4.008545276845057e-05, "loss": 0.0045, "num_tokens": 8862017.0, "reward": 0.813647449016571, "reward_std": 0.44948142766952515, "rewards/rollout_reward_func/mean": 0.813647449016571, "rewards/rollout_reward_func/std": 0.5290377140045166, "sampling/importance_sampling_ratio/max": 1.1821786165237427, "sampling/importance_sampling_ratio/mean": 0.9900407791137695, "sampling/importance_sampling_ratio/min": 0.9038989543914795, "sampling/sampling_logp_difference/max": 0.1208033561706543, "sampling/sampling_logp_difference/mean": 0.007969372905790806, "step": 417, "step_time": 12.174188406999292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.5068586654961109, "epoch": 0.00418, "grad_norm": 0.07077313214540482, "kl": 0.31447278894484043, "learning_rate": 4.0037913728913316e-05, "loss": 0.004, "step": 418, "step_time": 7.196217683999748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2128.0, "completions/max_terminated_length": 2128.0, "completions/mean_length": 565.75, "completions/mean_terminated_length": 556.2257690429688, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6323834545910358, "epoch": 0.00419, "frac_reward_zero_std": 0.5, "grad_norm": 0.6357153654098511, "kl": 0.40867386385798454, "learning_rate": 3.999028299684717e-05, "loss": 0.0742, "num_tokens": 8909254.0, "reward": 0.788351833820343, "reward_std": 0.3379203677177429, "rewards/rollout_reward_func/mean": 0.788351833820343, "rewards/rollout_reward_func/std": 0.5866556167602539, "sampling/importance_sampling_ratio/max": 1.1721258163452148, "sampling/importance_sampling_ratio/mean": 1.0068615674972534, "sampling/importance_sampling_ratio/min": 0.9164859056472778, "sampling/sampling_logp_difference/max": 0.14374256134033203, "sampling/sampling_logp_difference/mean": 0.014018390327692032, "step": 419, "step_time": 11.598465488000784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03145732032135129, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03145732032135129, "entropy": 0.6583252511918545, "epoch": 0.0042, "grad_norm": 0.27295878529548645, "kl": 0.4619406070560217, "learning_rate": 3.994256094040641e-05, "loss": 0.0669, "step": 420, "step_time": 6.182748077999804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1328.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 380.125, "completions/mean_terminated_length": 347.5517272949219, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1674706861376762, "epoch": 0.00421, "frac_reward_zero_std": 0.0, "grad_norm": 0.7664837837219238, "kl": 0.36417091079056263, "learning_rate": 3.9894747928451155e-05, "loss": 0.123, "num_tokens": 8949527.0, "reward": 0.4399344325065613, "reward_std": 0.8846874237060547, "rewards/rollout_reward_func/mean": 0.4399344325065613, "rewards/rollout_reward_func/std": 0.8675304651260376, "sampling/importance_sampling_ratio/max": 1.6735560894012451, "sampling/importance_sampling_ratio/mean": 0.9935944080352783, "sampling/importance_sampling_ratio/min": 0.4237861633300781, "sampling/sampling_logp_difference/max": 0.38243627548217773, "sampling/sampling_logp_difference/mean": 0.0320887565612793, "step": 421, "step_time": 9.138083714999993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.039810363901779056, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.039810363901779056, "entropy": 1.1941568721085787, "epoch": 0.00422, "grad_norm": 0.41534170508384705, "kl": 0.4071674682199955, "learning_rate": 3.9846844330544606e-05, "loss": 0.111, "step": 422, "step_time": 4.624446342999363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1362.0, "completions/max_terminated_length": 1362.0, "completions/mean_length": 427.8125, "completions/mean_terminated_length": 418.4838562011719, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.691256508231163, "epoch": 0.00423, "frac_reward_zero_std": 0.25, "grad_norm": 0.13096113502979279, "kl": 0.3204767778515816, "learning_rate": 3.9798850516950075e-05, "loss": -0.0129, "num_tokens": 8992498.0, "reward": 0.7209492921829224, "reward_std": 0.522672712802887, "rewards/rollout_reward_func/mean": 0.7209492921829224, "rewards/rollout_reward_func/std": 0.6726089119911194, "sampling/importance_sampling_ratio/max": 1.137343168258667, "sampling/importance_sampling_ratio/mean": 0.9931503534317017, "sampling/importance_sampling_ratio/min": 0.5922079086303711, "sampling/sampling_logp_difference/max": 0.296183705329895, "sampling/sampling_logp_difference/mean": 0.015551251359283924, "step": 423, "step_time": 9.759817981999731 }, { "clip_ratio/high_max": 0.031250000931322575, "clip_ratio/high_mean": 0.015625000465661287, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625000465661287, "entropy": 0.6818907149136066, "epoch": 0.00424, "grad_norm": 0.10639820992946625, "kl": 0.32660518772900105, "learning_rate": 3.975076685862821e-05, "loss": -0.0138, "step": 424, "step_time": 5.3184364929993535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1569.0, "completions/max_terminated_length": 1460.0, "completions/mean_length": 457.875, "completions/mean_terminated_length": 407.70001220703125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7869818229228258, "epoch": 0.00425, "frac_reward_zero_std": 0.25, "grad_norm": 0.34031882882118225, "kl": 0.7246896848082542, "learning_rate": 3.970259372723411e-05, "loss": -0.0492, "num_tokens": 9035238.0, "reward": 0.6294255256652832, "reward_std": 0.6414997577667236, "rewards/rollout_reward_func/mean": 0.6294255256652832, "rewards/rollout_reward_func/std": 0.7389188408851624, "sampling/importance_sampling_ratio/max": 1.2774693965911865, "sampling/importance_sampling_ratio/mean": 1.0037872791290283, "sampling/importance_sampling_ratio/min": 0.3939371109008789, "sampling/sampling_logp_difference/max": 0.33438634872436523, "sampling/sampling_logp_difference/mean": 0.031679876148700714, "step": 425, "step_time": 9.799073169000621 }, { "clip_ratio/high_max": 0.056299603544175625, "clip_ratio/high_mean": 0.028149801772087812, "clip_ratio/low_mean": 0.019097222946584225, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04724702564999461, "entropy": 0.7790105938911438, "epoch": 0.00426, "grad_norm": 0.2550952434539795, "kl": 0.7523721233010292, "learning_rate": 3.965433149511443e-05, "loss": -0.0561, "step": 426, "step_time": 5.118650550000439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1763.0, "completions/max_terminated_length": 1763.0, "completions/mean_length": 330.25, "completions/mean_terminated_length": 263.5666809082031, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6188713344745338, "epoch": 0.00427, "frac_reward_zero_std": 0.25, "grad_norm": 0.35352540016174316, "kl": 0.7319492176175117, "learning_rate": 3.960598053530449e-05, "loss": -0.0026, "num_tokens": 9072595.0, "reward": 0.8683192133903503, "reward_std": 0.3724495768547058, "rewards/rollout_reward_func/mean": 0.8683192133903503, "rewards/rollout_reward_func/std": 0.434596985578537, "sampling/importance_sampling_ratio/max": 1.1816773414611816, "sampling/importance_sampling_ratio/mean": 0.9978951215744019, "sampling/importance_sampling_ratio/min": 0.6604537963867188, "sampling/sampling_logp_difference/max": 0.235687255859375, "sampling/sampling_logp_difference/mean": 0.02654026448726654, "step": 427, "step_time": 9.94919064700025 }, { "clip_ratio/high_max": 0.0071428571827709675, "clip_ratio/high_mean": 0.004490546300075948, "clip_ratio/low_mean": 0.008928571827709675, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013419118011370301, "entropy": 0.6160035808570683, "epoch": 0.00428, "grad_norm": 0.2735897898674011, "kl": 0.7198671847581863, "learning_rate": 3.9557541221525476e-05, "loss": -0.0085, "step": 428, "step_time": 5.33350152300045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1569.0, "completions/max_terminated_length": 1292.0, "completions/mean_length": 361.75, "completions/mean_terminated_length": 290.03448486328125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.617784418631345, "epoch": 0.00429, "frac_reward_zero_std": 0.25, "grad_norm": 0.6073207855224609, "kl": 0.5806362852454185, "learning_rate": 3.9509013928181434e-05, "loss": 0.0848, "num_tokens": 9111062.0, "reward": 0.6871992349624634, "reward_std": 0.5972298383712769, "rewards/rollout_reward_func/mean": 0.6871992349624634, "rewards/rollout_reward_func/std": 0.6851840615272522, "sampling/importance_sampling_ratio/max": 1.0311706066131592, "sampling/importance_sampling_ratio/mean": 0.9797574877738953, "sampling/importance_sampling_ratio/min": 0.6313051581382751, "sampling/sampling_logp_difference/max": 0.2703840732574463, "sampling/sampling_logp_difference/mean": 0.018333567306399345, "step": 429, "step_time": 10.131091265000123 }, { "clip_ratio/high_max": 0.0069444444961845875, "clip_ratio/high_mean": 0.0034722222480922937, "clip_ratio/low_mean": 0.01930147036910057, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.022773692617192864, "entropy": 0.6113565536215901, "epoch": 0.0043, "grad_norm": 0.34583377838134766, "kl": 0.5921460725367069, "learning_rate": 3.9460399030356465e-05, "loss": 0.0701, "step": 430, "step_time": 5.559874318000311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 2193.0, "completions/max_terminated_length": 1713.0, "completions/mean_length": 616.25, "completions/mean_terminated_length": 481.4285888671875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7237390764057636, "epoch": 0.00431, "frac_reward_zero_std": 0.25, "grad_norm": 0.5052527785301208, "kl": 0.49842604249715805, "learning_rate": 3.9411696903811785e-05, "loss": 0.0923, "num_tokens": 9159253.0, "reward": 0.6870982050895691, "reward_std": 0.49878090620040894, "rewards/rollout_reward_func/mean": 0.6870982050895691, "rewards/rollout_reward_func/std": 0.6365674138069153, "sampling/importance_sampling_ratio/max": 1.4796446561813354, "sampling/importance_sampling_ratio/mean": 1.0035479068756104, "sampling/importance_sampling_ratio/min": 0.5520172715187073, "sampling/sampling_logp_difference/max": 0.2616100311279297, "sampling/sampling_logp_difference/mean": 0.028299886733293533, "step": 431, "step_time": 12.771567363999566 }, { "clip_ratio/high_max": 0.023215174209326506, "clip_ratio/high_mean": 0.011607587104663253, "clip_ratio/low_mean": 0.025163975078612566, "clip_ratio/low_min": 0.013297872617840767, "clip_ratio/region_mean": 0.03677156218327582, "entropy": 0.7146476320922375, "epoch": 0.00432, "grad_norm": 0.30248773097991943, "kl": 0.500138383358717, "learning_rate": 3.936290792498284e-05, "loss": 0.0828, "step": 432, "step_time": 6.20732506000013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2255.0, "completions/max_terminated_length": 2255.0, "completions/mean_length": 516.34375, "completions/mean_terminated_length": 475.4838562011719, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.775003120303154, "epoch": 0.00433, "frac_reward_zero_std": 0.25, "grad_norm": 0.6463140845298767, "kl": 0.41445986554026604, "learning_rate": 3.9314032470976384e-05, "loss": 0.1499, "num_tokens": 9205616.0, "reward": 0.5342856645584106, "reward_std": 0.667442798614502, "rewards/rollout_reward_func/mean": 0.5342856645584106, "rewards/rollout_reward_func/std": 0.788051426410675, "sampling/importance_sampling_ratio/max": 1.6366416215896606, "sampling/importance_sampling_ratio/mean": 1.0086488723754883, "sampling/importance_sampling_ratio/min": 0.5927960276603699, "sampling/sampling_logp_difference/max": 0.21361565589904785, "sampling/sampling_logp_difference/mean": 0.02608613297343254, "step": 433, "step_time": 12.318891019999228 }, { "clip_ratio/high_max": 0.009999999776482582, "clip_ratio/high_mean": 0.009424778632819653, "clip_ratio/low_mean": 0.021764458622783422, "clip_ratio/low_min": 0.005514706019312143, "clip_ratio/region_mean": 0.031189236789941788, "entropy": 0.7672653757035732, "epoch": 0.00434, "grad_norm": 0.4628550112247467, "kl": 0.42995897121727467, "learning_rate": 3.9265070919567546e-05, "loss": 0.135, "step": 434, "step_time": 6.409730055000637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2101.0, "completions/max_terminated_length": 2101.0, "completions/mean_length": 588.96875, "completions/mean_terminated_length": 555.0967407226562, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6887617260217667, "epoch": 0.00435, "frac_reward_zero_std": 0.25, "grad_norm": 0.5607677698135376, "kl": 0.9536722972989082, "learning_rate": 3.9216023649196966e-05, "loss": 0.122, "num_tokens": 9253362.0, "reward": 0.5980343818664551, "reward_std": 0.634412407875061, "rewards/rollout_reward_func/mean": 0.5980343818664551, "rewards/rollout_reward_func/std": 0.7442587018013, "sampling/importance_sampling_ratio/max": 1.7937744855880737, "sampling/importance_sampling_ratio/mean": 1.014878273010254, "sampling/importance_sampling_ratio/min": 0.7538774013519287, "sampling/sampling_logp_difference/max": 0.1400604248046875, "sampling/sampling_logp_difference/mean": 0.017403703182935715, "step": 435, "step_time": 11.949421984999844 }, { "clip_ratio/high_max": 0.005434782709926367, "clip_ratio/high_mean": 0.0027173913549631834, "clip_ratio/low_mean": 0.0407825147267431, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04349990584887564, "entropy": 0.6788029074668884, "epoch": 0.00436, "grad_norm": 0.3010694980621338, "kl": 0.9009863808751106, "learning_rate": 3.916689103896781e-05, "loss": 0.1133, "step": 436, "step_time": 6.840248767000503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 493.21875, "completions/mean_terminated_length": 482.51611328125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.44817981868982315, "epoch": 0.00437, "frac_reward_zero_std": 0.0, "grad_norm": 0.4188665449619293, "kl": 0.4141702465713024, "learning_rate": 3.9117673468642885e-05, "loss": 0.0486, "num_tokens": 9297968.0, "reward": 0.6286689043045044, "reward_std": 0.6820645332336426, "rewards/rollout_reward_func/mean": 0.6286689043045044, "rewards/rollout_reward_func/std": 0.7341936826705933, "sampling/importance_sampling_ratio/max": 1.1248565912246704, "sampling/importance_sampling_ratio/mean": 0.9944576025009155, "sampling/importance_sampling_ratio/min": 0.8664015531539917, "sampling/sampling_logp_difference/max": 0.1701526641845703, "sampling/sampling_logp_difference/mean": 0.016508255153894424, "step": 437, "step_time": 9.351461339999332 }, { "clip_ratio/high_max": 0.02500000037252903, "clip_ratio/high_mean": 0.012500000186264515, "clip_ratio/low_mean": 0.02302631549537182, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03552631568163633, "entropy": 0.449510607868433, "epoch": 0.00438, "grad_norm": 0.20624765753746033, "kl": 0.3952536918222904, "learning_rate": 3.9068371318641666e-05, "loss": 0.0431, "step": 438, "step_time": 5.400850549001007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 2497.0, "completions/max_terminated_length": 1485.0, "completions/mean_length": 519.65625, "completions/mean_terminated_length": 422.2333679199219, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7612930089235306, "epoch": 0.00439, "frac_reward_zero_std": 0.0, "grad_norm": 0.5924277901649475, "kl": 0.4853454567492008, "learning_rate": 3.901898497003737e-05, "loss": -0.1174, "num_tokens": 9343617.0, "reward": 0.4430018365383148, "reward_std": 0.9033223390579224, "rewards/rollout_reward_func/mean": 0.4430018365383148, "rewards/rollout_reward_func/std": 0.8613525032997131, "sampling/importance_sampling_ratio/max": 1.5103895664215088, "sampling/importance_sampling_ratio/mean": 1.0310665369033813, "sampling/importance_sampling_ratio/min": 0.7898485064506531, "sampling/sampling_logp_difference/max": 0.17703723907470703, "sampling/sampling_logp_difference/mean": 0.02401166781783104, "step": 439, "step_time": 12.966801169999599 }, { "clip_ratio/high_max": 0.0662530642002821, "clip_ratio/high_mean": 0.03312653210014105, "clip_ratio/low_mean": 0.011363636702299118, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.044490168802440166, "entropy": 0.778548426926136, "epoch": 0.0044, "grad_norm": 0.4461935758590698, "kl": 0.4848218411207199, "learning_rate": 3.896951480455404e-05, "loss": -0.1315, "step": 440, "step_time": 6.829145051999149 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1515.0, "completions/max_terminated_length": 1395.0, "completions/mean_length": 526.3125, "completions/mean_terminated_length": 494.4193420410156, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8120886832475662, "epoch": 0.00441, "frac_reward_zero_std": 0.5, "grad_norm": 0.11419082432985306, "kl": 0.397040281444788, "learning_rate": 3.8919961204563525e-05, "loss": 0.0067, "num_tokens": 9389893.0, "reward": 0.782626211643219, "reward_std": 0.4112010598182678, "rewards/rollout_reward_func/mean": 0.782626211643219, "rewards/rollout_reward_func/std": 0.5986263155937195, "sampling/importance_sampling_ratio/max": 1.2733439207077026, "sampling/importance_sampling_ratio/mean": 1.0017342567443848, "sampling/importance_sampling_ratio/min": 0.5117787718772888, "sampling/sampling_logp_difference/max": 0.24714183807373047, "sampling/sampling_logp_difference/mean": 0.02754221111536026, "step": 441, "step_time": 10.086894578002102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.8461352325975895, "epoch": 0.00442, "grad_norm": 0.08889985084533691, "kl": 0.39744686521589756, "learning_rate": 3.887032455308262e-05, "loss": 0.0055, "step": 442, "step_time": 5.102252286000294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1346.0, "completions/max_terminated_length": 1346.0, "completions/mean_length": 514.6875, "completions/mean_terminated_length": 503.8064270019531, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7164932899177074, "epoch": 0.00443, "frac_reward_zero_std": 0.25, "grad_norm": 0.47923052310943604, "kl": 0.5218353010714054, "learning_rate": 3.8820605233770034e-05, "loss": 0.0529, "num_tokens": 9435080.0, "reward": 0.6826307773590088, "reward_std": 0.5425764322280884, "rewards/rollout_reward_func/mean": 0.6826307773590088, "rewards/rollout_reward_func/std": 0.6418085694313049, "sampling/importance_sampling_ratio/max": 1.1360286474227905, "sampling/importance_sampling_ratio/mean": 0.9886323809623718, "sampling/importance_sampling_ratio/min": 0.7796914577484131, "sampling/sampling_logp_difference/max": 0.21608591079711914, "sampling/sampling_logp_difference/mean": 0.023869294673204422, "step": 443, "step_time": 9.90990811300071 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.0373263880610466, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04357638955116272, "entropy": 0.7709286138415337, "epoch": 0.00444, "grad_norm": 0.26149868965148926, "kl": 0.5516389608383179, "learning_rate": 3.877080363092342e-05, "loss": 0.0455, "step": 444, "step_time": 4.807577876000323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 2153.0, "completions/max_terminated_length": 1359.0, "completions/mean_length": 498.28125, "completions/mean_terminated_length": 376.21429443359375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3396442532539368, "epoch": 0.00445, "frac_reward_zero_std": 0.0, "grad_norm": 0.9089914560317993, "kl": 0.687832809984684, "learning_rate": 3.8720920129476466e-05, "loss": 0.0625, "num_tokens": 9479221.0, "reward": 0.6785304546356201, "reward_std": 0.6081631183624268, "rewards/rollout_reward_func/mean": 0.6785304546356201, "rewards/rollout_reward_func/std": 0.6542966365814209, "sampling/importance_sampling_ratio/max": 1.6463688611984253, "sampling/importance_sampling_ratio/mean": 0.9985048174858093, "sampling/importance_sampling_ratio/min": 1.713856487284282e-11, "sampling/sampling_logp_difference/max": 14.664700508117676, "sampling/sampling_logp_difference/mean": 0.10124332457780838, "step": 445, "step_time": 12.543712220000089 }, { "clip_ratio/high_max": 0.03740490926429629, "clip_ratio/high_mean": 0.018702454632148147, "clip_ratio/low_mean": 0.017973856534808874, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03667631116695702, "entropy": 1.3819062560796738, "epoch": 0.00446, "grad_norm": 0.6510471701622009, "kl": 0.6801687963306904, "learning_rate": 3.8670955114995835e-05, "loss": 0.041, "step": 446, "step_time": 6.142647737000516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 1441.0, "completions/max_terminated_length": 1441.0, "completions/mean_length": 486.03125, "completions/mean_terminated_length": 383.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.5354949533939362, "epoch": 0.00447, "frac_reward_zero_std": 0.0, "grad_norm": 0.7881168127059937, "kl": 0.787871066480875, "learning_rate": 3.862090897367828e-05, "loss": 0.0896, "num_tokens": 9523397.0, "reward": 0.6504464745521545, "reward_std": 0.63535475730896, "rewards/rollout_reward_func/mean": 0.6504464745521545, "rewards/rollout_reward_func/std": 0.7051829695701599, "sampling/importance_sampling_ratio/max": 1.2266960144042969, "sampling/importance_sampling_ratio/mean": 0.9285136461257935, "sampling/importance_sampling_ratio/min": 5.333894785252852e-19, "sampling/sampling_logp_difference/max": 17.86741828918457, "sampling/sampling_logp_difference/mean": 0.19462639093399048, "step": 447, "step_time": 10.267027798001436 }, { "clip_ratio/high_max": 0.019290497759357095, "clip_ratio/high_mean": 0.013376591843552887, "clip_ratio/low_mean": 0.034337913850322366, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.047714504529722035, "entropy": 1.5575924292206764, "epoch": 0.00448, "grad_norm": 0.5269476771354675, "kl": 0.7615318465977907, "learning_rate": 3.857078209234759e-05, "loss": 0.072, "step": 448, "step_time": 4.847990880999532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 334.0625, "completions/mean_terminated_length": 320.5483703613281, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.858503982424736, "epoch": 0.00449, "frac_reward_zero_std": 0.5, "grad_norm": 4.938475131988525, "kl": 21.415592132136226, "learning_rate": 3.8520574858451616e-05, "loss": 0.1016, "num_tokens": 9560936.0, "reward": 0.8705825805664062, "reward_std": 0.281859815120697, "rewards/rollout_reward_func/mean": 0.8705825805664062, "rewards/rollout_reward_func/std": 0.4286879003047943, "sampling/importance_sampling_ratio/max": 1.095273494720459, "sampling/importance_sampling_ratio/mean": 0.965221107006073, "sampling/importance_sampling_ratio/min": 0.5582570433616638, "sampling/sampling_logp_difference/max": 0.2190532684326172, "sampling/sampling_logp_difference/mean": 0.022545209154486656, "step": 449, "step_time": 9.157616808999592 }, { "clip_ratio/high_max": 0.0069444444961845875, "clip_ratio/high_mean": 0.0034722222480922937, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034722222480922937, "entropy": 0.8774667121469975, "epoch": 0.0045, "grad_norm": 0.6333616375923157, "kl": 2.374910404905677, "learning_rate": 3.84702876600593e-05, "loss": 0.0287, "step": 450, "step_time": 5.240094436000618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1484.0, "completions/max_terminated_length": 1484.0, "completions/mean_length": 495.34375, "completions/mean_terminated_length": 453.0666809082031, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3805782571434975, "epoch": 0.00451, "frac_reward_zero_std": 0.25, "grad_norm": 0.6407269835472107, "kl": 0.5974510908126831, "learning_rate": 3.841992088585763e-05, "loss": 0.0018, "num_tokens": 9604829.0, "reward": 0.7463973164558411, "reward_std": 0.5117801427841187, "rewards/rollout_reward_func/mean": 0.7463973164558411, "rewards/rollout_reward_func/std": 0.6264862418174744, "sampling/importance_sampling_ratio/max": 2.262645959854126, "sampling/importance_sampling_ratio/mean": 0.9918235540390015, "sampling/importance_sampling_ratio/min": 0.4139770269393921, "sampling/sampling_logp_difference/max": 0.2761549949645996, "sampling/sampling_logp_difference/mean": 0.02844393439590931, "step": 451, "step_time": 10.305366427999616 }, { "clip_ratio/high_max": 0.03327205963432789, "clip_ratio/high_mean": 0.016636029817163944, "clip_ratio/low_mean": 0.0205543152987957, "clip_ratio/low_min": 0.0059523810632526875, "clip_ratio/region_mean": 0.037190345115959644, "entropy": 1.3870307505130768, "epoch": 0.00452, "grad_norm": 0.6760247945785522, "kl": 0.5739407800137997, "learning_rate": 3.8369474925148706e-05, "loss": -0.0025, "step": 452, "step_time": 4.984406611999475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1565.0, "completions/max_terminated_length": 1390.0, "completions/mean_length": 426.9375, "completions/mean_terminated_length": 373.4666748046875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.214351199567318, "epoch": 0.00453, "frac_reward_zero_std": 0.25, "grad_norm": 0.30370408296585083, "kl": 0.7088330276310444, "learning_rate": 3.831895016784664e-05, "loss": 0.0448, "num_tokens": 9646542.0, "reward": 0.5902712345123291, "reward_std": 0.5872074365615845, "rewards/rollout_reward_func/mean": 0.5902712345123291, "rewards/rollout_reward_func/std": 0.7532962560653687, "sampling/importance_sampling_ratio/max": 1.1550109386444092, "sampling/importance_sampling_ratio/mean": 0.9750118255615234, "sampling/importance_sampling_ratio/min": 7.547195908933825e-21, "sampling/sampling_logp_difference/max": 20.8245849609375, "sampling/sampling_logp_difference/mean": 0.21912896633148193, "step": 453, "step_time": 9.56126573200072 }, { "clip_ratio/high_max": 0.003968254197388887, "clip_ratio/high_mean": 0.0019841270986944437, "clip_ratio/low_mean": 0.005434782709926367, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0074189098086208105, "entropy": 1.223532274365425, "epoch": 0.00454, "grad_norm": 0.24873875081539154, "kl": 0.6858462058007717, "learning_rate": 3.826834700447459e-05, "loss": 0.0402, "step": 454, "step_time": 5.016719011000532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1695.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 524.46875, "completions/mean_terminated_length": 467.6551818847656, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.5669775754213333, "epoch": 0.00455, "frac_reward_zero_std": 0.0, "grad_norm": 0.6875037550926208, "kl": 1.1661696191877127, "learning_rate": 3.821766582616181e-05, "loss": 0.0315, "num_tokens": 9692245.0, "reward": 0.4047694802284241, "reward_std": 0.8787835836410522, "rewards/rollout_reward_func/mean": 0.4047694802284241, "rewards/rollout_reward_func/std": 0.8711251020431519, "sampling/importance_sampling_ratio/max": 1.6462332010269165, "sampling/importance_sampling_ratio/mean": 0.9627418518066406, "sampling/importance_sampling_ratio/min": 1.3104529736200111e-14, "sampling/sampling_logp_difference/max": 17.800901412963867, "sampling/sampling_logp_difference/mean": 0.13874788582324982, "step": 455, "step_time": 10.855132805997528 }, { "clip_ratio/high_max": 0.02959558879956603, "clip_ratio/high_mean": 0.015435549663379788, "clip_ratio/low_mean": 0.034109477419406176, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04954502684995532, "entropy": 1.5736504048109055, "epoch": 0.00456, "grad_norm": 0.4192183315753937, "kl": 1.078044280409813, "learning_rate": 3.8166907024640474e-05, "loss": 0.02, "step": 456, "step_time": 5.288144027000271 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 2275.0, "completions/max_terminated_length": 2182.0, "completions/mean_length": 545.625, "completions/mean_terminated_length": 458.13336181640625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.122809462249279, "epoch": 0.00457, "frac_reward_zero_std": 0.25, "grad_norm": 0.5375316739082336, "kl": 0.5882059931755066, "learning_rate": 3.811607099224277e-05, "loss": 0.0859, "num_tokens": 9738032.0, "reward": 0.5058398246765137, "reward_std": 0.6888864040374756, "rewards/rollout_reward_func/mean": 0.5058398246765137, "rewards/rollout_reward_func/std": 0.8238844871520996, "sampling/importance_sampling_ratio/max": 1.1222752332687378, "sampling/importance_sampling_ratio/mean": 0.9842860698699951, "sampling/importance_sampling_ratio/min": 0.6343880295753479, "sampling/sampling_logp_difference/max": 0.24544620513916016, "sampling/sampling_logp_difference/mean": 0.024616260081529617, "step": 457, "step_time": 12.31547042299917 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.05624999990686774, "clip_ratio/low_min": 0.03645833395421505, "clip_ratio/region_mean": 0.06666666688397527, "entropy": 1.094105139374733, "epoch": 0.00458, "grad_norm": 0.2954750061035156, "kl": 0.5987138971686363, "learning_rate": 3.806515812189784e-05, "loss": 0.0755, "step": 458, "step_time": 6.937802001000819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1582.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 423.28125, "completions/mean_terminated_length": 385.9032287597656, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.093129899352789, "epoch": 0.00459, "frac_reward_zero_std": 0.0, "grad_norm": 1.030273675918579, "kl": 0.5537241250276566, "learning_rate": 3.801416880712871e-05, "loss": 0.0604, "num_tokens": 9780170.0, "reward": 0.7124940156936646, "reward_std": 0.6642521619796753, "rewards/rollout_reward_func/mean": 0.7124940156936646, "rewards/rollout_reward_func/std": 0.6407222151756287, "sampling/importance_sampling_ratio/max": 1.6744110584259033, "sampling/importance_sampling_ratio/mean": 0.9825272560119629, "sampling/importance_sampling_ratio/min": 2.663660779944621e-05, "sampling/sampling_logp_difference/max": 4.027030944824219, "sampling/sampling_logp_difference/mean": 0.08052331209182739, "step": 459, "step_time": 10.046790912000688 }, { "clip_ratio/high_max": 0.012106537818908691, "clip_ratio/high_mean": 0.006053268909454346, "clip_ratio/low_mean": 0.056175597012043, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.062228865921497345, "entropy": 1.0732331685721874, "epoch": 0.0046, "grad_norm": 0.29261037707328796, "kl": 0.5964209847152233, "learning_rate": 3.7963103442049294e-05, "loss": 0.0486, "step": 460, "step_time": 5.072303401998397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 2326.0, "completions/max_terminated_length": 2326.0, "completions/mean_length": 410.21875, "completions/mean_terminated_length": 400.9000244140625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0436114892363548, "epoch": 0.00461, "frac_reward_zero_std": 0.25, "grad_norm": 0.4768045246601105, "kl": 0.449652461335063, "learning_rate": 3.79119624213613e-05, "loss": 0.0223, "num_tokens": 9821703.0, "reward": 0.7665736675262451, "reward_std": 0.4182940125465393, "rewards/rollout_reward_func/mean": 0.7665736675262451, "rewards/rollout_reward_func/std": 0.5140154361724854, "sampling/importance_sampling_ratio/max": 1.262930154800415, "sampling/importance_sampling_ratio/mean": 1.008344054222107, "sampling/importance_sampling_ratio/min": 0.7482228875160217, "sampling/sampling_logp_difference/max": 0.19679975509643555, "sampling/sampling_logp_difference/mean": 0.021031880751252174, "step": 461, "step_time": 11.834759370999564 }, { "clip_ratio/high_max": 0.01605392200872302, "clip_ratio/high_mean": 0.00802696100436151, "clip_ratio/low_mean": 0.01809210516512394, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02611906616948545, "entropy": 1.0229994840919971, "epoch": 0.00462, "grad_norm": 0.2823757231235504, "kl": 0.46054428815841675, "learning_rate": 3.7860746140351226e-05, "loss": 0.0146, "step": 462, "step_time": 6.308760343001268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 2378.0, "completions/max_terminated_length": 2378.0, "completions/mean_length": 530.125, "completions/mean_terminated_length": 531.1034545898438, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1973317191004753, "epoch": 0.00463, "frac_reward_zero_std": 0.5, "grad_norm": 0.6766795516014099, "kl": 0.7389031872153282, "learning_rate": 3.780945499488727e-05, "loss": 0.0413, "num_tokens": 9867079.0, "reward": 0.8698341250419617, "reward_std": 0.2998374402523041, "rewards/rollout_reward_func/mean": 0.8698341250419617, "rewards/rollout_reward_func/std": 0.4278963804244995, "sampling/importance_sampling_ratio/max": 1.0496039390563965, "sampling/importance_sampling_ratio/mean": 0.9348657131195068, "sampling/importance_sampling_ratio/min": 7.410222740428924e-16, "sampling/sampling_logp_difference/max": 20.921424865722656, "sampling/sampling_logp_difference/mean": 0.16691476106643677, "step": 463, "step_time": 12.610264130998985 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.0223214291036129, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02380952425301075, "entropy": 1.1659424528479576, "epoch": 0.00464, "grad_norm": 0.24842731654644012, "kl": 0.7236576974391937, "learning_rate": 3.775808938141628e-05, "loss": 0.032, "step": 464, "step_time": 6.615663444001257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1603.0, "completions/max_terminated_length": 1548.0, "completions/mean_length": 532.84375, "completions/mean_terminated_length": 479.17242431640625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0108060911297798, "epoch": 0.00465, "frac_reward_zero_std": 0.0, "grad_norm": 0.8511902093887329, "kl": 0.6536842063069344, "learning_rate": 3.770664969696072e-05, "loss": 0.1898, "num_tokens": 9913388.0, "reward": 0.7105947732925415, "reward_std": 0.5624051690101624, "rewards/rollout_reward_func/mean": 0.7105947732925415, "rewards/rollout_reward_func/std": 0.5903745889663696, "sampling/importance_sampling_ratio/max": 2.398188591003418, "sampling/importance_sampling_ratio/mean": 1.0086796283721924, "sampling/importance_sampling_ratio/min": 7.360312988431705e-13, "sampling/sampling_logp_difference/max": 6.783562660217285, "sampling/sampling_logp_difference/mean": 0.11508473008871078, "step": 465, "step_time": 11.00953804999881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03301691776141524, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03301691776141524, "entropy": 0.9729696325957775, "epoch": 0.00466, "grad_norm": 0.5354580283164978, "kl": 0.681936664506793, "learning_rate": 3.765513633911554e-05, "loss": 0.1725, "step": 466, "step_time": 5.17786936199991 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1661.0, "completions/max_terminated_length": 1661.0, "completions/mean_length": 520.4375, "completions/mean_terminated_length": 520.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6219911873340607, "epoch": 0.00467, "frac_reward_zero_std": 0.0, "grad_norm": 0.2188957929611206, "kl": 0.6472328398376703, "learning_rate": 3.760354970604514e-05, "loss": -0.0069, "num_tokens": 9959161.0, "reward": 0.4244612157344818, "reward_std": 0.7769855260848999, "rewards/rollout_reward_func/mean": 0.4244612157344818, "rewards/rollout_reward_func/std": 0.7630940675735474, "sampling/importance_sampling_ratio/max": 1.0864322185516357, "sampling/importance_sampling_ratio/mean": 0.9854584336280823, "sampling/importance_sampling_ratio/min": 0.874225378036499, "sampling/sampling_logp_difference/max": 0.13099098205566406, "sampling/sampling_logp_difference/mean": 0.009804178029298782, "step": 467, "step_time": 9.754313990998526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.03645833395421505, "clip_ratio/low_min": 0.02083333395421505, "clip_ratio/region_mean": 0.03645833395421505, "entropy": 0.563082929700613, "epoch": 0.00468, "grad_norm": 0.10164756327867508, "kl": 0.7860602140426636, "learning_rate": 3.755189019648032e-05, "loss": -0.0086, "step": 468, "step_time": 5.270110600000407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3206.0, "completions/max_terminated_length": 3206.0, "completions/mean_length": 528.125, "completions/mean_terminated_length": 528.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5126740429550409, "epoch": 0.00469, "frac_reward_zero_std": 0.25, "grad_norm": 0.12042150646448135, "kl": 0.5964270494878292, "learning_rate": 3.7500158209715135e-05, "loss": 0.004, "num_tokens": 10004720.0, "reward": 0.8104295134544373, "reward_std": 0.45511794090270996, "rewards/rollout_reward_func/mean": 0.8104295134544373, "rewards/rollout_reward_func/std": 0.5357728600502014, "sampling/importance_sampling_ratio/max": 1.0965182781219482, "sampling/importance_sampling_ratio/mean": 0.9918382167816162, "sampling/importance_sampling_ratio/min": 0.8274417519569397, "sampling/sampling_logp_difference/max": 0.11722326278686523, "sampling/sampling_logp_difference/mean": 0.010351026430726051, "step": 469, "step_time": 13.852934088999973 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.4798568859696388, "epoch": 0.0047, "grad_norm": 0.09970849007368088, "kl": 0.6267503686249256, "learning_rate": 3.7448354145603865e-05, "loss": 0.0033, "step": 470, "step_time": 8.400388255998223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1515.0, "completions/max_terminated_length": 1515.0, "completions/mean_length": 385.59375, "completions/mean_terminated_length": 385.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5171377323567867, "epoch": 0.00471, "frac_reward_zero_std": 0.0, "grad_norm": 0.1574157178401947, "kl": 0.733577374368906, "learning_rate": 3.739647840455788e-05, "loss": -0.0089, "num_tokens": 10045681.0, "reward": 0.6434653401374817, "reward_std": 0.6578971147537231, "rewards/rollout_reward_func/mean": 0.6434653401374817, "rewards/rollout_reward_func/std": 0.662765622138977, "sampling/importance_sampling_ratio/max": 1.2740424871444702, "sampling/importance_sampling_ratio/mean": 1.0161633491516113, "sampling/importance_sampling_ratio/min": 0.9133926033973694, "sampling/sampling_logp_difference/max": 0.12415409088134766, "sampling/sampling_logp_difference/mean": 0.011702499352395535, "step": 471, "step_time": 9.177744720999726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02864583395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02864583395421505, "entropy": 0.4940875172615051, "epoch": 0.00472, "grad_norm": 0.12548820674419403, "kl": 0.8413002640008926, "learning_rate": 3.7344531387542564e-05, "loss": -0.01, "step": 472, "step_time": 5.453268938998917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 457.90625, "completions/mean_terminated_length": 471.6451416015625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6001304239034653, "epoch": 0.00473, "frac_reward_zero_std": 0.25, "grad_norm": 0.08683111518621445, "kl": 0.5429613664746284, "learning_rate": 3.729251349607425e-05, "loss": -0.0115, "num_tokens": 10089548.0, "reward": 0.783685564994812, "reward_std": 0.5306647419929504, "rewards/rollout_reward_func/mean": 0.783685564994812, "rewards/rollout_reward_func/std": 0.5982518792152405, "sampling/importance_sampling_ratio/max": 1.072777271270752, "sampling/importance_sampling_ratio/mean": 0.9953516125679016, "sampling/importance_sampling_ratio/min": 0.8949263095855713, "sampling/sampling_logp_difference/max": 0.11258363723754883, "sampling/sampling_logp_difference/mean": 0.009919467382133007, "step": 473, "step_time": 9.313935684999706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5870741158723831, "epoch": 0.00474, "grad_norm": 0.08301960676908493, "kl": 0.5766873061656952, "learning_rate": 3.7240425132217044e-05, "loss": -0.0122, "step": 474, "step_time": 4.955919628000629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 490.15625, "completions/mean_terminated_length": 490.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31833041086792946, "epoch": 0.00475, "frac_reward_zero_std": 0.75, "grad_norm": 0.14354941248893738, "kl": 0.8118269108235836, "learning_rate": 3.718826669857978e-05, "loss": 0.0027, "num_tokens": 10133333.0, "reward": 0.8393549919128418, "reward_std": 0.18864783644676208, "rewards/rollout_reward_func/mean": 0.8393549919128418, "rewards/rollout_reward_func/std": 0.4566113352775574, "sampling/importance_sampling_ratio/max": 1.0650440454483032, "sampling/importance_sampling_ratio/mean": 1.005326509475708, "sampling/importance_sampling_ratio/min": 0.9646697640419006, "sampling/sampling_logp_difference/max": 0.06259727478027344, "sampling/sampling_logp_difference/mean": 0.0032950122840702534, "step": 475, "step_time": 9.35456932499892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.3057207502424717, "epoch": 0.00476, "grad_norm": 0.027404064312577248, "kl": 0.8280913829803467, "learning_rate": 3.7136038598312885e-05, "loss": 0.0024, "step": 476, "step_time": 5.1320573880002485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2209.0, "completions/max_terminated_length": 2209.0, "completions/mean_length": 594.15625, "completions/mean_terminated_length": 594.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3448783792555332, "epoch": 0.00477, "frac_reward_zero_std": 0.0, "grad_norm": 0.16239647567272186, "kl": 0.732345562428236, "learning_rate": 3.708374123510527e-05, "loss": 0.0168, "num_tokens": 10180869.0, "reward": 0.5551204681396484, "reward_std": 0.6834776401519775, "rewards/rollout_reward_func/mean": 0.5551204681396484, "rewards/rollout_reward_func/std": 0.7159563899040222, "sampling/importance_sampling_ratio/max": 1.2107473611831665, "sampling/importance_sampling_ratio/mean": 1.0140385627746582, "sampling/importance_sampling_ratio/min": 0.9273513555526733, "sampling/sampling_logp_difference/max": 0.17542505264282227, "sampling/sampling_logp_difference/mean": 0.009475840255618095, "step": 477, "step_time": 11.797506796000562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010714286006987095, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010714286006987095, "entropy": 0.33411956019699574, "epoch": 0.00478, "grad_norm": 0.12549777328968048, "kl": 0.8119839802384377, "learning_rate": 3.703137501318117e-05, "loss": 0.0159, "step": 478, "step_time": 6.427534594999997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2128.0, "completions/max_terminated_length": 2128.0, "completions/mean_length": 473.96875, "completions/mean_terminated_length": 473.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3012659018859267, "epoch": 0.00479, "frac_reward_zero_std": 0.75, "grad_norm": 0.08269713819026947, "kl": 0.5017359852790833, "learning_rate": 3.697894033729709e-05, "loss": 0.0045, "num_tokens": 10223403.0, "reward": 0.967336893081665, "reward_std": 0.09238521754741669, "rewards/rollout_reward_func/mean": 0.967336893081665, "rewards/rollout_reward_func/std": 0.18477044999599457, "sampling/importance_sampling_ratio/max": 1.1319270133972168, "sampling/importance_sampling_ratio/mean": 1.009019136428833, "sampling/importance_sampling_ratio/min": 0.9230421781539917, "sampling/sampling_logp_difference/max": 0.14977431297302246, "sampling/sampling_logp_difference/mean": 0.005610480438917875, "step": 479, "step_time": 11.259786672999326 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.2984716733917594, "epoch": 0.0048, "grad_norm": 0.04043376445770264, "kl": 0.49343884363770485, "learning_rate": 3.692643761273861e-05, "loss": 0.0043, "step": 480, "step_time": 6.007130442000744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2218.0, "completions/max_terminated_length": 2218.0, "completions/mean_length": 466.5, "completions/mean_terminated_length": 466.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3821600992232561, "epoch": 0.00481, "frac_reward_zero_std": 0.5, "grad_norm": 0.0655553936958313, "kl": 0.42466361075639725, "learning_rate": 3.687386724531729e-05, "loss": 0.0006, "num_tokens": 10265927.0, "reward": 0.812758207321167, "reward_std": 0.347007155418396, "rewards/rollout_reward_func/mean": 0.812758207321167, "rewards/rollout_reward_func/std": 0.5281403064727783, "sampling/importance_sampling_ratio/max": 1.030918002128601, "sampling/importance_sampling_ratio/mean": 0.9922679662704468, "sampling/importance_sampling_ratio/min": 0.8156518936157227, "sampling/sampling_logp_difference/max": 0.1494278907775879, "sampling/sampling_logp_difference/mean": 0.011483888141810894, "step": 481, "step_time": 11.387103468998248 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 0.38347337394952774, "epoch": 0.00482, "grad_norm": 0.05898604542016983, "kl": 0.42921921238303185, "learning_rate": 3.6821229641367566e-05, "loss": 0.0003, "step": 482, "step_time": 6.326993283000775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2450.0, "completions/max_terminated_length": 2450.0, "completions/mean_length": 574.15625, "completions/mean_terminated_length": 564.3225708007812, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.41273708269000053, "epoch": 0.00483, "frac_reward_zero_std": 0.25, "grad_norm": 0.11220815777778625, "kl": 0.6541156321763992, "learning_rate": 3.67685252077435e-05, "loss": 0.0088, "num_tokens": 10313203.0, "reward": 0.7673109173774719, "reward_std": 0.3704186677932739, "rewards/rollout_reward_func/mean": 0.7673109173774719, "rewards/rollout_reward_func/std": 0.4474433958530426, "sampling/importance_sampling_ratio/max": 1.0116230249404907, "sampling/importance_sampling_ratio/mean": 0.9781012535095215, "sampling/importance_sampling_ratio/min": 0.6289200782775879, "sampling/sampling_logp_difference/max": 0.23468303680419922, "sampling/sampling_logp_difference/mean": 0.013235636986792088, "step": 483, "step_time": 11.822276812999917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "entropy": 0.4214457720518112, "epoch": 0.00484, "grad_norm": 0.10633428394794464, "kl": 0.6561166197061539, "learning_rate": 3.6715754351815744e-05, "loss": 0.008, "step": 484, "step_time": 7.065469060000396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1499.0, "completions/max_terminated_length": 1499.0, "completions/mean_length": 527.71875, "completions/mean_terminated_length": 535.1000366210938, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7265043295919895, "epoch": 0.00485, "frac_reward_zero_std": 0.0, "grad_norm": 0.8186985850334167, "kl": 1.4046226516366005, "learning_rate": 3.666291748146834e-05, "loss": 0.1119, "num_tokens": 10359713.0, "reward": 0.5031924843788147, "reward_std": 0.843131422996521, "rewards/rollout_reward_func/mean": 0.5031924843788147, "rewards/rollout_reward_func/std": 0.8325822353363037, "sampling/importance_sampling_ratio/max": 1.115256667137146, "sampling/importance_sampling_ratio/mean": 0.9996767044067383, "sampling/importance_sampling_ratio/min": 0.8041400909423828, "sampling/sampling_logp_difference/max": 0.25143861770629883, "sampling/sampling_logp_difference/mean": 0.015744581818580627, "step": 485, "step_time": 9.856755808999878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.01838235300965607, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01838235300965607, "entropy": 0.7156567815691233, "epoch": 0.00486, "grad_norm": 0.39733797311782837, "kl": 1.4506948068737984, "learning_rate": 3.6610015005095585e-05, "loss": 0.102, "step": 486, "step_time": 5.552538841000569 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 447.8125, "completions/mean_terminated_length": 447.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3454949576407671, "epoch": 0.00487, "frac_reward_zero_std": 0.0, "grad_norm": 0.08707001060247421, "kl": 0.6311333999037743, "learning_rate": 3.6557047331598855e-05, "loss": -0.0028, "num_tokens": 10402684.0, "reward": 0.7697407007217407, "reward_std": 0.5020853281021118, "rewards/rollout_reward_func/mean": 0.7697407007217407, "rewards/rollout_reward_func/std": 0.5098025798797607, "sampling/importance_sampling_ratio/max": 1.0449756383895874, "sampling/importance_sampling_ratio/mean": 0.9987537860870361, "sampling/importance_sampling_ratio/min": 0.9550891518592834, "sampling/sampling_logp_difference/max": 0.0710706114768982, "sampling/sampling_logp_difference/mean": 0.004729702137410641, "step": 487, "step_time": 9.074799856001846 }, { "clip_ratio/high_max": 0.011904762126505375, "clip_ratio/high_mean": 0.0059523810632526875, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.016369048040360212, "entropy": 0.33793653175234795, "epoch": 0.00488, "grad_norm": 0.0718548595905304, "kl": 0.618007555603981, "learning_rate": 3.6504014870383484e-05, "loss": -0.0033, "step": 488, "step_time": 4.914530513001409 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2891.0, "completions/max_terminated_length": 2891.0, "completions/mean_length": 551.1875, "completions/mean_terminated_length": 551.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5333796851336956, "epoch": 0.00489, "frac_reward_zero_std": 0.0, "grad_norm": 0.1519799530506134, "kl": 1.8762893043458462, "learning_rate": 3.6450918031355536e-05, "loss": -0.0058, "num_tokens": 10448661.0, "reward": 0.6209142208099365, "reward_std": 0.6578450202941895, "rewards/rollout_reward_func/mean": 0.6209142208099365, "rewards/rollout_reward_func/std": 0.7031481862068176, "sampling/importance_sampling_ratio/max": 1.1494615077972412, "sampling/importance_sampling_ratio/mean": 1.0023226737976074, "sampling/importance_sampling_ratio/min": 0.9370383024215698, "sampling/sampling_logp_difference/max": 0.12659621238708496, "sampling/sampling_logp_difference/mean": 0.011314250528812408, "step": 489, "step_time": 13.303078814999026 }, { "clip_ratio/high_max": 0.02818627515807748, "clip_ratio/high_mean": 0.01409313757903874, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01409313757903874, "entropy": 0.5415518395602703, "epoch": 0.0049, "grad_norm": 0.1291104257106781, "kl": 1.7062889263033867, "learning_rate": 3.63977572249187e-05, "loss": -0.0069, "step": 490, "step_time": 8.140433490999385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2104.0, "completions/max_terminated_length": 2104.0, "completions/mean_length": 407.96875, "completions/mean_terminated_length": 407.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.424353189766407, "epoch": 0.00491, "frac_reward_zero_std": 0.0, "grad_norm": 0.13289621472358704, "kl": 0.6515089757740498, "learning_rate": 3.6344532861971076e-05, "loss": -0.0049, "num_tokens": 10488937.0, "reward": 0.6175147294998169, "reward_std": 0.7154800295829773, "rewards/rollout_reward_func/mean": 0.6175147294998169, "rewards/rollout_reward_func/std": 0.7102190256118774, "sampling/importance_sampling_ratio/max": 1.0238820314407349, "sampling/importance_sampling_ratio/mean": 0.9942148327827454, "sampling/importance_sampling_ratio/min": 0.8923596739768982, "sampling/sampling_logp_difference/max": 0.09479379653930664, "sampling/sampling_logp_difference/mean": 0.0050331465899944305, "step": 491, "step_time": 10.610302314002183 }, { "clip_ratio/high_max": 0.040865384973585606, "clip_ratio/high_mean": 0.020432692486792803, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020432692486792803, "entropy": 0.4506933633238077, "epoch": 0.00492, "grad_norm": 0.12129886448383331, "kl": 0.6438701152801514, "learning_rate": 3.629124535390202e-05, "loss": -0.0062, "step": 492, "step_time": 5.944788821000657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 390.59375, "completions/mean_terminated_length": 390.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4007849656045437, "epoch": 0.00493, "frac_reward_zero_std": 0.0, "grad_norm": 0.16031096875667572, "kl": 0.46651776880025864, "learning_rate": 3.623789511258896e-05, "loss": 0.0003, "num_tokens": 10529589.0, "reward": 0.49393191933631897, "reward_std": 0.7521201372146606, "rewards/rollout_reward_func/mean": 0.49393191933631897, "rewards/rollout_reward_func/std": 0.75495445728302, "sampling/importance_sampling_ratio/max": 1.156187653541565, "sampling/importance_sampling_ratio/mean": 1.0049922466278076, "sampling/importance_sampling_ratio/min": 0.935112714767456, "sampling/sampling_logp_difference/max": 0.0896749496459961, "sampling/sampling_logp_difference/mean": 0.006735292263329029, "step": 493, "step_time": 9.60535139500007 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.022991071920841932, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.030803571920841932, "entropy": 0.4026440232992172, "epoch": 0.00494, "grad_norm": 0.08772898465394974, "kl": 0.4554747976362705, "learning_rate": 3.6184482550394206e-05, "loss": -0.0017, "step": 494, "step_time": 4.93897836600172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2110.0, "completions/max_terminated_length": 2110.0, "completions/mean_length": 580.65625, "completions/mean_terminated_length": 580.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.564285496249795, "epoch": 0.00495, "frac_reward_zero_std": 0.25, "grad_norm": 0.20202191174030304, "kl": 0.7213362492620945, "learning_rate": 3.6131008080161774e-05, "loss": 0.0039, "num_tokens": 10577660.0, "reward": 0.7091980576515198, "reward_std": 0.5013554692268372, "rewards/rollout_reward_func/mean": 0.7091980576515198, "rewards/rollout_reward_func/std": 0.5913074612617493, "sampling/importance_sampling_ratio/max": 1.2181938886642456, "sampling/importance_sampling_ratio/mean": 0.9975022077560425, "sampling/importance_sampling_ratio/min": 0.8883779048919678, "sampling/sampling_logp_difference/max": 0.20598125457763672, "sampling/sampling_logp_difference/mean": 0.008637258782982826, "step": 495, "step_time": 10.80016254199927 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.5600418541580439, "epoch": 0.00496, "grad_norm": 0.08527921885251999, "kl": 0.694824568927288, "learning_rate": 3.607747211521419e-05, "loss": 0.0035, "step": 496, "step_time": 6.039077094999811 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 492.59375, "completions/mean_terminated_length": 492.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.33263266645371914, "epoch": 0.00497, "frac_reward_zero_std": 0.25, "grad_norm": 0.0911799743771553, "kl": 0.528871588408947, "learning_rate": 3.602387506934928e-05, "loss": 0.0006, "num_tokens": 10621546.0, "reward": 0.7763752937316895, "reward_std": 0.4175844192504883, "rewards/rollout_reward_func/mean": 0.7763752937316895, "rewards/rollout_reward_func/std": 0.5535637736320496, "sampling/importance_sampling_ratio/max": 1.0584094524383545, "sampling/importance_sampling_ratio/mean": 1.0058200359344482, "sampling/importance_sampling_ratio/min": 0.9602040648460388, "sampling/sampling_logp_difference/max": 0.03838300704956055, "sampling/sampling_logp_difference/mean": 0.0031736488454043865, "step": 497, "step_time": 9.73433467400082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.31813283637166023, "epoch": 0.00498, "grad_norm": 0.02865053154528141, "kl": 0.5288564935326576, "learning_rate": 3.597021735683699e-05, "loss": 0.0002, "step": 498, "step_time": 5.047162346000732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1659.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 566.15625, "completions/mean_terminated_length": 566.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4328548088669777, "epoch": 0.00499, "frac_reward_zero_std": 0.0, "grad_norm": 0.23194505274295807, "kl": 0.7636939361691475, "learning_rate": 3.591649939241619e-05, "loss": -0.0031, "num_tokens": 10669007.0, "reward": 0.5064021944999695, "reward_std": 0.809478759765625, "rewards/rollout_reward_func/mean": 0.5064021944999695, "rewards/rollout_reward_func/std": 0.8266685009002686, "sampling/importance_sampling_ratio/max": 1.1127376556396484, "sampling/importance_sampling_ratio/mean": 1.0042343139648438, "sampling/importance_sampling_ratio/min": 0.8871070742607117, "sampling/sampling_logp_difference/max": 0.11988973617553711, "sampling/sampling_logp_difference/mean": 0.00810575857758522, "step": 499, "step_time": 10.061138863999076 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.019744318444281816, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.035369318444281816, "entropy": 0.41315706726163626, "epoch": 0.005, "grad_norm": 0.1375640332698822, "kl": 0.9688799530267715, "learning_rate": 3.5862721591291456e-05, "loss": -0.0037, "step": 500, "step_time": 5.278838877000453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1677.0, "completions/max_terminated_length": 1677.0, "completions/mean_length": 556.8125, "completions/mean_terminated_length": 556.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3472451101988554, "epoch": 0.00501, "frac_reward_zero_std": 0.0, "grad_norm": 0.11012981086969376, "kl": 0.6334941312670708, "learning_rate": 3.580888436912985e-05, "loss": -0.0018, "num_tokens": 10716043.0, "reward": 0.7349107265472412, "reward_std": 0.5210787057876587, "rewards/rollout_reward_func/mean": 0.7349107265472412, "rewards/rollout_reward_func/std": 0.5298423767089844, "sampling/importance_sampling_ratio/max": 1.0288069248199463, "sampling/importance_sampling_ratio/mean": 1.0007152557373047, "sampling/importance_sampling_ratio/min": 0.9117796421051025, "sampling/sampling_logp_difference/max": 0.09209537506103516, "sampling/sampling_logp_difference/mean": 0.0038620263803750277, "step": 501, "step_time": 9.786617273998672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3452577479183674, "epoch": 0.00502, "grad_norm": 0.10044565051794052, "kl": 0.6182650700211525, "learning_rate": 3.5754988142057725e-05, "loss": -0.0019, "step": 502, "step_time": 5.304195419997086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 446.9375, "completions/mean_terminated_length": 446.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3792800158262253, "epoch": 0.00503, "frac_reward_zero_std": 0.0, "grad_norm": 0.06357674300670624, "kl": 0.6828403882682323, "learning_rate": 3.5701033326657503e-05, "loss": 0.0057, "num_tokens": 10759518.0, "reward": 0.6109251379966736, "reward_std": 0.5957663059234619, "rewards/rollout_reward_func/mean": 0.6109251379966736, "rewards/rollout_reward_func/std": 0.6196985244750977, "sampling/importance_sampling_ratio/max": 1.0334563255310059, "sampling/importance_sampling_ratio/mean": 0.9902770519256592, "sampling/importance_sampling_ratio/min": 0.6999945640563965, "sampling/sampling_logp_difference/max": 0.3216061592102051, "sampling/sampling_logp_difference/mean": 0.006732238456606865, "step": 503, "step_time": 9.270368804001919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3853929974138737, "epoch": 0.00504, "grad_norm": 0.0646006241440773, "kl": 0.6789948865771294, "learning_rate": 3.564702033996448e-05, "loss": 0.0057, "step": 504, "step_time": 5.3140321760001825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1443.0, "completions/max_terminated_length": 1443.0, "completions/mean_length": 404.96875, "completions/mean_terminated_length": 404.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2917573098093271, "epoch": 0.00505, "frac_reward_zero_std": 0.5, "grad_norm": 0.21615764498710632, "kl": 0.40063149482011795, "learning_rate": 3.559294959946352e-05, "loss": 0.0029, "num_tokens": 10800682.0, "reward": 0.9055366516113281, "reward_std": 0.2671825885772705, "rewards/rollout_reward_func/mean": 0.9055366516113281, "rewards/rollout_reward_func/std": 0.39137011766433716, "sampling/importance_sampling_ratio/max": 1.1126264333724976, "sampling/importance_sampling_ratio/mean": 1.0003222227096558, "sampling/importance_sampling_ratio/min": 0.9081681966781616, "sampling/sampling_logp_difference/max": 0.1038506031036377, "sampling/sampling_logp_difference/mean": 0.006222487427294254, "step": 505, "step_time": 8.881802491998315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.021875000093132257, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.021875000093132257, "entropy": 0.29359386675059795, "epoch": 0.00506, "grad_norm": 0.030892420560121536, "kl": 0.3939616084098816, "learning_rate": 3.553882152308595e-05, "loss": 0.002, "step": 506, "step_time": 5.368856798999332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3107.0, "completions/max_terminated_length": 3107.0, "completions/mean_length": 374.9375, "completions/mean_terminated_length": 374.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.27951158955693245, "epoch": 0.00507, "frac_reward_zero_std": 0.0, "grad_norm": 0.0928579792380333, "kl": 0.49409538321197033, "learning_rate": 3.5484636529206223e-05, "loss": 0.0014, "num_tokens": 10840033.0, "reward": 0.6985541582107544, "reward_std": 0.5420877933502197, "rewards/rollout_reward_func/mean": 0.6985541582107544, "rewards/rollout_reward_func/std": 0.5503457188606262, "sampling/importance_sampling_ratio/max": 1.0348352193832397, "sampling/importance_sampling_ratio/mean": 0.996739387512207, "sampling/importance_sampling_ratio/min": 0.8878487944602966, "sampling/sampling_logp_difference/max": 0.11935663223266602, "sampling/sampling_logp_difference/mean": 0.0045079430565238, "step": 507, "step_time": 13.02876386799835 }, { "clip_ratio/high_max": 0.02500000037252903, "clip_ratio/high_mean": 0.012500000186264515, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012500000186264515, "entropy": 0.29256257973611355, "epoch": 0.00508, "grad_norm": 0.08473310619592667, "kl": 0.47471788339316845, "learning_rate": 3.543039503663874e-05, "loss": 0.0007, "step": 508, "step_time": 7.729241019000256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1458.0, "completions/max_terminated_length": 1458.0, "completions/mean_length": 358.90625, "completions/mean_terminated_length": 358.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3459412567317486, "epoch": 0.00509, "frac_reward_zero_std": 0.5, "grad_norm": 0.05875483900308609, "kl": 0.4838414154946804, "learning_rate": 3.537609746463462e-05, "loss": -0.0061, "num_tokens": 10879403.0, "reward": 0.8344150185585022, "reward_std": 0.28975215554237366, "rewards/rollout_reward_func/mean": 0.8344150185585022, "rewards/rollout_reward_func/std": 0.4648338854312897, "sampling/importance_sampling_ratio/max": 1.2234967947006226, "sampling/importance_sampling_ratio/mean": 1.0124537944793701, "sampling/importance_sampling_ratio/min": 0.9216600060462952, "sampling/sampling_logp_difference/max": 0.13287687301635742, "sampling/sampling_logp_difference/mean": 0.007536227814853191, "step": 509, "step_time": 9.088742968001498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3735465109348297, "epoch": 0.0051, "grad_norm": 0.060096945613622665, "kl": 0.45342572778463364, "learning_rate": 3.532174423287839e-05, "loss": -0.0064, "step": 510, "step_time": 4.856813102999695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 546.25, "completions/mean_terminated_length": 546.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8162106350064278, "epoch": 0.00511, "frac_reward_zero_std": 0.25, "grad_norm": 0.2950285077095032, "kl": 0.49665737710893154, "learning_rate": 3.526733576148484e-05, "loss": -0.001, "num_tokens": 10926488.0, "reward": 0.7317491769790649, "reward_std": 0.4568134546279907, "rewards/rollout_reward_func/mean": 0.7317491769790649, "rewards/rollout_reward_func/std": 0.5334124565124512, "sampling/importance_sampling_ratio/max": 1.3884754180908203, "sampling/importance_sampling_ratio/mean": 1.0145983695983887, "sampling/importance_sampling_ratio/min": 0.8920437097549438, "sampling/sampling_logp_difference/max": 0.16352248191833496, "sampling/sampling_logp_difference/mean": 0.013201040215790272, "step": 511, "step_time": 10.227081407000696 }, { "clip_ratio/high_max": 0.009615384973585606, "clip_ratio/high_mean": 0.004807692486792803, "clip_ratio/low_mean": 0.029647436924278736, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03445512894541025, "entropy": 0.8260368891060352, "epoch": 0.00512, "grad_norm": 0.12642188370227814, "kl": 0.4887721296399832, "learning_rate": 3.52128724709957e-05, "loss": -0.0041, "step": 512, "step_time": 5.188174777998938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 466.0625, "completions/mean_terminated_length": 466.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.598879911005497, "epoch": 0.00513, "frac_reward_zero_std": 0.0, "grad_norm": 0.4971950650215149, "kl": 0.39408341981470585, "learning_rate": 3.5158354782376415e-05, "loss": 0.0703, "num_tokens": 10969953.0, "reward": 0.8384893536567688, "reward_std": 0.45682114362716675, "rewards/rollout_reward_func/mean": 0.8384893536567688, "rewards/rollout_reward_func/std": 0.4538170099258423, "sampling/importance_sampling_ratio/max": 1.1275700330734253, "sampling/importance_sampling_ratio/mean": 0.9889053106307983, "sampling/importance_sampling_ratio/min": 0.8003628849983215, "sampling/sampling_logp_difference/max": 0.28565502166748047, "sampling/sampling_logp_difference/mean": 0.021460406482219696, "step": 513, "step_time": 10.007047298999169 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.03485576994717121, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.05048076994717121, "entropy": 0.6013813726603985, "epoch": 0.00514, "grad_norm": 0.28997984528541565, "kl": 0.37309285812079906, "learning_rate": 3.510378311701291e-05, "loss": 0.0615, "step": 514, "step_time": 5.179987104999782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 579.0, "completions/mean_terminated_length": 579.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5991521999239922, "epoch": 0.00515, "frac_reward_zero_std": 0.0, "grad_norm": 0.16381268203258514, "kl": 0.37258241698145866, "learning_rate": 3.504915789670829e-05, "loss": -0.0051, "num_tokens": 11018598.0, "reward": 0.4680670201778412, "reward_std": 0.8145105838775635, "rewards/rollout_reward_func/mean": 0.4680670201778412, "rewards/rollout_reward_func/std": 0.7924445271492004, "sampling/importance_sampling_ratio/max": 1.1154096126556396, "sampling/importance_sampling_ratio/mean": 1.0058727264404297, "sampling/importance_sampling_ratio/min": 0.8896937966346741, "sampling/sampling_logp_difference/max": 0.12115478515625, "sampling/sampling_logp_difference/mean": 0.012801868841052055, "step": 515, "step_time": 9.247737884000344 }, { "clip_ratio/high_max": 0.03645833395421505, "clip_ratio/high_mean": 0.018229166977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018229166977107525, "entropy": 0.6004579588770866, "epoch": 0.00516, "grad_norm": 0.14377394318580627, "kl": 0.3425060696899891, "learning_rate": 3.499447954367964e-05, "loss": -0.0058, "step": 516, "step_time": 4.967373214000872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2290.0, "completions/max_terminated_length": 2290.0, "completions/mean_length": 513.59375, "completions/mean_terminated_length": 513.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4874865338206291, "epoch": 0.00517, "frac_reward_zero_std": 0.0, "grad_norm": 0.1575501263141632, "kl": 0.4966179057955742, "learning_rate": 3.493974848055469e-05, "loss": 0.0088, "num_tokens": 11063715.0, "reward": 0.8041245937347412, "reward_std": 0.48593273758888245, "rewards/rollout_reward_func/mean": 0.8041245937347412, "rewards/rollout_reward_func/std": 0.48197025060653687, "sampling/importance_sampling_ratio/max": 1.0993103981018066, "sampling/importance_sampling_ratio/mean": 0.9986966848373413, "sampling/importance_sampling_ratio/min": 0.8398870825767517, "sampling/sampling_logp_difference/max": 0.11062479019165039, "sampling/sampling_logp_difference/mean": 0.010282212868332863, "step": 517, "step_time": 11.943100001998573 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018229166977107525, "entropy": 0.4877443462610245, "epoch": 0.00518, "grad_norm": 0.13229955732822418, "kl": 0.49340154230594635, "learning_rate": 3.488496513036859e-05, "loss": 0.0071, "step": 518, "step_time": 6.4143642369981535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1515.0, "completions/max_terminated_length": 1515.0, "completions/mean_length": 443.03125, "completions/mean_terminated_length": 443.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.46937717869877815, "epoch": 0.00519, "frac_reward_zero_std": 0.25, "grad_norm": 0.1118779182434082, "kl": 0.3910116869956255, "learning_rate": 3.4830129916560654e-05, "loss": -0.0037, "num_tokens": 11107089.0, "reward": 0.8695473074913025, "reward_std": 0.3689759075641632, "rewards/rollout_reward_func/mean": 0.8695473074913025, "rewards/rollout_reward_func/std": 0.43328365683555603, "sampling/importance_sampling_ratio/max": 1.1191636323928833, "sampling/importance_sampling_ratio/mean": 1.0044898986816406, "sampling/importance_sampling_ratio/min": 0.8984665274620056, "sampling/sampling_logp_difference/max": 0.10597825050354004, "sampling/sampling_logp_difference/mean": 0.009178975597023964, "step": 519, "step_time": 9.394937339000535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4668683260679245, "epoch": 0.0052, "grad_norm": 0.07929416745901108, "kl": 0.3976453635841608, "learning_rate": 3.477524326297105e-05, "loss": -0.0041, "step": 520, "step_time": 4.928907775999505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1695.0, "completions/max_terminated_length": 1695.0, "completions/mean_length": 461.40625, "completions/mean_terminated_length": 461.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4774599280208349, "epoch": 0.00521, "frac_reward_zero_std": 0.0, "grad_norm": 0.05391731113195419, "kl": 0.8719459511339664, "learning_rate": 3.472030559383756e-05, "loss": 0.0003, "num_tokens": 11151324.0, "reward": 0.6225029230117798, "reward_std": 0.6879696846008301, "rewards/rollout_reward_func/mean": 0.6225029230117798, "rewards/rollout_reward_func/std": 0.6964209079742432, "sampling/importance_sampling_ratio/max": 1.101783037185669, "sampling/importance_sampling_ratio/mean": 1.0153234004974365, "sampling/importance_sampling_ratio/min": 0.9319213032722473, "sampling/sampling_logp_difference/max": 0.09651994705200195, "sampling/sampling_logp_difference/mean": 0.008457206189632416, "step": 521, "step_time": 9.803926083998704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.47505791671574116, "epoch": 0.00522, "grad_norm": 0.05584590509533882, "kl": 0.8767726235091686, "learning_rate": 3.466531733379226e-05, "loss": 0.0004, "step": 522, "step_time": 5.464582200000223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1479.0, "completions/max_terminated_length": 1479.0, "completions/mean_length": 418.75, "completions/mean_terminated_length": 418.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4962672460824251, "epoch": 0.00523, "frac_reward_zero_std": 0.0, "grad_norm": 0.09184712916612625, "kl": 0.2795450761914253, "learning_rate": 3.461027890785828e-05, "loss": -0.0069, "num_tokens": 11194000.0, "reward": 0.7133693695068359, "reward_std": 0.6237742304801941, "rewards/rollout_reward_func/mean": 0.7133693695068359, "rewards/rollout_reward_func/std": 0.6341961026191711, "sampling/importance_sampling_ratio/max": 1.1897399425506592, "sampling/importance_sampling_ratio/mean": 1.007777452468872, "sampling/importance_sampling_ratio/min": 0.8788117170333862, "sampling/sampling_logp_difference/max": 0.1218646764755249, "sampling/sampling_logp_difference/mean": 0.012939440086483955, "step": 523, "step_time": 9.01176127800045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.4928020816296339, "epoch": 0.00524, "grad_norm": 0.08682014793157578, "kl": 0.2773384265601635, "learning_rate": 3.455519074144647e-05, "loss": -0.0072, "step": 524, "step_time": 5.309568964000391 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 777.0, "completions/max_terminated_length": 777.0, "completions/mean_length": 311.5, "completions/mean_terminated_length": 311.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.38284888304769993, "epoch": 0.00525, "frac_reward_zero_std": 0.0, "grad_norm": 0.13519048690795898, "kl": 0.31442628614604473, "learning_rate": 3.4500053260352195e-05, "loss": -0.0005, "num_tokens": 11231328.0, "reward": 0.7467682361602783, "reward_std": 0.6308538913726807, "rewards/rollout_reward_func/mean": 0.7467682361602783, "rewards/rollout_reward_func/std": 0.6201589107513428, "sampling/importance_sampling_ratio/max": 1.0935240983963013, "sampling/importance_sampling_ratio/mean": 1.0025017261505127, "sampling/importance_sampling_ratio/min": 0.9424391984939575, "sampling/sampling_logp_difference/max": 0.061603546142578125, "sampling/sampling_logp_difference/mean": 0.005324853118509054, "step": 525, "step_time": 6.879217372999847 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.38307948037981987, "epoch": 0.00526, "grad_norm": 0.13849660754203796, "kl": 0.3114128950983286, "learning_rate": 3.444486689075193e-05, "loss": -0.0013, "step": 526, "step_time": 4.111061450000307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1499.0, "completions/max_terminated_length": 1499.0, "completions/mean_length": 385.90625, "completions/mean_terminated_length": 385.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.430482130497694, "epoch": 0.00527, "frac_reward_zero_std": 0.0, "grad_norm": 0.4708176553249359, "kl": 0.4153913054615259, "learning_rate": 3.438963205920007e-05, "loss": 0.0082, "num_tokens": 11271444.0, "reward": 0.7824954986572266, "reward_std": 0.615195631980896, "rewards/rollout_reward_func/mean": 0.7824954986572266, "rewards/rollout_reward_func/std": 0.5989040732383728, "sampling/importance_sampling_ratio/max": 1.376358151435852, "sampling/importance_sampling_ratio/mean": 1.0103178024291992, "sampling/importance_sampling_ratio/min": 0.887800395488739, "sampling/sampling_logp_difference/max": 0.14876127243041992, "sampling/sampling_logp_difference/mean": 0.011190340854227543, "step": 527, "step_time": 9.032065518999843 }, { "clip_ratio/high_max": 0.033333334140479565, "clip_ratio/high_mean": 0.016666667070239782, "clip_ratio/low_mean": 0.03645833395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.05312500149011612, "entropy": 0.4200764410197735, "epoch": 0.00528, "grad_norm": 0.132987380027771, "kl": 0.6477842535823584, "learning_rate": 3.433434919262556e-05, "loss": 0.0061, "step": 528, "step_time": 4.877926120999291 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2227.0, "completions/max_terminated_length": 2227.0, "completions/mean_length": 536.46875, "completions/mean_terminated_length": 536.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4935853146016598, "epoch": 0.00529, "frac_reward_zero_std": 0.25, "grad_norm": 0.16172848641872406, "kl": 0.3695905525237322, "learning_rate": 3.427901871832864e-05, "loss": -0.0013, "num_tokens": 11316560.0, "reward": 0.7500996589660645, "reward_std": 0.5460121631622314, "rewards/rollout_reward_func/mean": 0.7500996589660645, "rewards/rollout_reward_func/std": 0.6196237802505493, "sampling/importance_sampling_ratio/max": 1.11495840549469, "sampling/importance_sampling_ratio/mean": 0.9944877028465271, "sampling/importance_sampling_ratio/min": 0.8520992398262024, "sampling/sampling_logp_difference/max": 0.11047196388244629, "sampling/sampling_logp_difference/mean": 0.009878981858491898, "step": 529, "step_time": 11.46354291499938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.012500000186264515, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012500000186264515, "entropy": 0.48764992132782936, "epoch": 0.0053, "grad_norm": 0.09765961021184921, "kl": 0.3949160519987345, "learning_rate": 3.422364106397753e-05, "loss": -0.0023, "step": 530, "step_time": 6.373134007000772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1587.0, "completions/max_terminated_length": 1587.0, "completions/mean_length": 340.65625, "completions/mean_terminated_length": 340.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3703652126714587, "epoch": 0.00531, "frac_reward_zero_std": 0.0, "grad_norm": 0.1395559310913086, "kl": 0.43020712211728096, "learning_rate": 3.4168216657605094e-05, "loss": 0.0084, "num_tokens": 11354633.0, "reward": 0.7076616287231445, "reward_std": 0.5972615480422974, "rewards/rollout_reward_func/mean": 0.7076616287231445, "rewards/rollout_reward_func/std": 0.593453586101532, "sampling/importance_sampling_ratio/max": 1.1811062097549438, "sampling/importance_sampling_ratio/mean": 0.9956051707267761, "sampling/importance_sampling_ratio/min": 0.8731714487075806, "sampling/sampling_logp_difference/max": 0.1662130355834961, "sampling/sampling_logp_difference/mean": 0.010906975716352463, "step": 531, "step_time": 9.863075354998728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.36956482171081007, "epoch": 0.00532, "grad_norm": 0.13268868625164032, "kl": 0.43179685436189175, "learning_rate": 3.411274592760561e-05, "loss": 0.008, "step": 532, "step_time": 5.138532854000914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2407.0, "completions/max_terminated_length": 2407.0, "completions/mean_length": 506.9375, "completions/mean_terminated_length": 506.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4864988997578621, "epoch": 0.00533, "frac_reward_zero_std": 0.25, "grad_norm": 0.1067332997918129, "kl": 0.30680250469595194, "learning_rate": 3.4057229302731346e-05, "loss": -0.0057, "num_tokens": 11399438.0, "reward": 0.654613733291626, "reward_std": 0.576657772064209, "rewards/rollout_reward_func/mean": 0.654613733291626, "rewards/rollout_reward_func/std": 0.6960099339485168, "sampling/importance_sampling_ratio/max": 1.1055868864059448, "sampling/importance_sampling_ratio/mean": 1.0021995306015015, "sampling/importance_sampling_ratio/min": 0.8999146223068237, "sampling/sampling_logp_difference/max": 0.10502815246582031, "sampling/sampling_logp_difference/mean": 0.009253967553377151, "step": 533, "step_time": 12.640666126000724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.4886026941239834, "epoch": 0.00534, "grad_norm": 0.09587804973125458, "kl": 0.30502785090357065, "learning_rate": 3.400166721208935e-05, "loss": -0.0065, "step": 534, "step_time": 6.622647641000185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 407.34375, "completions/mean_terminated_length": 407.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5130869783461094, "epoch": 0.00535, "frac_reward_zero_std": 0.25, "grad_norm": 0.09834747016429901, "kl": 0.2923258077353239, "learning_rate": 3.3946060085138074e-05, "loss": -0.0069, "num_tokens": 11441173.0, "reward": 0.7199729681015015, "reward_std": 0.5867053270339966, "rewards/rollout_reward_func/mean": 0.7199729681015015, "rewards/rollout_reward_func/std": 0.6758123636245728, "sampling/importance_sampling_ratio/max": 1.095146894454956, "sampling/importance_sampling_ratio/mean": 0.9893099665641785, "sampling/importance_sampling_ratio/min": 0.8744808435440063, "sampling/sampling_logp_difference/max": 0.13837873935699463, "sampling/sampling_logp_difference/mean": 0.011219887994229794, "step": 535, "step_time": 9.142185073999826 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.512200377881527, "epoch": 0.00536, "grad_norm": 0.12818241119384766, "kl": 0.2922016382217407, "learning_rate": 3.3890408351684073e-05, "loss": -0.0071, "step": 536, "step_time": 5.030978691997916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1238.0, "completions/max_terminated_length": 1238.0, "completions/mean_length": 369.84375, "completions/mean_terminated_length": 369.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.47794514894485474, "epoch": 0.00537, "frac_reward_zero_std": 0.25, "grad_norm": 0.161370649933815, "kl": 0.31789403408765793, "learning_rate": 3.383471244187869e-05, "loss": -0.0009, "num_tokens": 11481303.0, "reward": 0.7053090333938599, "reward_std": 0.4743305444717407, "rewards/rollout_reward_func/mean": 0.7053090333938599, "rewards/rollout_reward_func/std": 0.5975028872489929, "sampling/importance_sampling_ratio/max": 1.11380934715271, "sampling/importance_sampling_ratio/mean": 1.0015394687652588, "sampling/importance_sampling_ratio/min": 0.8861750960350037, "sampling/sampling_logp_difference/max": 0.12105369567871094, "sampling/sampling_logp_difference/mean": 0.009354098699986935, "step": 537, "step_time": 8.219653758001186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.026041666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.026041666977107525, "entropy": 0.46487968787550926, "epoch": 0.00538, "grad_norm": 0.0701153501868248, "kl": 0.31389736756682396, "learning_rate": 3.37789727862147e-05, "loss": -0.0018, "step": 538, "step_time": 4.958228282001073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 563.8125, "completions/mean_terminated_length": 563.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.526688739657402, "epoch": 0.00539, "frac_reward_zero_std": 0.0, "grad_norm": 0.18581871688365936, "kl": 0.45191347412765026, "learning_rate": 3.372318981552303e-05, "loss": -0.004, "num_tokens": 11529289.0, "reward": 0.7716129422187805, "reward_std": 0.5001521706581116, "rewards/rollout_reward_func/mean": 0.7716129422187805, "rewards/rollout_reward_func/std": 0.5053718686103821, "sampling/importance_sampling_ratio/max": 1.1046065092086792, "sampling/importance_sampling_ratio/mean": 0.9909417033195496, "sampling/importance_sampling_ratio/min": 0.8470938205718994, "sampling/sampling_logp_difference/max": 0.16593551635742188, "sampling/sampling_logp_difference/mean": 0.008409309200942516, "step": 539, "step_time": 9.816829993001193 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "entropy": 0.5234719067811966, "epoch": 0.0054, "grad_norm": 0.08293615281581879, "kl": 0.41955691762268543, "learning_rate": 3.366736396096938e-05, "loss": -0.0054, "step": 540, "step_time": 6.027534117999494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 333.25, "completions/mean_terminated_length": 333.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4054288323968649, "epoch": 0.00541, "frac_reward_zero_std": 0.0, "grad_norm": 0.172298863530159, "kl": 0.3195253871381283, "learning_rate": 3.3611495654050945e-05, "loss": -0.0096, "num_tokens": 11568369.0, "reward": 0.6616854667663574, "reward_std": 0.5615912675857544, "rewards/rollout_reward_func/mean": 0.6616854667663574, "rewards/rollout_reward_func/std": 0.5700283050537109, "sampling/importance_sampling_ratio/max": 1.0809738636016846, "sampling/importance_sampling_ratio/mean": 0.9878034591674805, "sampling/importance_sampling_ratio/min": 0.6429808139801025, "sampling/sampling_logp_difference/max": 0.4599034786224365, "sampling/sampling_logp_difference/mean": 0.010869825258851051, "step": 541, "step_time": 8.767833933000475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.4016979690641165, "epoch": 0.00542, "grad_norm": 0.12061386555433273, "kl": 0.31516095623373985, "learning_rate": 3.355558532659303e-05, "loss": -0.0101, "step": 542, "step_time": 4.900742633998561 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2396.0, "completions/max_terminated_length": 2396.0, "completions/mean_length": 589.5625, "completions/mean_terminated_length": 589.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5315080210566521, "epoch": 0.00543, "frac_reward_zero_std": 0.0, "grad_norm": 0.215481236577034, "kl": 0.30421255342662334, "learning_rate": 3.349963341074571e-05, "loss": 0.0042, "num_tokens": 11616958.0, "reward": 0.6150786280632019, "reward_std": 0.5925868153572083, "rewards/rollout_reward_func/mean": 0.6150786280632019, "rewards/rollout_reward_func/std": 0.6651361584663391, "sampling/importance_sampling_ratio/max": 1.17099130153656, "sampling/importance_sampling_ratio/mean": 0.9932326674461365, "sampling/importance_sampling_ratio/min": 0.8389370441436768, "sampling/sampling_logp_difference/max": 0.11748075485229492, "sampling/sampling_logp_difference/mean": 0.011867680586874485, "step": 543, "step_time": 11.888332722001905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02083333395421505, "entropy": 0.5100703649222851, "epoch": 0.00544, "grad_norm": 0.1136183813214302, "kl": 0.30845778807997704, "learning_rate": 3.344364033898055e-05, "loss": 0.0029, "step": 544, "step_time": 7.130199317999541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2317.0, "completions/max_terminated_length": 2317.0, "completions/mean_length": 417.28125, "completions/mean_terminated_length": 417.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3789124246686697, "epoch": 0.00545, "frac_reward_zero_std": 0.25, "grad_norm": 0.14152386784553528, "kl": 0.45837744139134884, "learning_rate": 3.338760654408719e-05, "loss": 0.0073, "num_tokens": 11658816.0, "reward": 0.677391767501831, "reward_std": 0.4704969823360443, "rewards/rollout_reward_func/mean": 0.677391767501831, "rewards/rollout_reward_func/std": 0.5966210961341858, "sampling/importance_sampling_ratio/max": 1.2213798761367798, "sampling/importance_sampling_ratio/mean": 0.999250054359436, "sampling/importance_sampling_ratio/min": 0.9126931428909302, "sampling/sampling_logp_difference/max": 0.20040321350097656, "sampling/sampling_logp_difference/mean": 0.010599883273243904, "step": 545, "step_time": 11.297801421999793 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 0.37146046571433544, "epoch": 0.00546, "grad_norm": 0.10626354068517685, "kl": 0.46086648665368557, "learning_rate": 3.3331532459170074e-05, "loss": 0.0063, "step": 546, "step_time": 6.512457783999707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2326.0, "completions/max_terminated_length": 2326.0, "completions/mean_length": 546.6875, "completions/mean_terminated_length": 546.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4195861220359802, "epoch": 0.00547, "frac_reward_zero_std": 0.5, "grad_norm": 0.12504953145980835, "kl": 0.42696473374962807, "learning_rate": 3.3275418517645015e-05, "loss": -0.0081, "num_tokens": 11704907.0, "reward": 0.7797852754592896, "reward_std": 0.4174503684043884, "rewards/rollout_reward_func/mean": 0.7797852754592896, "rewards/rollout_reward_func/std": 0.6086251735687256, "sampling/importance_sampling_ratio/max": 1.1463268995285034, "sampling/importance_sampling_ratio/mean": 0.989812970161438, "sampling/importance_sampling_ratio/min": 0.7701388597488403, "sampling/sampling_logp_difference/max": 0.26119041442871094, "sampling/sampling_logp_difference/mean": 0.010592188686132431, "step": 547, "step_time": 12.276280082000994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.41219979524612427, "epoch": 0.00548, "grad_norm": 0.11916685104370117, "kl": 0.44770470820367336, "learning_rate": 3.3219265153235924e-05, "loss": -0.0082, "step": 548, "step_time": 6.471565919000568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 429.6875, "completions/mean_terminated_length": 429.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3336270645959303, "epoch": 0.00549, "frac_reward_zero_std": 0.5, "grad_norm": 0.05381379276514053, "kl": 0.45710672065615654, "learning_rate": 3.316307279997142e-05, "loss": 0.006, "num_tokens": 11747205.0, "reward": 0.8995962142944336, "reward_std": 0.21814721822738647, "rewards/rollout_reward_func/mean": 0.8995962142944336, "rewards/rollout_reward_func/std": 0.3171801269054413, "sampling/importance_sampling_ratio/max": 1.0784255266189575, "sampling/importance_sampling_ratio/mean": 1.0003219842910767, "sampling/importance_sampling_ratio/min": 0.9230097532272339, "sampling/sampling_logp_difference/max": 0.075888991355896, "sampling/sampling_logp_difference/mean": 0.006106840446591377, "step": 549, "step_time": 9.088553336999212 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.32931553991511464, "epoch": 0.0055, "grad_norm": 0.053478535264730453, "kl": 0.4581599608063698, "learning_rate": 3.310684189218144e-05, "loss": 0.0061, "step": 550, "step_time": 5.001961214000403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1641.0, "completions/max_terminated_length": 1641.0, "completions/mean_length": 504.59375, "completions/mean_terminated_length": 504.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37756587006151676, "epoch": 0.00551, "frac_reward_zero_std": 0.25, "grad_norm": 0.1436786651611328, "kl": 0.3908964414149523, "learning_rate": 3.3050572864494005e-05, "loss": -0.0019, "num_tokens": 11791994.0, "reward": 0.8082352876663208, "reward_std": 0.45691534876823425, "rewards/rollout_reward_func/mean": 0.8082352876663208, "rewards/rollout_reward_func/std": 0.539864718914032, "sampling/importance_sampling_ratio/max": 1.0850131511688232, "sampling/importance_sampling_ratio/mean": 1.0039724111557007, "sampling/importance_sampling_ratio/min": 0.8953409194946289, "sampling/sampling_logp_difference/max": 0.11101603507995605, "sampling/sampling_logp_difference/mean": 0.006258354987949133, "step": 551, "step_time": 10.165448798001307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.016666667070239782, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.016666667070239782, "entropy": 0.37676209956407547, "epoch": 0.00552, "grad_norm": 0.07014936953783035, "kl": 0.38909362629055977, "learning_rate": 3.299426615183169e-05, "loss": -0.0028, "step": 552, "step_time": 5.338153337998847 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1623.0, "completions/max_terminated_length": 1623.0, "completions/mean_length": 379.875, "completions/mean_terminated_length": 379.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.30264182947576046, "epoch": 0.00553, "frac_reward_zero_std": 0.0, "grad_norm": 0.17842286825180054, "kl": 0.3796873688697815, "learning_rate": 3.29379221894084e-05, "loss": 0.0088, "num_tokens": 11832375.0, "reward": 0.7472586035728455, "reward_std": 0.5348621606826782, "rewards/rollout_reward_func/mean": 0.7472586035728455, "rewards/rollout_reward_func/std": 0.560126543045044, "sampling/importance_sampling_ratio/max": 1.125741958618164, "sampling/importance_sampling_ratio/mean": 1.0091584920883179, "sampling/importance_sampling_ratio/min": 0.9251919388771057, "sampling/sampling_logp_difference/max": 0.11408209800720215, "sampling/sampling_logp_difference/mean": 0.008539148606359959, "step": 553, "step_time": 10.072733278001579 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02864583395421505, "entropy": 0.30467621982097626, "epoch": 0.00554, "grad_norm": 0.0904582217335701, "kl": 0.3964176904410124, "learning_rate": 3.288154141272596e-05, "loss": 0.0077, "step": 554, "step_time": 5.167214402999889 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2486.0, "completions/max_terminated_length": 2486.0, "completions/mean_length": 484.96875, "completions/mean_terminated_length": 484.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4197659455239773, "epoch": 0.00555, "frac_reward_zero_std": 0.0, "grad_norm": 0.17490452527999878, "kl": 0.39197992719709873, "learning_rate": 3.282512425757071e-05, "loss": 0.0026, "num_tokens": 11876702.0, "reward": 0.5841495394706726, "reward_std": 0.6853889226913452, "rewards/rollout_reward_func/mean": 0.5841495394706726, "rewards/rollout_reward_func/std": 0.6664652228355408, "sampling/importance_sampling_ratio/max": 1.0875792503356934, "sampling/importance_sampling_ratio/mean": 0.9796790480613708, "sampling/importance_sampling_ratio/min": 0.801962673664093, "sampling/sampling_logp_difference/max": 0.15452265739440918, "sampling/sampling_logp_difference/mean": 0.01243261806666851, "step": 555, "step_time": 11.596259391001695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "entropy": 0.42004838585853577, "epoch": 0.00556, "grad_norm": 0.16416428983211517, "kl": 0.3799601374194026, "learning_rate": 3.27686711600102e-05, "loss": 0.0019, "step": 556, "step_time": 6.675869939001132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 298.03125, "completions/mean_terminated_length": 298.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.296100702136755, "epoch": 0.00557, "frac_reward_zero_std": 0.0, "grad_norm": 0.08686459809541702, "kl": 0.3547234982252121, "learning_rate": 3.2712182556389786e-05, "loss": -0.0006, "num_tokens": 11913191.0, "reward": 0.675409197807312, "reward_std": 0.570483386516571, "rewards/rollout_reward_func/mean": 0.675409197807312, "rewards/rollout_reward_func/std": 0.6020766496658325, "sampling/importance_sampling_ratio/max": 1.1012393236160278, "sampling/importance_sampling_ratio/mean": 0.9939675331115723, "sampling/importance_sampling_ratio/min": 0.8649592995643616, "sampling/sampling_logp_difference/max": 0.1129312515258789, "sampling/sampling_logp_difference/mean": 0.007929746992886066, "step": 557, "step_time": 8.259547102998113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3019411563873291, "epoch": 0.00558, "grad_norm": 0.091214619576931, "kl": 0.35506474040448666, "learning_rate": 3.2655658883329256e-05, "loss": -0.0009, "step": 558, "step_time": 5.1560616370006755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1497.0, "completions/max_terminated_length": 1497.0, "completions/mean_length": 464.90625, "completions/mean_terminated_length": 464.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3624570667743683, "epoch": 0.00559, "frac_reward_zero_std": 0.0, "grad_norm": 0.08490628004074097, "kl": 0.29600737523287535, "learning_rate": 3.259910057771946e-05, "loss": -0.0085, "num_tokens": 11957600.0, "reward": 0.7073127031326294, "reward_std": 0.5970082879066467, "rewards/rollout_reward_func/mean": 0.7073127031326294, "rewards/rollout_reward_func/std": 0.5916892290115356, "sampling/importance_sampling_ratio/max": 1.1206960678100586, "sampling/importance_sampling_ratio/mean": 1.0066875219345093, "sampling/importance_sampling_ratio/min": 0.904128909111023, "sampling/sampling_logp_difference/max": 0.11385488510131836, "sampling/sampling_logp_difference/mean": 0.011059870943427086, "step": 559, "step_time": 9.106018709000637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.36344997957348824, "epoch": 0.0056, "grad_norm": 0.08217955380678177, "kl": 0.29725977778434753, "learning_rate": 3.2542508076718956e-05, "loss": -0.0089, "step": 560, "step_time": 5.761379186998056 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 414.84375, "completions/mean_terminated_length": 414.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.41970596089959145, "epoch": 0.00561, "frac_reward_zero_std": 0.25, "grad_norm": 0.09909054636955261, "kl": 0.3196252826601267, "learning_rate": 3.248588181775057e-05, "loss": -0.0082, "num_tokens": 12000508.0, "reward": 0.6170459985733032, "reward_std": 0.6098015308380127, "rewards/rollout_reward_func/mean": 0.6170459985733032, "rewards/rollout_reward_func/std": 0.7083413004875183, "sampling/importance_sampling_ratio/max": 1.0747047662734985, "sampling/importance_sampling_ratio/mean": 0.9975941181182861, "sampling/importance_sampling_ratio/min": 0.9365625381469727, "sampling/sampling_logp_difference/max": 0.07208812236785889, "sampling/sampling_logp_difference/mean": 0.006462081801146269, "step": 561, "step_time": 9.087883784000951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02083333395421505, "entropy": 0.4229220822453499, "epoch": 0.00562, "grad_norm": 0.07586663961410522, "kl": 0.3402908556163311, "learning_rate": 3.242922223849811e-05, "loss": -0.0088, "step": 562, "step_time": 4.93028025500189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1328.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 453.09375, "completions/mean_terminated_length": 453.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4017586875706911, "epoch": 0.00563, "frac_reward_zero_std": 0.0, "grad_norm": 0.2704007029533386, "kl": 2.8666939102113247, "learning_rate": 3.237252977690286e-05, "loss": 0.002, "num_tokens": 12044220.0, "reward": 0.5831002593040466, "reward_std": 0.7212445139884949, "rewards/rollout_reward_func/mean": 0.5831002593040466, "rewards/rollout_reward_func/std": 0.719884991645813, "sampling/importance_sampling_ratio/max": 1.0703608989715576, "sampling/importance_sampling_ratio/mean": 0.9909056425094604, "sampling/importance_sampling_ratio/min": 0.8945691585540771, "sampling/sampling_logp_difference/max": 0.11177682876586914, "sampling/sampling_logp_difference/mean": 0.008024544455111027, "step": 563, "step_time": 8.719878431000325 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.4108141139149666, "epoch": 0.00564, "grad_norm": 0.1385534256696701, "kl": 2.0604190062731504, "learning_rate": 3.231580487116033e-05, "loss": -0.0, "step": 564, "step_time": 5.174033473999771 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2468.0, "completions/max_terminated_length": 2468.0, "completions/mean_length": 477.75, "completions/mean_terminated_length": 477.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3659628857858479, "epoch": 0.00565, "frac_reward_zero_std": 0.25, "grad_norm": 0.07175809890031815, "kl": 0.5071796588599682, "learning_rate": 3.225904795971677e-05, "loss": -0.0032, "num_tokens": 12088522.0, "reward": 0.7741756439208984, "reward_std": 0.4622188210487366, "rewards/rollout_reward_func/mean": 0.7741756439208984, "rewards/rollout_reward_func/std": 0.5586991906166077, "sampling/importance_sampling_ratio/max": 1.1290048360824585, "sampling/importance_sampling_ratio/mean": 0.9980067014694214, "sampling/importance_sampling_ratio/min": 0.9059526920318604, "sampling/sampling_logp_difference/max": 0.12132120132446289, "sampling/sampling_logp_difference/mean": 0.008246284909546375, "step": 565, "step_time": 11.708574593999401 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.37562560057267547, "epoch": 0.00566, "grad_norm": 0.06708162277936935, "kl": 0.5217779166996479, "learning_rate": 3.2202259481265794e-05, "loss": -0.0032, "step": 566, "step_time": 6.66321255100047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1506.0, "completions/max_terminated_length": 1506.0, "completions/mean_length": 521.40625, "completions/mean_terminated_length": 521.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4155130423605442, "epoch": 0.00567, "frac_reward_zero_std": 0.5, "grad_norm": 0.09766510128974915, "kl": 0.4768945798277855, "learning_rate": 3.214543987474504e-05, "loss": -0.0007, "num_tokens": 12134313.0, "reward": 0.9317415952682495, "reward_std": 0.193063884973526, "rewards/rollout_reward_func/mean": 0.9317415952682495, "rewards/rollout_reward_func/std": 0.26864174008369446, "sampling/importance_sampling_ratio/max": 1.234529972076416, "sampling/importance_sampling_ratio/mean": 0.9976354837417603, "sampling/importance_sampling_ratio/min": 0.8276654481887817, "sampling/sampling_logp_difference/max": 0.18911504745483398, "sampling/sampling_logp_difference/mean": 0.008160065859556198, "step": 567, "step_time": 10.213662353000473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.42637862265110016, "epoch": 0.00568, "grad_norm": 0.038058292120695114, "kl": 0.5169894844293594, "learning_rate": 3.208858957933274e-05, "loss": -0.0011, "step": 568, "step_time": 5.069229577999977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1623.0, "completions/max_terminated_length": 1623.0, "completions/mean_length": 442.09375, "completions/mean_terminated_length": 427.70965576171875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5644087493419647, "epoch": 0.00569, "frac_reward_zero_std": 0.0, "grad_norm": 0.5098052620887756, "kl": 0.515187680721283, "learning_rate": 3.2031709034444315e-05, "loss": 0.0707, "num_tokens": 12175931.0, "reward": 0.6837495565414429, "reward_std": 0.62574303150177, "rewards/rollout_reward_func/mean": 0.6837495565414429, "rewards/rollout_reward_func/std": 0.6424649953842163, "sampling/importance_sampling_ratio/max": 1.2839444875717163, "sampling/importance_sampling_ratio/mean": 0.9769929051399231, "sampling/importance_sampling_ratio/min": 1.4714591767273077e-09, "sampling/sampling_logp_difference/max": 20.390804290771484, "sampling/sampling_logp_difference/mean": 0.18781882524490356, "step": 569, "step_time": 9.480088152999997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.024671053513884544, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.024671053513884544, "entropy": 0.5846715532243252, "epoch": 0.0057, "grad_norm": 0.21348436176776886, "kl": 0.516626138240099, "learning_rate": 3.197479867972902e-05, "loss": 0.0613, "step": 570, "step_time": 5.2605685430007725 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1650.0, "completions/max_terminated_length": 1650.0, "completions/mean_length": 640.5, "completions/mean_terminated_length": 640.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4847934879362583, "epoch": 0.00571, "frac_reward_zero_std": 0.0, "grad_norm": 0.12151510268449783, "kl": 0.4768736660480499, "learning_rate": 3.191785895506648e-05, "loss": 0.0026, "num_tokens": 12226091.0, "reward": 0.6866567134857178, "reward_std": 0.6221722960472107, "rewards/rollout_reward_func/mean": 0.6866567134857178, "rewards/rollout_reward_func/std": 0.6390647292137146, "sampling/importance_sampling_ratio/max": 1.0979670286178589, "sampling/importance_sampling_ratio/mean": 0.9896626472473145, "sampling/importance_sampling_ratio/min": 0.8469372987747192, "sampling/sampling_logp_difference/max": 0.12269675731658936, "sampling/sampling_logp_difference/mean": 0.007781230844557285, "step": 571, "step_time": 10.403405858998667 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.506165400147438, "epoch": 0.00572, "grad_norm": 0.06464884430170059, "kl": 0.46445992588996887, "learning_rate": 3.186089030056338e-05, "loss": 0.002, "step": 572, "step_time": 5.387988760001463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 352.71875, "completions/mean_terminated_length": 352.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4123638980090618, "epoch": 0.00573, "frac_reward_zero_std": 0.25, "grad_norm": 0.09213770925998688, "kl": 0.661315493285656, "learning_rate": 3.180389315654998e-05, "loss": 0.0001, "num_tokens": 12264411.0, "reward": 0.8984953165054321, "reward_std": 0.28709858655929565, "rewards/rollout_reward_func/mean": 0.8984953165054321, "rewards/rollout_reward_func/std": 0.32069429755210876, "sampling/importance_sampling_ratio/max": 1.0856057405471802, "sampling/importance_sampling_ratio/mean": 0.9984844923019409, "sampling/importance_sampling_ratio/min": 0.9197169542312622, "sampling/sampling_logp_difference/max": 0.10216856002807617, "sampling/sampling_logp_difference/mean": 0.004332900047302246, "step": 573, "step_time": 8.753526705000695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.423136368393898, "epoch": 0.00574, "grad_norm": 0.08540187776088715, "kl": 0.6504442952573299, "learning_rate": 3.174686796357676e-05, "loss": -0.0003, "step": 574, "step_time": 5.730858199998693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 410.46875, "completions/mean_terminated_length": 410.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6608035881072283, "epoch": 0.00575, "frac_reward_zero_std": 0.5, "grad_norm": 0.1366148740053177, "kl": 0.5743981748819351, "learning_rate": 3.168981516241099e-05, "loss": 0.0082, "num_tokens": 12305224.0, "reward": 0.7467158436775208, "reward_std": 0.4210696220397949, "rewards/rollout_reward_func/mean": 0.7467158436775208, "rewards/rollout_reward_func/std": 0.6241487860679626, "sampling/importance_sampling_ratio/max": 1.058247685432434, "sampling/importance_sampling_ratio/mean": 0.9541581869125366, "sampling/importance_sampling_ratio/min": 6.007283022803692e-15, "sampling/sampling_logp_difference/max": 28.16082763671875, "sampling/sampling_logp_difference/mean": 0.3300233781337738, "step": 575, "step_time": 8.76605225000003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.6730016060173512, "epoch": 0.00576, "grad_norm": 0.1340959370136261, "kl": 0.5741300731897354, "learning_rate": 3.163273519403334e-05, "loss": 0.0077, "step": 576, "step_time": 4.804675080000379 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1758.0, "completions/max_terminated_length": 1758.0, "completions/mean_length": 516.25, "completions/mean_terminated_length": 516.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6393284127116203, "epoch": 0.00577, "frac_reward_zero_std": 0.0, "grad_norm": 0.2235678732395172, "kl": 0.49273017421364784, "learning_rate": 3.157562849963446e-05, "loss": -0.0028, "num_tokens": 12351268.0, "reward": 0.5245541334152222, "reward_std": 0.763297438621521, "rewards/rollout_reward_func/mean": 0.5245541334152222, "rewards/rollout_reward_func/std": 0.7555153965950012, "sampling/importance_sampling_ratio/max": 1.2586884498596191, "sampling/importance_sampling_ratio/mean": 0.9746410846710205, "sampling/importance_sampling_ratio/min": 2.3042399988071338e-08, "sampling/sampling_logp_difference/max": 17.635059356689453, "sampling/sampling_logp_difference/mean": 0.15788152813911438, "step": 577, "step_time": 9.904402458999357 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.012500000186264515, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012500000186264515, "entropy": 0.6420818828046322, "epoch": 0.00578, "grad_norm": 0.1367652267217636, "kl": 0.5285598412156105, "learning_rate": 3.151849552061157e-05, "loss": -0.004, "step": 578, "step_time": 5.964555905001362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 499.34375, "completions/mean_terminated_length": 499.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6080514304339886, "epoch": 0.00579, "frac_reward_zero_std": 0.0, "grad_norm": 0.12540005147457123, "kl": 0.27797746658325195, "learning_rate": 3.146133669856507e-05, "loss": 0.0061, "num_tokens": 12395793.0, "reward": 0.8400832414627075, "reward_std": 0.45231306552886963, "rewards/rollout_reward_func/mean": 0.8400832414627075, "rewards/rollout_reward_func/std": 0.45130276679992676, "sampling/importance_sampling_ratio/max": 1.1198567152023315, "sampling/importance_sampling_ratio/mean": 1.0064997673034668, "sampling/importance_sampling_ratio/min": 0.8537509441375732, "sampling/sampling_logp_difference/max": 0.11326026916503906, "sampling/sampling_logp_difference/mean": 0.010830363258719444, "step": 579, "step_time": 9.362523710998175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.602245569229126, "epoch": 0.0058, "grad_norm": 0.12005381286144257, "kl": 0.2763416599482298, "learning_rate": 3.1404152475295105e-05, "loss": 0.0047, "step": 580, "step_time": 5.161043709999831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2281.0, "completions/max_terminated_length": 2281.0, "completions/mean_length": 425.0625, "completions/mean_terminated_length": 425.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7401918396353722, "epoch": 0.00581, "frac_reward_zero_std": 0.0, "grad_norm": 0.2083757519721985, "kl": 0.5610102973878384, "learning_rate": 3.134694329279813e-05, "loss": -0.0268, "num_tokens": 12436560.0, "reward": 0.6791177988052368, "reward_std": 0.6722683906555176, "rewards/rollout_reward_func/mean": 0.6791177988052368, "rewards/rollout_reward_func/std": 0.6477085947990417, "sampling/importance_sampling_ratio/max": 1.0916416645050049, "sampling/importance_sampling_ratio/mean": 0.984660267829895, "sampling/importance_sampling_ratio/min": 0.6940886974334717, "sampling/sampling_logp_difference/max": 0.2283477783203125, "sampling/sampling_logp_difference/mean": 0.02102738246321678, "step": 581, "step_time": 11.93707867699959 }, { "clip_ratio/high_max": 0.009785353671759367, "clip_ratio/high_mean": 0.0048926768358796835, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015309343812987208, "entropy": 0.7327540032565594, "epoch": 0.00582, "grad_norm": 0.1625490039587021, "kl": 0.5680489130318165, "learning_rate": 3.128970959326357e-05, "loss": -0.0289, "step": 582, "step_time": 6.350104418999763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1481.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 452.1875, "completions/mean_terminated_length": 452.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.501691497862339, "epoch": 0.00583, "frac_reward_zero_std": 0.25, "grad_norm": 0.1479087769985199, "kl": 0.3027223628014326, "learning_rate": 3.123245181907028e-05, "loss": -0.0011, "num_tokens": 12479847.0, "reward": 0.7717241048812866, "reward_std": 0.4319953918457031, "rewards/rollout_reward_func/mean": 0.7717241048812866, "rewards/rollout_reward_func/std": 0.5015233159065247, "sampling/importance_sampling_ratio/max": 1.086424469947815, "sampling/importance_sampling_ratio/mean": 0.9991668462753296, "sampling/importance_sampling_ratio/min": 0.9265663623809814, "sampling/sampling_logp_difference/max": 0.0828787088394165, "sampling/sampling_logp_difference/mean": 0.005055688321590424, "step": 583, "step_time": 9.247478971001328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.49142682179808617, "epoch": 0.00584, "grad_norm": 0.06352712213993073, "kl": 0.30181758292019367, "learning_rate": 3.117517041278326e-05, "loss": -0.0019, "step": 584, "step_time": 4.988868404999266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1704.0, "completions/max_terminated_length": 1704.0, "completions/mean_length": 455.6875, "completions/mean_terminated_length": 455.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5886896839365363, "epoch": 0.00585, "frac_reward_zero_std": 0.0, "grad_norm": 0.13919532299041748, "kl": 0.5166134927421808, "learning_rate": 3.111786581715013e-05, "loss": 0.0043, "num_tokens": 12523400.0, "reward": 0.6700101494789124, "reward_std": 0.5500497817993164, "rewards/rollout_reward_func/mean": 0.6700101494789124, "rewards/rollout_reward_func/std": 0.5583261251449585, "sampling/importance_sampling_ratio/max": 1.0595790147781372, "sampling/importance_sampling_ratio/mean": 0.9581432342529297, "sampling/importance_sampling_ratio/min": 1.5639598505146068e-09, "sampling/sampling_logp_difference/max": 20.084232330322266, "sampling/sampling_logp_difference/mean": 0.16901150345802307, "step": 585, "step_time": 10.170830973999728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 0.5761001566424966, "epoch": 0.00586, "grad_norm": 0.09881673753261566, "kl": 0.5217230133712292, "learning_rate": 3.106053847509778e-05, "loss": 0.004, "step": 586, "step_time": 5.287318641001548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1641.0, "completions/max_terminated_length": 1641.0, "completions/mean_length": 538.0625, "completions/mean_terminated_length": 538.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.520511981099844, "epoch": 0.00587, "frac_reward_zero_std": 0.5, "grad_norm": 0.10017482191324234, "kl": 0.3669740930199623, "learning_rate": 3.100318882972889e-05, "loss": 0.0073, "num_tokens": 12569783.0, "reward": 0.8437544107437134, "reward_std": 0.3017948865890503, "rewards/rollout_reward_func/mean": 0.8437544107437134, "rewards/rollout_reward_func/std": 0.44129276275634766, "sampling/importance_sampling_ratio/max": 1.0681722164154053, "sampling/importance_sampling_ratio/mean": 1.0045231580734253, "sampling/importance_sampling_ratio/min": 0.9145300388336182, "sampling/sampling_logp_difference/max": 0.0934455394744873, "sampling/sampling_logp_difference/mean": 0.006206669844686985, "step": 587, "step_time": 9.696036195998204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.5152311250567436, "epoch": 0.00588, "grad_norm": 0.08958832919597626, "kl": 0.36654483154416084, "learning_rate": 3.094581732431854e-05, "loss": 0.0067, "step": 588, "step_time": 6.13678265100134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1740.0, "completions/max_terminated_length": 1740.0, "completions/mean_length": 527.28125, "completions/mean_terminated_length": 527.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4482666216790676, "epoch": 0.00589, "frac_reward_zero_std": 0.0, "grad_norm": 0.10306570678949356, "kl": 0.5773727148771286, "learning_rate": 3.0888424402310766e-05, "loss": 0.0039, "num_tokens": 12615183.0, "reward": 0.7152698636054993, "reward_std": 0.5860652923583984, "rewards/rollout_reward_func/mean": 0.7152698636054993, "rewards/rollout_reward_func/std": 0.5820181369781494, "sampling/importance_sampling_ratio/max": 1.099186897277832, "sampling/importance_sampling_ratio/mean": 0.9942620992660522, "sampling/importance_sampling_ratio/min": 0.9068278670310974, "sampling/sampling_logp_difference/max": 0.09771919250488281, "sampling/sampling_logp_difference/mean": 0.0055274502374231815, "step": 589, "step_time": 9.816971997001929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "entropy": 0.44227445498108864, "epoch": 0.0059, "grad_norm": 0.08669721335172653, "kl": 0.5730516836047173, "learning_rate": 3.0831010507315155e-05, "loss": 0.003, "step": 590, "step_time": 5.4475504430010915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1641.0, "completions/max_terminated_length": 1641.0, "completions/mean_length": 360.5625, "completions/mean_terminated_length": 360.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3384764548391104, "epoch": 0.00591, "frac_reward_zero_std": 0.25, "grad_norm": 0.09179352968931198, "kl": 0.516386884264648, "learning_rate": 3.0773576083103386e-05, "loss": 0.0096, "num_tokens": 12652938.0, "reward": 0.902368426322937, "reward_std": 0.27614378929138184, "rewards/rollout_reward_func/mean": 0.902368426322937, "rewards/rollout_reward_func/std": 0.3087669909000397, "sampling/importance_sampling_ratio/max": 1.1015880107879639, "sampling/importance_sampling_ratio/mean": 1.0030512809753418, "sampling/importance_sampling_ratio/min": 0.8795523047447205, "sampling/sampling_logp_difference/max": 0.12045025825500488, "sampling/sampling_logp_difference/mean": 0.007553433068096638, "step": 591, "step_time": 10.033407161999094 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.33482577838003635, "epoch": 0.00592, "grad_norm": 0.0952804908156395, "kl": 0.5288467295467854, "learning_rate": 3.0716121573605826e-05, "loss": 0.0099, "step": 592, "step_time": 5.181197704000624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 555.3125, "completions/mean_terminated_length": 555.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.504514642059803, "epoch": 0.00593, "frac_reward_zero_std": 0.25, "grad_norm": 0.09838860481977463, "kl": 0.3883006116375327, "learning_rate": 3.065864742290809e-05, "loss": -0.0075, "num_tokens": 12699896.0, "reward": 0.8037946224212646, "reward_std": 0.38145798444747925, "rewards/rollout_reward_func/mean": 0.8037946224212646, "rewards/rollout_reward_func/std": 0.4848446249961853, "sampling/importance_sampling_ratio/max": 1.1356154680252075, "sampling/importance_sampling_ratio/mean": 1.0168426036834717, "sampling/importance_sampling_ratio/min": 0.9226841926574707, "sampling/sampling_logp_difference/max": 0.12740755081176758, "sampling/sampling_logp_difference/mean": 0.009253796190023422, "step": 593, "step_time": 9.391119271998832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.502670168876648, "epoch": 0.00594, "grad_norm": 0.06462952494621277, "kl": 0.38608703203499317, "learning_rate": 3.060115407524758e-05, "loss": -0.0079, "step": 594, "step_time": 5.859306608999759 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1328.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 403.78125, "completions/mean_terminated_length": 403.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6413045972585678, "epoch": 0.00595, "frac_reward_zero_std": 0.25, "grad_norm": 0.14148499071598053, "kl": 0.41145807690918446, "learning_rate": 3.0543641975010134e-05, "loss": -0.0231, "num_tokens": 12742264.0, "reward": 0.7042146921157837, "reward_std": 0.4486362636089325, "rewards/rollout_reward_func/mean": 0.7042146921157837, "rewards/rollout_reward_func/std": 0.5981276631355286, "sampling/importance_sampling_ratio/max": 1.059429407119751, "sampling/importance_sampling_ratio/mean": 0.9807701110839844, "sampling/importance_sampling_ratio/min": 0.6806769371032715, "sampling/sampling_logp_difference/max": 0.29903197288513184, "sampling/sampling_logp_difference/mean": 0.013181829825043678, "step": 595, "step_time": 8.88261481400059 }, { "clip_ratio/high_max": 0.014423076994717121, "clip_ratio/high_mean": 0.0072115384973585606, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0072115384973585606, "entropy": 0.6430346108973026, "epoch": 0.00596, "grad_norm": 0.12684553861618042, "kl": 0.41644865833222866, "learning_rate": 3.048611156672647e-05, "loss": -0.0239, "step": 596, "step_time": 4.697782388999258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 306.5625, "completions/mean_terminated_length": 306.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.40994327887892723, "epoch": 0.00597, "frac_reward_zero_std": 0.25, "grad_norm": 0.0868150144815445, "kl": 0.37658828496932983, "learning_rate": 3.0428563295068854e-05, "loss": -0.0011, "num_tokens": 12779661.0, "reward": 0.619807779788971, "reward_std": 0.6082636117935181, "rewards/rollout_reward_func/mean": 0.619807779788971, "rewards/rollout_reward_func/std": 0.7064664363861084, "sampling/importance_sampling_ratio/max": 1.0950103998184204, "sampling/importance_sampling_ratio/mean": 0.9991506934165955, "sampling/importance_sampling_ratio/min": 0.9144590497016907, "sampling/sampling_logp_difference/max": 0.09076833724975586, "sampling/sampling_logp_difference/mean": 0.0041608139872550964, "step": 597, "step_time": 8.823957671001153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.41098373755812645, "epoch": 0.00598, "grad_norm": 0.08424228429794312, "kl": 0.37633007019758224, "learning_rate": 3.037099760484762e-05, "loss": -0.0018, "step": 598, "step_time": 5.284293427000193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 274.0625, "completions/mean_terminated_length": 274.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4251229241490364, "epoch": 0.00599, "frac_reward_zero_std": 0.25, "grad_norm": 0.07931900024414062, "kl": 0.45578944496810436, "learning_rate": 3.031341494100772e-05, "loss": -0.0076, "num_tokens": 12814855.0, "reward": 0.834463894367218, "reward_std": 0.40080738067626953, "rewards/rollout_reward_func/mean": 0.834463894367218, "rewards/rollout_reward_func/std": 0.4647304117679596, "sampling/importance_sampling_ratio/max": 1.1055585145950317, "sampling/importance_sampling_ratio/mean": 0.9983465671539307, "sampling/importance_sampling_ratio/min": 0.8689449429512024, "sampling/sampling_logp_difference/max": 0.09982478618621826, "sampling/sampling_logp_difference/mean": 0.00707439798861742, "step": 599, "step_time": 8.624751502998151 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.4275769218802452, "epoch": 0.006, "grad_norm": 0.056961484253406525, "kl": 0.4322901824489236, "learning_rate": 3.0255815748625327e-05, "loss": -0.0079, "step": 600, "step_time": 4.86098822599979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1506.0, "completions/max_terminated_length": 1506.0, "completions/mean_length": 547.3125, "completions/mean_terminated_length": 547.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5691350176930428, "epoch": 0.00601, "frac_reward_zero_std": 0.0, "grad_norm": 0.1424078494310379, "kl": 0.4461846426129341, "learning_rate": 3.019820047290434e-05, "loss": 0.0032, "num_tokens": 12862760.0, "reward": 0.5247148275375366, "reward_std": 0.7231556177139282, "rewards/rollout_reward_func/mean": 0.5247148275375366, "rewards/rollout_reward_func/std": 0.7148227095603943, "sampling/importance_sampling_ratio/max": 1.2160152196884155, "sampling/importance_sampling_ratio/mean": 1.011520266532898, "sampling/importance_sampling_ratio/min": 0.907426118850708, "sampling/sampling_logp_difference/max": 0.12141132354736328, "sampling/sampling_logp_difference/mean": 0.009865621104836464, "step": 601, "step_time": 9.749405386000944 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.575811579823494, "epoch": 0.00602, "grad_norm": 0.0820920392870903, "kl": 0.4647238105535507, "learning_rate": 3.0140569559173e-05, "loss": 0.0023, "step": 602, "step_time": 4.9680079140025555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1650.0, "completions/max_terminated_length": 1650.0, "completions/mean_length": 547.90625, "completions/mean_terminated_length": 547.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5715395659208298, "epoch": 0.00603, "frac_reward_zero_std": 0.0, "grad_norm": 0.09343595057725906, "kl": 0.42891627736389637, "learning_rate": 3.0082923452880388e-05, "loss": -0.0022, "num_tokens": 12910261.0, "reward": 0.653104305267334, "reward_std": 0.6500746011734009, "rewards/rollout_reward_func/mean": 0.653104305267334, "rewards/rollout_reward_func/std": 0.648777425289154, "sampling/importance_sampling_ratio/max": 1.0754069089889526, "sampling/importance_sampling_ratio/mean": 0.9937113523483276, "sampling/importance_sampling_ratio/min": 0.8960033059120178, "sampling/sampling_logp_difference/max": 0.10288715362548828, "sampling/sampling_logp_difference/mean": 0.007913121953606606, "step": 603, "step_time": 9.888332751000235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.5738043859601021, "epoch": 0.00604, "grad_norm": 0.07753729075193405, "kl": 0.42548897862434387, "learning_rate": 3.0025262599593042e-05, "loss": -0.0029, "step": 604, "step_time": 5.319296497998039 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1560.0, "completions/max_terminated_length": 1560.0, "completions/mean_length": 422.28125, "completions/mean_terminated_length": 422.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5191176645457745, "epoch": 0.00605, "frac_reward_zero_std": 0.0, "grad_norm": 0.19083406031131744, "kl": 0.33550788275897503, "learning_rate": 2.996758744499148e-05, "loss": 0.0003, "num_tokens": 12952982.0, "reward": 0.616969108581543, "reward_std": 0.6397192478179932, "rewards/rollout_reward_func/mean": 0.616969108581543, "rewards/rollout_reward_func/std": 0.710909366607666, "sampling/importance_sampling_ratio/max": 1.0759576559066772, "sampling/importance_sampling_ratio/mean": 0.998822808265686, "sampling/importance_sampling_ratio/min": 0.9395571351051331, "sampling/sampling_logp_difference/max": 0.07026863098144531, "sampling/sampling_logp_difference/mean": 0.006134954746812582, "step": 605, "step_time": 9.77811695499895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.011363636702299118, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011363636702299118, "entropy": 0.5139883402734995, "epoch": 0.00606, "grad_norm": 0.08938684314489365, "kl": 0.3377068694680929, "learning_rate": 2.9909898434866736e-05, "loss": -0.0012, "step": 606, "step_time": 5.091025103000902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1274.0, "completions/max_terminated_length": 1274.0, "completions/mean_length": 408.09375, "completions/mean_terminated_length": 408.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7177777178585529, "epoch": 0.00607, "frac_reward_zero_std": 0.0, "grad_norm": 0.10784514993429184, "kl": 0.6885517444461584, "learning_rate": 2.9852196015116967e-05, "loss": 0.0032, "num_tokens": 12995560.0, "reward": 0.3487059473991394, "reward_std": 0.7434321641921997, "rewards/rollout_reward_func/mean": 0.3487059473991394, "rewards/rollout_reward_func/std": 0.8863958120346069, "sampling/importance_sampling_ratio/max": 1.0894161462783813, "sampling/importance_sampling_ratio/mean": 0.9985247850418091, "sampling/importance_sampling_ratio/min": 0.8560672402381897, "sampling/sampling_logp_difference/max": 0.11421680450439453, "sampling/sampling_logp_difference/mean": 0.009432528167963028, "step": 607, "step_time": 8.386096500999884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.7138000652194023, "epoch": 0.00608, "grad_norm": 0.10827646404504776, "kl": 0.7162043806165457, "learning_rate": 2.979448063174396e-05, "loss": 0.0028, "step": 608, "step_time": 5.186202984000374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1677.0, "completions/max_terminated_length": 1677.0, "completions/mean_length": 423.5, "completions/mean_terminated_length": 423.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4271097928285599, "epoch": 0.00609, "frac_reward_zero_std": 0.25, "grad_norm": 0.12318001687526703, "kl": 0.4253476168960333, "learning_rate": 2.973675273084971e-05, "loss": 0.0142, "num_tokens": 13037054.0, "reward": 0.7428580522537231, "reward_std": 0.43809574842453003, "rewards/rollout_reward_func/mean": 0.7428580522537231, "rewards/rollout_reward_func/std": 0.5157787203788757, "sampling/importance_sampling_ratio/max": 1.0403938293457031, "sampling/importance_sampling_ratio/mean": 0.9892637729644775, "sampling/importance_sampling_ratio/min": 0.911882758140564, "sampling/sampling_logp_difference/max": 0.15670394897460938, "sampling/sampling_logp_difference/mean": 0.006880485452711582, "step": 609, "step_time": 9.58389119000094 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.42188177071511745, "epoch": 0.0061, "grad_norm": 0.07836359739303589, "kl": 0.42856265790760517, "learning_rate": 2.967901275863296e-05, "loss": 0.0136, "step": 610, "step_time": 5.246965765999448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 473.65625, "completions/mean_terminated_length": 473.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7589611411094666, "epoch": 0.00611, "frac_reward_zero_std": 0.0, "grad_norm": 0.23587100207805634, "kl": 0.5267760939896107, "learning_rate": 2.9621261161385757e-05, "loss": 0.0093, "num_tokens": 13081393.0, "reward": 0.803894579410553, "reward_std": 0.48921704292297363, "rewards/rollout_reward_func/mean": 0.803894579410553, "rewards/rollout_reward_func/std": 0.48086225986480713, "sampling/importance_sampling_ratio/max": 1.0960631370544434, "sampling/importance_sampling_ratio/mean": 0.9776537418365479, "sampling/importance_sampling_ratio/min": 0.7485916614532471, "sampling/sampling_logp_difference/max": 0.1954936981201172, "sampling/sampling_logp_difference/mean": 0.01172509416937828, "step": 611, "step_time": 8.921076436000476 }, { "clip_ratio/high_max": 0.029647436924278736, "clip_ratio/high_mean": 0.014823718462139368, "clip_ratio/low_mean": 0.02223557699471712, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0370592949911952, "entropy": 0.7544623985886574, "epoch": 0.00612, "grad_norm": 0.1458652764558792, "kl": 0.5131476167589426, "learning_rate": 2.9563498385490017e-05, "loss": 0.0066, "step": 612, "step_time": 5.242380847000277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1578.0, "completions/max_terminated_length": 1578.0, "completions/mean_length": 365.21875, "completions/mean_terminated_length": 365.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6418046578764915, "epoch": 0.00613, "frac_reward_zero_std": 0.0, "grad_norm": 0.12356330454349518, "kl": 0.5000123083591461, "learning_rate": 2.950572487741403e-05, "loss": -0.0088, "num_tokens": 13122030.0, "reward": 0.6171623468399048, "reward_std": 0.723473072052002, "rewards/rollout_reward_func/mean": 0.6171623468399048, "rewards/rollout_reward_func/std": 0.7088453769683838, "sampling/importance_sampling_ratio/max": 1.111025333404541, "sampling/importance_sampling_ratio/mean": 0.9994046092033386, "sampling/importance_sampling_ratio/min": 0.9245622158050537, "sampling/sampling_logp_difference/max": 0.12442588806152344, "sampling/sampling_logp_difference/mean": 0.00968586839735508, "step": 613, "step_time": 9.022503465999762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6407427974045277, "epoch": 0.00614, "grad_norm": 0.12368261814117432, "kl": 0.4953759703785181, "learning_rate": 2.944794108370907e-05, "loss": -0.0096, "step": 614, "step_time": 4.9605884320008045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1632.0, "completions/max_terminated_length": 1632.0, "completions/mean_length": 369.65625, "completions/mean_terminated_length": 369.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4006569907069206, "epoch": 0.00615, "frac_reward_zero_std": 0.5, "grad_norm": 0.08050976693630219, "kl": 0.4273281116038561, "learning_rate": 2.9390147451005878e-05, "loss": -0.0015, "num_tokens": 13161839.0, "reward": 0.8093158006668091, "reward_std": 0.37371525168418884, "rewards/rollout_reward_func/mean": 0.8093158006668091, "rewards/rollout_reward_func/std": 0.5383738875389099, "sampling/importance_sampling_ratio/max": 1.0740777254104614, "sampling/importance_sampling_ratio/mean": 0.9966112375259399, "sampling/importance_sampling_ratio/min": 0.9186843037605286, "sampling/sampling_logp_difference/max": 0.08527398109436035, "sampling/sampling_logp_difference/mean": 0.006293640937656164, "step": 615, "step_time": 9.71281879900107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.40008807368576527, "epoch": 0.00616, "grad_norm": 0.0802089124917984, "kl": 0.4280249886214733, "learning_rate": 2.9332344426011278e-05, "loss": -0.0015, "step": 616, "step_time": 5.127886775000661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 566.3125, "completions/mean_terminated_length": 557.6774291992188, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7057352848351002, "epoch": 0.00617, "frac_reward_zero_std": 0.0, "grad_norm": 0.14989297091960907, "kl": 0.7584332153201103, "learning_rate": 2.9274532455504666e-05, "loss": -0.0177, "num_tokens": 13209462.0, "reward": 0.7659485340118408, "reward_std": 0.4661831557750702, "rewards/rollout_reward_func/mean": 0.7659485340118408, "rewards/rollout_reward_func/std": 0.4495016038417816, "sampling/importance_sampling_ratio/max": 1.1118388175964355, "sampling/importance_sampling_ratio/mean": 0.9902971386909485, "sampling/importance_sampling_ratio/min": 0.7871984243392944, "sampling/sampling_logp_difference/max": 0.1581587791442871, "sampling/sampling_logp_difference/mean": 0.017192410305142403, "step": 617, "step_time": 9.808198737998282 }, { "clip_ratio/high_max": 0.00657894741743803, "clip_ratio/high_mean": 0.003289473708719015, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003289473708719015, "entropy": 0.7052802257239819, "epoch": 0.00618, "grad_norm": 0.14489416778087616, "kl": 0.763823963701725, "learning_rate": 2.9216711986334594e-05, "loss": -0.0185, "step": 618, "step_time": 5.6863293620008335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1364.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 393.0, "completions/mean_terminated_length": 393.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4863830730319023, "epoch": 0.00619, "frac_reward_zero_std": 0.0, "grad_norm": 0.12885260581970215, "kl": 0.5892898049205542, "learning_rate": 2.9158883465415297e-05, "loss": 0.0002, "num_tokens": 13250751.0, "reward": 0.5459235310554504, "reward_std": 0.6902710199356079, "rewards/rollout_reward_func/mean": 0.5459235310554504, "rewards/rollout_reward_func/std": 0.6814104914665222, "sampling/importance_sampling_ratio/max": 1.1560460329055786, "sampling/importance_sampling_ratio/mean": 1.0135465860366821, "sampling/importance_sampling_ratio/min": 0.8835705518722534, "sampling/sampling_logp_difference/max": 0.12421226501464844, "sampling/sampling_logp_difference/mean": 0.01317360159009695, "step": 619, "step_time": 8.937434004001261 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "entropy": 0.4928767215460539, "epoch": 0.0062, "grad_norm": 0.11897242814302444, "kl": 0.5968330595642328, "learning_rate": 2.9101047339723255e-05, "loss": -0.001, "step": 620, "step_time": 4.811779992000083 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 420.0625, "completions/mean_terminated_length": 420.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4869573563337326, "epoch": 0.00621, "frac_reward_zero_std": 0.25, "grad_norm": 0.176227867603302, "kl": 0.49819397926330566, "learning_rate": 2.9043204056293716e-05, "loss": 0.006, "num_tokens": 13293001.0, "reward": 0.7705073952674866, "reward_std": 0.43380892276763916, "rewards/rollout_reward_func/mean": 0.7705073952674866, "rewards/rollout_reward_func/std": 0.5067090392112732, "sampling/importance_sampling_ratio/max": 1.0954996347427368, "sampling/importance_sampling_ratio/mean": 0.9957519769668579, "sampling/importance_sampling_ratio/min": 0.8558043837547302, "sampling/sampling_logp_difference/max": 0.13954877853393555, "sampling/sampling_logp_difference/mean": 0.008638905361294746, "step": 621, "step_time": 8.835664296000687 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02864583395421505, "entropy": 0.49181753769516945, "epoch": 0.00622, "grad_norm": 0.0989910140633583, "kl": 0.4895835258066654, "learning_rate": 2.8985354062217246e-05, "loss": 0.0053, "step": 622, "step_time": 5.580028197000502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1722.0, "completions/max_terminated_length": 1722.0, "completions/mean_length": 384.8125, "completions/mean_terminated_length": 384.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7197863534092903, "epoch": 0.00623, "frac_reward_zero_std": 0.0, "grad_norm": 0.19594210386276245, "kl": 0.6230366211384535, "learning_rate": 2.8927497804636304e-05, "loss": -0.0242, "num_tokens": 13332965.0, "reward": 0.6515341997146606, "reward_std": 0.6772237420082092, "rewards/rollout_reward_func/mean": 0.6515341997146606, "rewards/rollout_reward_func/std": 0.6501209735870361, "sampling/importance_sampling_ratio/max": 1.0946398973464966, "sampling/importance_sampling_ratio/mean": 0.960591733455658, "sampling/importance_sampling_ratio/min": 5.274367595120566e-06, "sampling/sampling_logp_difference/max": 12.148569107055664, "sampling/sampling_logp_difference/mean": 0.12574166059494019, "step": 623, "step_time": 9.714293344001817 }, { "clip_ratio/high_max": 0.009868420660495758, "clip_ratio/high_mean": 0.004934210330247879, "clip_ratio/low_mean": 0.018229166977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.023163377307355404, "entropy": 0.7198676355183125, "epoch": 0.00624, "grad_norm": 0.16341347992420197, "kl": 0.6240034308284521, "learning_rate": 2.8869635730741742e-05, "loss": -0.0262, "step": 624, "step_time": 5.359311576999971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1428.0, "completions/max_terminated_length": 795.0, "completions/mean_length": 481.625, "completions/mean_terminated_length": 438.0333557128906, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8013619296252728, "epoch": 0.00625, "frac_reward_zero_std": 0.0, "grad_norm": 0.30885300040245056, "kl": 0.6822346318513155, "learning_rate": 2.881176828776937e-05, "loss": 0.0134, "num_tokens": 13378410.0, "reward": 0.5566784143447876, "reward_std": 0.7137014269828796, "rewards/rollout_reward_func/mean": 0.5566784143447876, "rewards/rollout_reward_func/std": 0.757149338722229, "sampling/importance_sampling_ratio/max": 1.0989179611206055, "sampling/importance_sampling_ratio/mean": 0.964061975479126, "sampling/importance_sampling_ratio/min": 0.4961639642715454, "sampling/sampling_logp_difference/max": 0.1795966625213623, "sampling/sampling_logp_difference/mean": 0.018946843221783638, "step": 625, "step_time": 9.732599025000127 }, { "clip_ratio/high_max": 0.010273972526192665, "clip_ratio/high_mean": 0.0051369862630963326, "clip_ratio/low_mean": 0.018640351481735706, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02377733774483204, "entropy": 0.8045603930950165, "epoch": 0.00626, "grad_norm": 0.24097101390361786, "kl": 0.6838685758411884, "learning_rate": 2.875389592299651e-05, "loss": 0.0092, "step": 626, "step_time": 4.8307952879995355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 434.5, "completions/mean_terminated_length": 434.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.48684485256671906, "epoch": 0.00627, "frac_reward_zero_std": 0.0, "grad_norm": 0.11421525478363037, "kl": 0.4638411719352007, "learning_rate": 2.8696019083738517e-05, "loss": 0.0025, "num_tokens": 13420049.0, "reward": 0.8114265203475952, "reward_std": 0.5333664417266846, "rewards/rollout_reward_func/mean": 0.8114265203475952, "rewards/rollout_reward_func/std": 0.5340898036956787, "sampling/importance_sampling_ratio/max": 1.0689771175384521, "sampling/importance_sampling_ratio/mean": 0.990003228187561, "sampling/importance_sampling_ratio/min": 0.8724367022514343, "sampling/sampling_logp_difference/max": 0.13772892951965332, "sampling/sampling_logp_difference/mean": 0.008078338578343391, "step": 627, "step_time": 9.097519206999095 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018229166977107525, "entropy": 0.4908955618739128, "epoch": 0.00628, "grad_norm": 0.07936190813779831, "kl": 0.46563735231757164, "learning_rate": 2.8638138217345327e-05, "loss": 0.0023, "step": 628, "step_time": 5.710140464000688 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 486.1875, "completions/mean_terminated_length": 486.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5021084733307362, "epoch": 0.00629, "frac_reward_zero_std": 0.25, "grad_norm": 0.09525078535079956, "kl": 0.6768454611301422, "learning_rate": 2.8580253771198012e-05, "loss": 0.0046, "num_tokens": 13462965.0, "reward": 0.7807223796844482, "reward_std": 0.47522425651550293, "rewards/rollout_reward_func/mean": 0.7807223796844482, "rewards/rollout_reward_func/std": 0.5478250980377197, "sampling/importance_sampling_ratio/max": 1.109971284866333, "sampling/importance_sampling_ratio/mean": 1.0030155181884766, "sampling/importance_sampling_ratio/min": 0.9234203100204468, "sampling/sampling_logp_difference/max": 0.10630989074707031, "sampling/sampling_logp_difference/mean": 0.0074302321299910545, "step": 629, "step_time": 9.516410457999882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5117664933204651, "epoch": 0.0063, "grad_norm": 0.08659612387418747, "kl": 0.6507186740636826, "learning_rate": 2.85223661927053e-05, "loss": 0.0042, "step": 630, "step_time": 5.063330372001474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1668.0, "completions/max_terminated_length": 1668.0, "completions/mean_length": 615.1875, "completions/mean_terminated_length": 615.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.857459269464016, "epoch": 0.00631, "frac_reward_zero_std": 0.0, "grad_norm": 0.11946825683116913, "kl": 0.887879129499197, "learning_rate": 2.8464475929300158e-05, "loss": 0.0121, "num_tokens": 13512797.0, "reward": 0.6532071828842163, "reward_std": 0.6255672574043274, "rewards/rollout_reward_func/mean": 0.6532071828842163, "rewards/rollout_reward_func/std": 0.6522220969200134, "sampling/importance_sampling_ratio/max": 1.1253125667572021, "sampling/importance_sampling_ratio/mean": 0.9411844611167908, "sampling/importance_sampling_ratio/min": 1.2002272209410934e-14, "sampling/sampling_logp_difference/max": 18.921184539794922, "sampling/sampling_logp_difference/mean": 0.3349829316139221, "step": 631, "step_time": 10.07367118699949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.8665166050195694, "epoch": 0.00632, "grad_norm": 0.11494209617376328, "kl": 0.8955479543656111, "learning_rate": 2.8406583428436275e-05, "loss": 0.011, "step": 632, "step_time": 5.889669369999865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2164.0, "completions/max_terminated_length": 2164.0, "completions/mean_length": 416.8125, "completions/mean_terminated_length": 416.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6383174583315849, "epoch": 0.00633, "frac_reward_zero_std": 0.25, "grad_norm": 0.24835091829299927, "kl": 1.005930669605732, "learning_rate": 2.8348689137584644e-05, "loss": 0.0071, "num_tokens": 13553650.0, "reward": 0.6588102579116821, "reward_std": 0.5754861235618591, "rewards/rollout_reward_func/mean": 0.6588102579116821, "rewards/rollout_reward_func/std": 0.7381277084350586, "sampling/importance_sampling_ratio/max": 1.1651555299758911, "sampling/importance_sampling_ratio/mean": 0.9851903915405273, "sampling/importance_sampling_ratio/min": 0.6526539325714111, "sampling/sampling_logp_difference/max": 0.14830303192138672, "sampling/sampling_logp_difference/mean": 0.013200982473790646, "step": 633, "step_time": 10.8360227080002 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 0.6520508117973804, "epoch": 0.00634, "grad_norm": 0.12313735485076904, "kl": 0.797771267592907, "learning_rate": 2.8290793504230095e-05, "loss": 0.0042, "step": 634, "step_time": 6.088868143000582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 501.8125, "completions/mean_terminated_length": 501.774169921875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.015525758266449, "epoch": 0.00635, "frac_reward_zero_std": 0.25, "grad_norm": 0.14272905886173248, "kl": 0.6198849156498909, "learning_rate": 2.8232896975867835e-05, "loss": 0.0089, "num_tokens": 13598805.0, "reward": 0.6900439262390137, "reward_std": 0.5458639860153198, "rewards/rollout_reward_func/mean": 0.6900439262390137, "rewards/rollout_reward_func/std": 0.6840822696685791, "sampling/importance_sampling_ratio/max": 1.098848819732666, "sampling/importance_sampling_ratio/mean": 0.9921183586120605, "sampling/importance_sampling_ratio/min": 0.6232786178588867, "sampling/sampling_logp_difference/max": 0.2392582893371582, "sampling/sampling_logp_difference/mean": 0.015653226524591446, "step": 635, "step_time": 10.49013152399948 }, { "clip_ratio/high_max": 0.008928571827709675, "clip_ratio/high_mean": 0.004464285913854837, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010714286006987095, "entropy": 1.0151630192995071, "epoch": 0.00636, "grad_norm": 0.13675272464752197, "kl": 0.6207849178463221, "learning_rate": 2.8174999999999994e-05, "loss": 0.0078, "step": 636, "step_time": 5.027756585000134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1569.0, "completions/max_terminated_length": 1569.0, "completions/mean_length": 564.125, "completions/mean_terminated_length": 579.7000122070312, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0972016528248787, "epoch": 0.00637, "frac_reward_zero_std": 0.0, "grad_norm": 0.6932968497276306, "kl": 1.2824897840619087, "learning_rate": 2.8117103024132155e-05, "loss": 0.019, "num_tokens": 13645377.0, "reward": 0.6819467544555664, "reward_std": 0.6308374404907227, "rewards/rollout_reward_func/mean": 0.6819467544555664, "rewards/rollout_reward_func/std": 0.6473856568336487, "sampling/importance_sampling_ratio/max": 1.2656062841415405, "sampling/importance_sampling_ratio/mean": 1.0122840404510498, "sampling/importance_sampling_ratio/min": 0.7770255208015442, "sampling/sampling_logp_difference/max": 0.18423748016357422, "sampling/sampling_logp_difference/mean": 0.01654386892914772, "step": 637, "step_time": 10.591357500999038 }, { "clip_ratio/high_max": 0.0012499999720603228, "clip_ratio/high_mean": 0.0006249999860301614, "clip_ratio/low_mean": 0.012824820470996201, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013449820457026362, "entropy": 1.09095811098814, "epoch": 0.00638, "grad_norm": 0.4448341131210327, "kl": 1.1638783626258373, "learning_rate": 2.8059206495769902e-05, "loss": 0.0087, "step": 638, "step_time": 5.146915859999353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 479.625, "completions/mean_terminated_length": 479.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6057291701436043, "epoch": 0.00639, "frac_reward_zero_std": 0.0, "grad_norm": 0.15011882781982422, "kl": 1.0337778590619564, "learning_rate": 2.800131086241535e-05, "loss": 0.0007, "num_tokens": 13689983.0, "reward": 0.6543053984642029, "reward_std": 0.6809693574905396, "rewards/rollout_reward_func/mean": 0.6543053984642029, "rewards/rollout_reward_func/std": 0.6964138746261597, "sampling/importance_sampling_ratio/max": 1.057211995124817, "sampling/importance_sampling_ratio/mean": 1.0012651681900024, "sampling/importance_sampling_ratio/min": 0.8926209807395935, "sampling/sampling_logp_difference/max": 0.11516523361206055, "sampling/sampling_logp_difference/mean": 0.006944102235138416, "step": 639, "step_time": 9.330801150998923 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.6177234649658203, "epoch": 0.0064, "grad_norm": 0.1523134559392929, "kl": 0.9891086220741272, "learning_rate": 2.7943416571563723e-05, "loss": -0.0006, "step": 640, "step_time": 4.743995305999306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2348.0, "completions/max_terminated_length": 2348.0, "completions/mean_length": 395.75, "completions/mean_terminated_length": 395.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8083413131535053, "epoch": 0.00641, "frac_reward_zero_std": 0.25, "grad_norm": 0.10742172598838806, "kl": 0.8453612104058266, "learning_rate": 2.7885524070699833e-05, "loss": 0.0163, "num_tokens": 13729064.0, "reward": 0.6497833132743835, "reward_std": 0.5518386960029602, "rewards/rollout_reward_func/mean": 0.6497833132743835, "rewards/rollout_reward_func/std": 0.6563442945480347, "sampling/importance_sampling_ratio/max": 1.0338083505630493, "sampling/importance_sampling_ratio/mean": 0.9590967893600464, "sampling/importance_sampling_ratio/min": 4.256565766525e-09, "sampling/sampling_logp_difference/max": 18.86598014831543, "sampling/sampling_logp_difference/mean": 0.17395895719528198, "step": 641, "step_time": 11.467671235000125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.8080075830221176, "epoch": 0.00642, "grad_norm": 0.10245522111654282, "kl": 0.849089689552784, "learning_rate": 2.7827633807294686e-05, "loss": 0.0159, "step": 642, "step_time": 7.069695621001301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 759.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 393.8125, "completions/mean_terminated_length": 393.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6212741583585739, "epoch": 0.00643, "frac_reward_zero_std": 0.25, "grad_norm": 0.10215840488672256, "kl": 0.8425377011299133, "learning_rate": 2.7769746228801985e-05, "loss": -0.0078, "num_tokens": 13769987.0, "reward": 0.7758016586303711, "reward_std": 0.4857882559299469, "rewards/rollout_reward_func/mean": 0.7758016586303711, "rewards/rollout_reward_func/std": 0.5592657327651978, "sampling/importance_sampling_ratio/max": 1.336084008216858, "sampling/importance_sampling_ratio/mean": 1.0115832090377808, "sampling/importance_sampling_ratio/min": 0.8974045515060425, "sampling/sampling_logp_difference/max": 0.11218404769897461, "sampling/sampling_logp_difference/mean": 0.009678516536951065, "step": 643, "step_time": 7.186557586999697 }, { "clip_ratio/high_max": 0.008928571827709675, "clip_ratio/high_mean": 0.004464285913854837, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "entropy": 0.6337397433817387, "epoch": 0.00644, "grad_norm": 0.08195611834526062, "kl": 0.7992485724389553, "learning_rate": 2.771186178265466e-05, "loss": -0.0079, "step": 644, "step_time": 3.6657310420014255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2173.0, "completions/max_terminated_length": 2173.0, "completions/mean_length": 487.4375, "completions/mean_terminated_length": 482.45159912109375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0900296568870544, "epoch": 0.00645, "frac_reward_zero_std": 0.25, "grad_norm": 0.13356754183769226, "kl": 0.6051291301846504, "learning_rate": 2.7653980916261477e-05, "loss": -0.0009, "num_tokens": 13813326.0, "reward": 0.8427495956420898, "reward_std": 0.44477134943008423, "rewards/rollout_reward_func/mean": 0.8427495956420898, "rewards/rollout_reward_func/std": 0.5159029960632324, "sampling/importance_sampling_ratio/max": 1.5823391675949097, "sampling/importance_sampling_ratio/mean": 1.0122947692871094, "sampling/importance_sampling_ratio/min": 0.8863759636878967, "sampling/sampling_logp_difference/max": 0.35186100006103516, "sampling/sampling_logp_difference/mean": 0.021974224597215652, "step": 645, "step_time": 11.93042401599996 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 1.1035547405481339, "epoch": 0.00646, "grad_norm": 0.11173863708972931, "kl": 0.6116589680314064, "learning_rate": 2.7596104077003483e-05, "loss": -0.0017, "step": 646, "step_time": 6.241976428999806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1418.0, "completions/max_terminated_length": 1418.0, "completions/mean_length": 457.8125, "completions/mean_terminated_length": 474.2069091796875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1415569633245468, "epoch": 0.00647, "frac_reward_zero_std": 0.25, "grad_norm": 0.2128709852695465, "kl": 0.6534502804279327, "learning_rate": 2.7538231712230623e-05, "loss": 0.0172, "num_tokens": 13856638.0, "reward": 0.7510300874710083, "reward_std": 0.5209698677062988, "rewards/rollout_reward_func/mean": 0.7510300874710083, "rewards/rollout_reward_func/std": 0.6136438250541687, "sampling/importance_sampling_ratio/max": 1.1332603693008423, "sampling/importance_sampling_ratio/mean": 0.9847872257232666, "sampling/importance_sampling_ratio/min": 0.466079980134964, "sampling/sampling_logp_difference/max": 0.15185809135437012, "sampling/sampling_logp_difference/mean": 0.020482774823904037, "step": 647, "step_time": 9.333800880000126 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.007894736947491765, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009630848071537912, "entropy": 1.1453294306993484, "epoch": 0.00648, "grad_norm": 0.17278021574020386, "kl": 0.6570774409919977, "learning_rate": 2.7480364269258255e-05, "loss": 0.0153, "step": 648, "step_time": 5.50069900499966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2182.0, "completions/max_terminated_length": 2182.0, "completions/mean_length": 541.5, "completions/mean_terminated_length": 541.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7011066637933254, "epoch": 0.00649, "frac_reward_zero_std": 0.5, "grad_norm": 0.09471003711223602, "kl": 0.5237404555082321, "learning_rate": 2.7422502195363693e-05, "loss": -0.0, "num_tokens": 13902117.0, "reward": 0.8426467180252075, "reward_std": 0.2737981379032135, "rewards/rollout_reward_func/mean": 0.8426467180252075, "rewards/rollout_reward_func/std": 0.44368377327919006, "sampling/importance_sampling_ratio/max": 1.1711316108703613, "sampling/importance_sampling_ratio/mean": 1.005659580230713, "sampling/importance_sampling_ratio/min": 0.917296826839447, "sampling/sampling_logp_difference/max": 0.1213994026184082, "sampling/sampling_logp_difference/mean": 0.008596457540988922, "step": 649, "step_time": 11.448997059000249 }, { "clip_ratio/high_max": 0.004629629664123058, "clip_ratio/high_mean": 0.002314814832061529, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002314814832061529, "entropy": 0.7071905508637428, "epoch": 0.0065, "grad_norm": 0.0820893719792366, "kl": 0.5222240090370178, "learning_rate": 2.7364645937782738e-05, "loss": -0.0007, "step": 650, "step_time": 6.345207302000745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1677.0, "completions/max_terminated_length": 1677.0, "completions/mean_length": 592.65625, "completions/mean_terminated_length": 592.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8206194788217545, "epoch": 0.00651, "frac_reward_zero_std": 0.0, "grad_norm": 0.13450738787651062, "kl": 0.7958608604967594, "learning_rate": 2.7306795943706275e-05, "loss": 0.0257, "num_tokens": 13949817.0, "reward": 0.49897462129592896, "reward_std": 0.6753718852996826, "rewards/rollout_reward_func/mean": 0.49897462129592896, "rewards/rollout_reward_func/std": 0.708142101764679, "sampling/importance_sampling_ratio/max": 1.1142643690109253, "sampling/importance_sampling_ratio/mean": 1.0010110139846802, "sampling/importance_sampling_ratio/min": 0.7740837335586548, "sampling/sampling_logp_difference/max": 0.13983392715454102, "sampling/sampling_logp_difference/mean": 0.009906493127346039, "step": 651, "step_time": 9.936947167000653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.8301103934645653, "epoch": 0.00652, "grad_norm": 0.13324153423309326, "kl": 0.7929955944418907, "learning_rate": 2.7248952660276736e-05, "loss": 0.0248, "step": 652, "step_time": 5.876749145000758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2423.0, "completions/max_terminated_length": 2423.0, "completions/mean_length": 459.65625, "completions/mean_terminated_length": 459.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2283014953136444, "epoch": 0.00653, "frac_reward_zero_std": 0.25, "grad_norm": 0.1803331971168518, "kl": 0.3932484406977892, "learning_rate": 2.7191116534584694e-05, "loss": -0.005, "num_tokens": 13993442.0, "reward": 0.6484390497207642, "reward_std": 0.5652444362640381, "rewards/rollout_reward_func/mean": 0.6484390497207642, "rewards/rollout_reward_func/std": 0.6558213829994202, "sampling/importance_sampling_ratio/max": 1.0612245798110962, "sampling/importance_sampling_ratio/mean": 0.9960914850234985, "sampling/importance_sampling_ratio/min": 0.8089492917060852, "sampling/sampling_logp_difference/max": 0.11570453643798828, "sampling/sampling_logp_difference/mean": 0.012173733673989773, "step": 653, "step_time": 12.102700354999797 }, { "clip_ratio/high_max": 0.0069444444961845875, "clip_ratio/high_mean": 0.0034722222480922937, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034722222480922937, "entropy": 1.2325156182050705, "epoch": 0.00654, "grad_norm": 0.16483159363269806, "kl": 0.3938225358724594, "learning_rate": 2.7133288013665403e-05, "loss": -0.0066, "step": 654, "step_time": 6.696307837999484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2436.0, "completions/max_terminated_length": 2436.0, "completions/mean_length": 571.125, "completions/mean_terminated_length": 571.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.53065774589777, "epoch": 0.00655, "frac_reward_zero_std": 0.0, "grad_norm": 0.5866507291793823, "kl": 0.8260210007429123, "learning_rate": 2.707546754449532e-05, "loss": -0.0584, "num_tokens": 14041215.0, "reward": 0.6212628483772278, "reward_std": 0.6795332431793213, "rewards/rollout_reward_func/mean": 0.6212628483772278, "rewards/rollout_reward_func/std": 0.7028239369392395, "sampling/importance_sampling_ratio/max": 1.8599905967712402, "sampling/importance_sampling_ratio/mean": 1.0590673685073853, "sampling/importance_sampling_ratio/min": 0.813526451587677, "sampling/sampling_logp_difference/max": 0.15545177459716797, "sampling/sampling_logp_difference/mean": 0.023006757721304893, "step": 655, "step_time": 13.03811471500012 }, { "clip_ratio/high_max": 0.041196742095053196, "clip_ratio/high_mean": 0.020598371047526598, "clip_ratio/low_mean": 0.014204545877873898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.034802916925400496, "entropy": 1.541254810988903, "epoch": 0.00656, "grad_norm": 0.38020452857017517, "kl": 0.8384718149900436, "learning_rate": 2.7017655573988713e-05, "loss": -0.0688, "step": 656, "step_time": 6.687485275001563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2972.0, "completions/max_terminated_length": 2972.0, "completions/mean_length": 529.0625, "completions/mean_terminated_length": 519.1612548828125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1787228621542454, "epoch": 0.00657, "frac_reward_zero_std": 0.0, "grad_norm": 0.41373953223228455, "kl": 1.0077118165791035, "learning_rate": 2.6959852548994113e-05, "loss": 0.0678, "num_tokens": 14086805.0, "reward": 0.5057070851325989, "reward_std": 0.735596776008606, "rewards/rollout_reward_func/mean": 0.5057070851325989, "rewards/rollout_reward_func/std": 0.8265308141708374, "sampling/importance_sampling_ratio/max": 1.3977831602096558, "sampling/importance_sampling_ratio/mean": 1.012218713760376, "sampling/importance_sampling_ratio/min": 0.8442020416259766, "sampling/sampling_logp_difference/max": 0.1700754165649414, "sampling/sampling_logp_difference/mean": 0.020416509360074997, "step": 657, "step_time": 13.672226025999407 }, { "clip_ratio/high_max": 0.0234375, "clip_ratio/high_mean": 0.01171875, "clip_ratio/low_mean": 0.027555419132113457, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03927416913211346, "entropy": 1.2260119952261448, "epoch": 0.00658, "grad_norm": 0.307167649269104, "kl": 0.9820808209478855, "learning_rate": 2.690205891629092e-05, "loss": 0.0599, "step": 658, "step_time": 7.710518516000775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 1421.0, "completions/max_terminated_length": 1247.0, "completions/mean_length": 280.1875, "completions/mean_terminated_length": 229.6785888671875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.8902708292007446, "epoch": 0.00659, "frac_reward_zero_std": 0.0, "grad_norm": 0.4071134924888611, "kl": 0.7107551423832774, "learning_rate": 2.684427512258596e-05, "loss": -0.015, "num_tokens": 14122939.0, "reward": 0.7148203253746033, "reward_std": 0.6430315971374512, "rewards/rollout_reward_func/mean": 0.7148203253746033, "rewards/rollout_reward_func/std": 0.6356889605522156, "sampling/importance_sampling_ratio/max": 1.1211047172546387, "sampling/importance_sampling_ratio/mean": 0.9351804256439209, "sampling/importance_sampling_ratio/min": 2.8238371641009508e-09, "sampling/sampling_logp_difference/max": 15.705831527709961, "sampling/sampling_logp_difference/mean": 0.10264220833778381, "step": 659, "step_time": 9.824641962998612 }, { "clip_ratio/high_max": 0.022703490452840924, "clip_ratio/high_mean": 0.011351745226420462, "clip_ratio/low_mean": 0.02747140557039529, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.038823151611723006, "entropy": 1.8988015204668045, "epoch": 0.0066, "grad_norm": 0.25697481632232666, "kl": 0.6951753869652748, "learning_rate": 2.678650161450997e-05, "loss": -0.022, "step": 660, "step_time": 4.766948145999777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 769.0, "completions/max_terminated_length": 769.0, "completions/mean_length": 364.1875, "completions/mean_terminated_length": 374.9032287597656, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.4577197134494781, "epoch": 0.00661, "frac_reward_zero_std": 0.0, "grad_norm": 0.4391370117664337, "kl": 0.7287758477032185, "learning_rate": 2.6728738838614227e-05, "loss": -0.007, "num_tokens": 14163292.0, "reward": 0.7738041877746582, "reward_std": 0.5734278559684753, "rewards/rollout_reward_func/mean": 0.7738041877746582, "rewards/rollout_reward_func/std": 0.5634637475013733, "sampling/importance_sampling_ratio/max": 1.18551766872406, "sampling/importance_sampling_ratio/mean": 0.9943677186965942, "sampling/importance_sampling_ratio/min": 0.8867871165275574, "sampling/sampling_logp_difference/max": 0.3499867916107178, "sampling/sampling_logp_difference/mean": 0.02743874490261078, "step": 661, "step_time": 7.4562769320009465 }, { "clip_ratio/high_max": 0.05152000393718481, "clip_ratio/high_mean": 0.02816384844481945, "clip_ratio/low_mean": 0.030330882407724857, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.05849473178386688, "entropy": 1.4725112169981003, "epoch": 0.00662, "grad_norm": 0.2607904374599457, "kl": 0.7220948375761509, "learning_rate": 2.667098724136703e-05, "loss": -0.0134, "step": 662, "step_time": 4.424637936000181 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 359.53125, "completions/mean_terminated_length": 303.5862121582031, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.393335998058319, "epoch": 0.00663, "frac_reward_zero_std": 0.0, "grad_norm": 0.3379133939743042, "kl": 0.8875260874629021, "learning_rate": 2.6613247269150282e-05, "loss": -0.0718, "num_tokens": 14202181.0, "reward": 0.4805394113063812, "reward_std": 0.762537956237793, "rewards/rollout_reward_func/mean": 0.4805394113063812, "rewards/rollout_reward_func/std": 0.7311950325965881, "sampling/importance_sampling_ratio/max": 1.1765316724777222, "sampling/importance_sampling_ratio/mean": 1.0039348602294922, "sampling/importance_sampling_ratio/min": 0.6795212030410767, "sampling/sampling_logp_difference/max": 0.18822813034057617, "sampling/sampling_logp_difference/mean": 0.023971963673830032, "step": 663, "step_time": 9.072532972000772 }, { "clip_ratio/high_max": 0.017361111473292112, "clip_ratio/high_mean": 0.013368055457249284, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013368055457249284, "entropy": 1.435183346271515, "epoch": 0.00664, "grad_norm": 0.3006400465965271, "kl": 0.8903929218649864, "learning_rate": 2.6555519368256036e-05, "loss": -0.0761, "step": 664, "step_time": 4.776134168000681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 2254.0, "completions/max_terminated_length": 1316.0, "completions/mean_length": 408.15625, "completions/mean_terminated_length": 325.03997802734375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.67374587059021, "epoch": 0.00665, "frac_reward_zero_std": 0.5, "grad_norm": 0.5392087697982788, "kl": 0.7288739830255508, "learning_rate": 2.649780398488303e-05, "loss": 0.0654, "num_tokens": 14244562.0, "reward": 0.5876118540763855, "reward_std": 0.41286391019821167, "rewards/rollout_reward_func/mean": 0.5876118540763855, "rewards/rollout_reward_func/std": 0.7088811993598938, "sampling/importance_sampling_ratio/max": 1.7188539505004883, "sampling/importance_sampling_ratio/mean": 0.9138962030410767, "sampling/importance_sampling_ratio/min": 2.248552984720375e-16, "sampling/sampling_logp_difference/max": 16.787982940673828, "sampling/sampling_logp_difference/mean": 0.20920827984809875, "step": 665, "step_time": 12.501571529000103 }, { "clip_ratio/high_max": 0.009246880654245615, "clip_ratio/high_mean": 0.0046234403271228075, "clip_ratio/low_mean": 0.0055147059028968215, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010138146230019629, "entropy": 2.655212238430977, "epoch": 0.00666, "grad_norm": 0.46830040216445923, "kl": 0.7148131728172302, "learning_rate": 2.6440101565133254e-05, "loss": 0.0542, "step": 666, "step_time": 6.687977091999528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1454.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 418.90625, "completions/mean_terminated_length": 401.2413635253906, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.5864566937088966, "epoch": 0.00667, "frac_reward_zero_std": 0.5, "grad_norm": 0.20882566273212433, "kl": 0.6763225998729467, "learning_rate": 2.638241255500851e-05, "loss": -0.0079, "num_tokens": 14285803.0, "reward": 0.9025489091873169, "reward_std": 0.27563339471817017, "rewards/rollout_reward_func/mean": 0.9025489091873169, "rewards/rollout_reward_func/std": 0.3994756042957306, "sampling/importance_sampling_ratio/max": 1.344700813293457, "sampling/importance_sampling_ratio/mean": 0.9822447299957275, "sampling/importance_sampling_ratio/min": 0.3291591703891754, "sampling/sampling_logp_difference/max": 0.15352106094360352, "sampling/sampling_logp_difference/mean": 0.02695241943001747, "step": 667, "step_time": 9.615987593000682 }, { "clip_ratio/high_max": 0.005310457549057901, "clip_ratio/high_mean": 0.0026552287745289505, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01046772877452895, "entropy": 1.531601034104824, "epoch": 0.00668, "grad_norm": 0.1167457327246666, "kl": 0.6634593438357115, "learning_rate": 2.632473740040694e-05, "loss": -0.0092, "step": 668, "step_time": 4.861015993002184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 2060.0, "completions/max_terminated_length": 2060.0, "completions/mean_length": 326.03125, "completions/mean_terminated_length": 317.370361328125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.5305556319653988, "epoch": 0.00669, "frac_reward_zero_std": 0.0, "grad_norm": 0.3066040277481079, "kl": 0.5632982198148966, "learning_rate": 2.6267076547119603e-05, "loss": -0.0423, "num_tokens": 14323493.0, "reward": 0.4753793478012085, "reward_std": 0.7109097242355347, "rewards/rollout_reward_func/mean": 0.4753793478012085, "rewards/rollout_reward_func/std": 0.6874598860740662, "sampling/importance_sampling_ratio/max": 1.227396011352539, "sampling/importance_sampling_ratio/mean": 0.962302565574646, "sampling/importance_sampling_ratio/min": 9.929890937288022e-11, "sampling/sampling_logp_difference/max": 6.969689846038818, "sampling/sampling_logp_difference/mean": 0.12597346305847168, "step": 669, "step_time": 11.406856760000665 }, { "clip_ratio/high_max": 0.01354166679084301, "clip_ratio/high_mean": 0.006770833395421505, "clip_ratio/low_mean": 0.003289473708719015, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01006030710414052, "entropy": 1.5015221238136292, "epoch": 0.0067, "grad_norm": 0.27476590871810913, "kl": 0.5604269523173571, "learning_rate": 2.6209430440826997e-05, "loss": -0.0454, "step": 670, "step_time": 5.891541323000638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 497.09375, "completions/mean_terminated_length": 484.63336181640625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3076284416019917, "epoch": 0.00671, "frac_reward_zero_std": 0.5, "grad_norm": 0.14092528820037842, "kl": 0.49279915541410446, "learning_rate": 2.6151799527095647e-05, "loss": -0.0255, "num_tokens": 14368743.0, "reward": 0.930097758769989, "reward_std": 0.1977134346961975, "rewards/rollout_reward_func/mean": 0.930097758769989, "rewards/rollout_reward_func/std": 0.27506223320961, "sampling/importance_sampling_ratio/max": 1.1339079141616821, "sampling/importance_sampling_ratio/mean": 1.0054246187210083, "sampling/importance_sampling_ratio/min": 0.5040604472160339, "sampling/sampling_logp_difference/max": 0.14563703536987305, "sampling/sampling_logp_difference/mean": 0.024851636961102486, "step": 671, "step_time": 9.980755476000013 }, { "clip_ratio/high_max": 0.004629629664123058, "clip_ratio/high_mean": 0.002314814832061529, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002314814832061529, "entropy": 1.300974577665329, "epoch": 0.00672, "grad_norm": 0.14383628964424133, "kl": 0.4911116976290941, "learning_rate": 2.6094184251374664e-05, "loss": -0.0267, "step": 672, "step_time": 5.454401121000046 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 2056.0, "completions/max_terminated_length": 2056.0, "completions/mean_length": 530.96875, "completions/mean_terminated_length": 548.6000366210938, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.4012742266058922, "epoch": 0.00673, "frac_reward_zero_std": 0.0, "grad_norm": 0.6270006895065308, "kl": 0.8144995905458927, "learning_rate": 2.603658505899227e-05, "loss": 0.0814, "num_tokens": 14415088.0, "reward": 0.7084897756576538, "reward_std": 0.5686041712760925, "rewards/rollout_reward_func/mean": 0.7084897756576538, "rewards/rollout_reward_func/std": 0.5928260684013367, "sampling/importance_sampling_ratio/max": 1.244694709777832, "sampling/importance_sampling_ratio/mean": 0.9808709621429443, "sampling/importance_sampling_ratio/min": 0.7011222839355469, "sampling/sampling_logp_difference/max": 0.13872861862182617, "sampling/sampling_logp_difference/mean": 0.0213931892067194, "step": 673, "step_time": 11.206018448999203 }, { "clip_ratio/high_max": 0.009798534912988544, "clip_ratio/high_mean": 0.004899267456494272, "clip_ratio/low_mean": 0.028549862559884787, "clip_ratio/low_min": 0.0059523810632526875, "clip_ratio/region_mean": 0.033449129783548415, "entropy": 1.3458092212677002, "epoch": 0.00674, "grad_norm": 0.3565709888935089, "kl": 0.7886452712118626, "learning_rate": 2.597900239515237e-05, "loss": 0.0716, "step": 674, "step_time": 5.915087358000164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1409.0, "completions/max_terminated_length": 1409.0, "completions/mean_length": 241.40625, "completions/mean_terminated_length": 241.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7851799763739109, "epoch": 0.00675, "frac_reward_zero_std": 0.25, "grad_norm": 0.11144138127565384, "kl": 0.4456556849181652, "learning_rate": 2.5921436704931136e-05, "loss": -0.0158, "num_tokens": 14450677.0, "reward": 0.56097412109375, "reward_std": 0.6715291738510132, "rewards/rollout_reward_func/mean": 0.56097412109375, "rewards/rollout_reward_func/std": 0.7971053719520569, "sampling/importance_sampling_ratio/max": 1.0926412343978882, "sampling/importance_sampling_ratio/mean": 1.0006117820739746, "sampling/importance_sampling_ratio/min": 0.8974263668060303, "sampling/sampling_logp_difference/max": 0.13373374938964844, "sampling/sampling_logp_difference/mean": 0.01576418988406658, "step": 675, "step_time": 8.505170551999981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.7680271789431572, "epoch": 0.00676, "grad_norm": 0.11421357840299606, "kl": 0.4383186399936676, "learning_rate": 2.5863888433273513e-05, "loss": -0.0163, "step": 676, "step_time": 5.473657123998237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 807.0, "completions/max_terminated_length": 804.0, "completions/mean_length": 388.78125, "completions/mean_terminated_length": 362.6206970214844, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3012077435851097, "epoch": 0.00677, "frac_reward_zero_std": 0.0, "grad_norm": 0.6690041422843933, "kl": 0.5492084212601185, "learning_rate": 2.5806358024989856e-05, "loss": 0.0437, "num_tokens": 14491471.0, "reward": 0.7170978784561157, "reward_std": 0.5851150751113892, "rewards/rollout_reward_func/mean": 0.7170978784561157, "rewards/rollout_reward_func/std": 0.631672739982605, "sampling/importance_sampling_ratio/max": 1.131829023361206, "sampling/importance_sampling_ratio/mean": 0.9957635402679443, "sampling/importance_sampling_ratio/min": 0.8241360187530518, "sampling/sampling_logp_difference/max": 0.16472101211547852, "sampling/sampling_logp_difference/mean": 0.026053790003061295, "step": 677, "step_time": 7.393145431999983 }, { "clip_ratio/high_max": 0.005570410052314401, "clip_ratio/high_mean": 0.0027852050261572003, "clip_ratio/low_mean": 0.03037925623357296, "clip_ratio/low_min": 0.014705882407724857, "clip_ratio/region_mean": 0.03316446114331484, "entropy": 1.289243072271347, "epoch": 0.00678, "grad_norm": 0.3822157382965088, "kl": 0.546156607568264, "learning_rate": 2.5748845924752408e-05, "loss": 0.0316, "step": 678, "step_time": 3.708640031998584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1632.0, "completions/max_terminated_length": 1632.0, "completions/mean_length": 377.0625, "completions/mean_terminated_length": 332.16668701171875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8844984024763107, "epoch": 0.00679, "frac_reward_zero_std": 0.75, "grad_norm": 0.25438347458839417, "kl": 0.3517576362937689, "learning_rate": 2.5691352577091907e-05, "loss": -0.0542, "num_tokens": 14531140.0, "reward": 0.8963966369628906, "reward_std": 0.14301955699920654, "rewards/rollout_reward_func/mean": 0.8963966369628906, "rewards/rollout_reward_func/std": 0.32732275128364563, "sampling/importance_sampling_ratio/max": 1.0775418281555176, "sampling/importance_sampling_ratio/mean": 0.997086763381958, "sampling/importance_sampling_ratio/min": 0.8817697167396545, "sampling/sampling_logp_difference/max": 0.17714166641235352, "sampling/sampling_logp_difference/mean": 0.021321740001440048, "step": 679, "step_time": 10.146402291001323 }, { "clip_ratio/high_max": 0.0037499999161809683, "clip_ratio/high_mean": 0.0018749999580904841, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018749999580904841, "entropy": 0.8848742060363293, "epoch": 0.0068, "grad_norm": 0.24926041066646576, "kl": 0.3507483918219805, "learning_rate": 2.5633878426394168e-05, "loss": -0.0571, "step": 680, "step_time": 5.214440614999148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2980.0, "completions/max_terminated_length": 2980.0, "completions/mean_length": 487.78125, "completions/mean_terminated_length": 487.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9083694852888584, "epoch": 0.00681, "frac_reward_zero_std": 0.25, "grad_norm": 0.18544551730155945, "kl": 0.3924887180328369, "learning_rate": 2.5576423916896612e-05, "loss": -0.0078, "num_tokens": 14574763.0, "reward": 0.8747926354408264, "reward_std": 0.35413995385169983, "rewards/rollout_reward_func/mean": 0.8747926354408264, "rewards/rollout_reward_func/std": 0.41613277792930603, "sampling/importance_sampling_ratio/max": 1.1327003240585327, "sampling/importance_sampling_ratio/mean": 0.9939380288124084, "sampling/importance_sampling_ratio/min": 0.6737723350524902, "sampling/sampling_logp_difference/max": 0.13111591339111328, "sampling/sampling_logp_difference/mean": 0.01877637580037117, "step": 681, "step_time": 13.721596947000762 }, { "clip_ratio/high_max": 0.0037878789007663727, "clip_ratio/high_mean": 0.0018939394503831863, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018939394503831863, "entropy": 0.9190357867628336, "epoch": 0.00682, "grad_norm": 0.1716865748167038, "kl": 0.384127713739872, "learning_rate": 2.5518989492684836e-05, "loss": -0.0092, "step": 682, "step_time": 7.825973353999871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1470.0, "completions/max_terminated_length": 1470.0, "completions/mean_length": 331.3125, "completions/mean_terminated_length": 331.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.707732617855072, "epoch": 0.00683, "frac_reward_zero_std": 0.0, "grad_norm": 0.09878436475992203, "kl": 0.7656391635537148, "learning_rate": 2.5461575597689225e-05, "loss": -0.0012, "num_tokens": 14613181.0, "reward": 0.67828369140625, "reward_std": 0.6265932321548462, "rewards/rollout_reward_func/mean": 0.67828369140625, "rewards/rollout_reward_func/std": 0.6532485485076904, "sampling/importance_sampling_ratio/max": 1.0738097429275513, "sampling/importance_sampling_ratio/mean": 0.9970142245292664, "sampling/importance_sampling_ratio/min": 0.9386981129646301, "sampling/sampling_logp_difference/max": 0.06281089782714844, "sampling/sampling_logp_difference/mean": 0.005994507111608982, "step": 683, "step_time": 8.785819166000692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.7254329584538937, "epoch": 0.00684, "grad_norm": 0.1002422422170639, "kl": 0.759078361093998, "learning_rate": 2.5404182675681452e-05, "loss": -0.0019, "step": 684, "step_time": 4.851697735000016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 394.875, "completions/mean_terminated_length": 380.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.361230343580246, "epoch": 0.00685, "frac_reward_zero_std": 0.25, "grad_norm": 0.5200825929641724, "kl": 0.7233906798064709, "learning_rate": 2.5346811170271105e-05, "loss": 0.0453, "num_tokens": 14654035.0, "reward": 0.718002200126648, "reward_std": 0.5491012930870056, "rewards/rollout_reward_func/mean": 0.718002200126648, "rewards/rollout_reward_func/std": 0.6312875151634216, "sampling/importance_sampling_ratio/max": 1.3188776969909668, "sampling/importance_sampling_ratio/mean": 1.002336025238037, "sampling/importance_sampling_ratio/min": 0.41147667169570923, "sampling/sampling_logp_difference/max": 0.18509578704833984, "sampling/sampling_logp_difference/mean": 0.019617415964603424, "step": 685, "step_time": 10.087098416999652 }, { "clip_ratio/high_max": 0.003289473708719015, "clip_ratio/high_mean": 0.0016447368543595076, "clip_ratio/low_mean": 0.02385822287760675, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.025502959731966257, "entropy": 1.363755002617836, "epoch": 0.00686, "grad_norm": 0.39830732345581055, "kl": 0.7229347825050354, "learning_rate": 2.5289461524902213e-05, "loss": 0.0361, "step": 686, "step_time": 5.441897245001201 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1609.0, "completions/max_terminated_length": 1609.0, "completions/mean_length": 504.625, "completions/mean_terminated_length": 496.258056640625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1272527761757374, "epoch": 0.00687, "frac_reward_zero_std": 0.25, "grad_norm": 0.33773767948150635, "kl": 0.5532964393496513, "learning_rate": 2.5232134182849854e-05, "loss": 0.0118, "num_tokens": 14698680.0, "reward": 0.7107675075531006, "reward_std": 0.5160337090492249, "rewards/rollout_reward_func/mean": 0.7107675075531006, "rewards/rollout_reward_func/std": 0.6397936940193176, "sampling/importance_sampling_ratio/max": 1.2284618616104126, "sampling/importance_sampling_ratio/mean": 0.99765944480896, "sampling/importance_sampling_ratio/min": 0.35769927501678467, "sampling/sampling_logp_difference/max": 0.24544048309326172, "sampling/sampling_logp_difference/mean": 0.024900276213884354, "step": 687, "step_time": 9.763300689998687 }, { "clip_ratio/high_max": 0.018914473708719015, "clip_ratio/high_mean": 0.009457236854359508, "clip_ratio/low_mean": 0.005102040711790323, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01455927756614983, "entropy": 1.1201859042048454, "epoch": 0.00688, "grad_norm": 0.2613888382911682, "kl": 0.5484176091849804, "learning_rate": 2.5174829587216732e-05, "loss": 0.0085, "step": 688, "step_time": 5.182210902998122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1515.0, "completions/max_terminated_length": 1515.0, "completions/mean_length": 373.71875, "completions/mean_terminated_length": 373.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.4456119015812874, "epoch": 0.00689, "frac_reward_zero_std": 0.25, "grad_norm": 0.42587509751319885, "kl": 0.6998508032411337, "learning_rate": 2.511754818092971e-05, "loss": -0.0245, "num_tokens": 14738397.0, "reward": 0.6812533736228943, "reward_std": 0.5124555826187134, "rewards/rollout_reward_func/mean": 0.6812533736228943, "rewards/rollout_reward_func/std": 0.6483588218688965, "sampling/importance_sampling_ratio/max": 2.1487958431243896, "sampling/importance_sampling_ratio/mean": 1.013498067855835, "sampling/importance_sampling_ratio/min": 0.665615439414978, "sampling/sampling_logp_difference/max": 0.18748903274536133, "sampling/sampling_logp_difference/mean": 0.025975249707698822, "step": 689, "step_time": 10.052162328000122 }, { "clip_ratio/high_max": 0.022463151952251792, "clip_ratio/high_mean": 0.011231575976125896, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011231575976125896, "entropy": 1.4244138225913048, "epoch": 0.0069, "grad_norm": 0.20140257477760315, "kl": 0.6265592705458403, "learning_rate": 2.5060290406736426e-05, "loss": -0.029, "step": 690, "step_time": 4.961189535000813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1457.0, "completions/max_terminated_length": 1382.0, "completions/mean_length": 459.625, "completions/mean_terminated_length": 398.03448486328125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.432263195514679, "epoch": 0.00691, "frac_reward_zero_std": 0.25, "grad_norm": 0.3029285669326782, "kl": 0.7591924164444208, "learning_rate": 2.5003056707201856e-05, "loss": -0.0642, "num_tokens": 14781633.0, "reward": 0.7464747428894043, "reward_std": 0.5117542743682861, "rewards/rollout_reward_func/mean": 0.7464747428894043, "rewards/rollout_reward_func/std": 0.6230671405792236, "sampling/importance_sampling_ratio/max": 1.142200231552124, "sampling/importance_sampling_ratio/mean": 0.9592963457107544, "sampling/importance_sampling_ratio/min": 1.449623893146068e-10, "sampling/sampling_logp_difference/max": 14.074394226074219, "sampling/sampling_logp_difference/mean": 0.09957902133464813, "step": 691, "step_time": 10.20332884599884 }, { "clip_ratio/high_max": 0.015499345026910305, "clip_ratio/high_mean": 0.0077496725134551525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0077496725134551525, "entropy": 1.4460472166538239, "epoch": 0.00692, "grad_norm": 0.2520861029624939, "kl": 0.6727859005331993, "learning_rate": 2.494584752470489e-05, "loss": -0.0681, "step": 692, "step_time": 5.010406078999949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1393.0, "completions/max_terminated_length": 1393.0, "completions/mean_length": 388.15625, "completions/mean_terminated_length": 356.45159912109375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7672892287373543, "epoch": 0.00693, "frac_reward_zero_std": 0.25, "grad_norm": 0.14021097123622894, "kl": 0.417260080575943, "learning_rate": 2.488866330143492e-05, "loss": 0.0091, "num_tokens": 14821672.0, "reward": 0.7438250780105591, "reward_std": 0.46629229187965393, "rewards/rollout_reward_func/mean": 0.7438250780105591, "rewards/rollout_reward_func/std": 0.5693174004554749, "sampling/importance_sampling_ratio/max": 1.1666210889816284, "sampling/importance_sampling_ratio/mean": 0.9937714338302612, "sampling/importance_sampling_ratio/min": 0.5442620515823364, "sampling/sampling_logp_difference/max": 0.19278931617736816, "sampling/sampling_logp_difference/mean": 0.017872517928481102, "step": 693, "step_time": 9.887387307000608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.8109859488904476, "epoch": 0.00694, "grad_norm": 0.1129506528377533, "kl": 0.3958737775683403, "learning_rate": 2.4831504479388422e-05, "loss": 0.0075, "step": 694, "step_time": 4.830535877998955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 1628.0, "completions/max_terminated_length": 1628.0, "completions/mean_length": 436.1875, "completions/mean_terminated_length": 426.2857360839844, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.08399760723114, "epoch": 0.00695, "frac_reward_zero_std": 0.5, "grad_norm": 0.3590414822101593, "kl": 0.6357752941548824, "learning_rate": 2.4774371500365542e-05, "loss": -0.0385, "num_tokens": 14864452.0, "reward": 0.7552591562271118, "reward_std": 0.42560189962387085, "rewards/rollout_reward_func/mean": 0.7552591562271118, "rewards/rollout_reward_func/std": 0.6578947901725769, "sampling/importance_sampling_ratio/max": 1.1043645143508911, "sampling/importance_sampling_ratio/mean": 0.9214215874671936, "sampling/importance_sampling_ratio/min": 1.4216697263869094e-10, "sampling/sampling_logp_difference/max": 20.573848724365234, "sampling/sampling_logp_difference/mean": 0.08915406465530396, "step": 695, "step_time": 10.856133879000481 }, { "clip_ratio/high_max": 0.03622623346745968, "clip_ratio/high_mean": 0.01811311673372984, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.028529783710837364, "entropy": 2.1425178349018097, "epoch": 0.00696, "grad_norm": 0.31522229313850403, "kl": 0.641034372150898, "learning_rate": 2.471726480596666e-05, "loss": -0.0437, "step": 696, "step_time": 5.592085682000288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 1657.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 566.375, "completions/mean_terminated_length": 568.0435180664062, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.7294999957084656, "epoch": 0.00697, "frac_reward_zero_std": 0.25, "grad_norm": 0.9458325505256653, "kl": 0.6317094583064318, "learning_rate": 2.4660184837589005e-05, "loss": 0.178, "num_tokens": 14911311.0, "reward": 0.7208899259567261, "reward_std": 0.572650671005249, "rewards/rollout_reward_func/mean": 0.7208899259567261, "rewards/rollout_reward_func/std": 0.6751996278762817, "sampling/importance_sampling_ratio/max": 1.5962508916854858, "sampling/importance_sampling_ratio/mean": 0.9164923429489136, "sampling/importance_sampling_ratio/min": 2.6460259330840508e-11, "sampling/sampling_logp_difference/max": 19.871564865112305, "sampling/sampling_logp_difference/mean": 0.14492501318454742, "step": 697, "step_time": 11.668958866001049 }, { "clip_ratio/high_max": 0.005942691001109779, "clip_ratio/high_mean": 0.0029713455005548894, "clip_ratio/low_mean": 0.007885062135756016, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010856407636310905, "entropy": 2.729355826973915, "epoch": 0.00698, "grad_norm": 0.7655652165412903, "kl": 0.613725220784545, "learning_rate": 2.4603132036423232e-05, "loss": 0.1638, "step": 698, "step_time": 5.34634921000179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 2414.0, "completions/max_terminated_length": 1454.0, "completions/mean_length": 501.40625, "completions/mean_terminated_length": 414.862060546875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.7624883577227592, "epoch": 0.00699, "frac_reward_zero_std": 0.25, "grad_norm": 0.9360024333000183, "kl": 0.8862244747579098, "learning_rate": 2.4546106843450006e-05, "loss": 0.2543, "num_tokens": 14956200.0, "reward": 0.6918745040893555, "reward_std": 0.5755195617675781, "rewards/rollout_reward_func/mean": 0.6918745040893555, "rewards/rollout_reward_func/std": 0.6748337149620056, "sampling/importance_sampling_ratio/max": 1.212828278541565, "sampling/importance_sampling_ratio/mean": 0.9981621503829956, "sampling/importance_sampling_ratio/min": 0.4551490247249603, "sampling/sampling_logp_difference/max": 0.17870235443115234, "sampling/sampling_logp_difference/mean": 0.03124580904841423, "step": 699, "step_time": 13.425782066000465 }, { "clip_ratio/high_max": 0.006048386916518211, "clip_ratio/high_mean": 0.0030241934582591057, "clip_ratio/low_mean": 0.014476496260613203, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01750068971887231, "entropy": 1.7123970836400986, "epoch": 0.007, "grad_norm": 0.7259509563446045, "kl": 0.8704575300216675, "learning_rate": 2.448910969943661e-05, "loss": 0.2365, "step": 700, "step_time": 6.738776567000059 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1675.0, "completions/max_terminated_length": 1332.0, "completions/mean_length": 379.65625, "completions/mean_terminated_length": 283.5862121582031, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.6103920862078667, "epoch": 0.00701, "frac_reward_zero_std": 0.25, "grad_norm": 0.5980591177940369, "kl": 0.6773737147450447, "learning_rate": 2.443214104493351e-05, "loss": 0.0482, "num_tokens": 14996347.0, "reward": 0.6873558163642883, "reward_std": 0.5149618983268738, "rewards/rollout_reward_func/mean": 0.6873558163642883, "rewards/rollout_reward_func/std": 0.6841953992843628, "sampling/importance_sampling_ratio/max": 1.585257649421692, "sampling/importance_sampling_ratio/mean": 0.9252548813819885, "sampling/importance_sampling_ratio/min": 1.3128098608206074e-10, "sampling/sampling_logp_difference/max": 21.36931800842285, "sampling/sampling_logp_difference/mean": 0.21904118359088898, "step": 701, "step_time": 10.33521248699708 }, { "clip_ratio/high_max": 0.005681818351149559, "clip_ratio/high_mean": 0.0028409091755747795, "clip_ratio/low_mean": 0.006127451080828905, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008968360256403685, "entropy": 1.5787501335144043, "epoch": 0.00702, "grad_norm": 0.4490593373775482, "kl": 0.6811163760721684, "learning_rate": 2.4375201320270983e-05, "loss": 0.039, "step": 702, "step_time": 5.274412706000476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2468.0, "completions/max_terminated_length": 2468.0, "completions/mean_length": 598.96875, "completions/mean_terminated_length": 563.8386840820312, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0753675028681755, "epoch": 0.00703, "frac_reward_zero_std": 0.0, "grad_norm": 0.4241155683994293, "kl": 0.48618271201848984, "learning_rate": 2.4318290965555676e-05, "loss": -0.0442, "num_tokens": 15045200.0, "reward": 0.5675130486488342, "reward_std": 0.7186901569366455, "rewards/rollout_reward_func/mean": 0.5675130486488342, "rewards/rollout_reward_func/std": 0.7826125025749207, "sampling/importance_sampling_ratio/max": 1.619917869567871, "sampling/importance_sampling_ratio/mean": 1.0378124713897705, "sampling/importance_sampling_ratio/min": 0.8905327916145325, "sampling/sampling_logp_difference/max": 0.270355224609375, "sampling/sampling_logp_difference/mean": 0.02427513897418976, "step": 703, "step_time": 13.000380245999622 }, { "clip_ratio/high_max": 0.0062500000931322575, "clip_ratio/high_mean": 0.0031250000465661287, "clip_ratio/low_mean": 0.016741071827709675, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.019866071874275804, "entropy": 1.0530270412564278, "epoch": 0.00704, "grad_norm": 0.24866102635860443, "kl": 0.49735296703875065, "learning_rate": 2.4261410420667253e-05, "loss": -0.0479, "step": 704, "step_time": 6.629847572999097 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1592.0, "completions/max_terminated_length": 1418.0, "completions/mean_length": 380.21875, "completions/mean_terminated_length": 341.1290283203125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9883750714361668, "epoch": 0.00705, "frac_reward_zero_std": 0.25, "grad_norm": 0.1422155350446701, "kl": 1.2019672468304634, "learning_rate": 2.4204560125254952e-05, "loss": -0.0147, "num_tokens": 15085170.0, "reward": 0.7096593976020813, "reward_std": 0.5171562433242798, "rewards/rollout_reward_func/mean": 0.7096593976020813, "rewards/rollout_reward_func/std": 0.6452889442443848, "sampling/importance_sampling_ratio/max": 1.3105453252792358, "sampling/importance_sampling_ratio/mean": 0.9765889644622803, "sampling/importance_sampling_ratio/min": 4.852287408679823e-12, "sampling/sampling_logp_difference/max": 14.849479675292969, "sampling/sampling_logp_difference/mean": 0.13510771095752716, "step": 705, "step_time": 9.474860646999332 }, { "clip_ratio/high_max": 0.006414473755285144, "clip_ratio/high_mean": 0.003207236877642572, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003207236877642572, "entropy": 0.9801279231905937, "epoch": 0.00706, "grad_norm": 0.14131958782672882, "kl": 1.1020101010799408, "learning_rate": 2.4147740518734204e-05, "loss": -0.0153, "step": 706, "step_time": 5.614420600001722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 2369.0, "completions/max_terminated_length": 2369.0, "completions/mean_length": 456.34375, "completions/mean_terminated_length": 415.9666748046875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2038102857768536, "epoch": 0.00707, "frac_reward_zero_std": 0.0, "grad_norm": 0.672252893447876, "kl": 0.9043542891740799, "learning_rate": 2.409095204028323e-05, "loss": 0.0239, "num_tokens": 15126253.0, "reward": 0.6859968304634094, "reward_std": 0.6881729364395142, "rewards/rollout_reward_func/mean": 0.6859968304634094, "rewards/rollout_reward_func/std": 0.690031886100769, "sampling/importance_sampling_ratio/max": 1.583370327949524, "sampling/importance_sampling_ratio/mean": 0.9943335056304932, "sampling/importance_sampling_ratio/min": 0.7072291970252991, "sampling/sampling_logp_difference/max": 0.16990280151367188, "sampling/sampling_logp_difference/mean": 0.026607949286699295, "step": 707, "step_time": 12.175110191001295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 1.199298270046711, "epoch": 0.00708, "grad_norm": 0.44158777594566345, "kl": 0.8683750033378601, "learning_rate": 2.403419512883965e-05, "loss": 0.0138, "step": 708, "step_time": 6.519639432000076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 463.5, "completions/mean_terminated_length": 463.9000244140625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.116987619549036, "epoch": 0.00709, "frac_reward_zero_std": 0.0, "grad_norm": 0.7402669191360474, "kl": 0.661150511354208, "learning_rate": 2.3977470223097128e-05, "loss": 0.1309, "num_tokens": 15169051.0, "reward": 0.7392411231994629, "reward_std": 0.562883734703064, "rewards/rollout_reward_func/mean": 0.7392411231994629, "rewards/rollout_reward_func/std": 0.5827648043632507, "sampling/importance_sampling_ratio/max": 1.1432288885116577, "sampling/importance_sampling_ratio/mean": 0.9619666934013367, "sampling/importance_sampling_ratio/min": 2.364702129855484e-13, "sampling/sampling_logp_difference/max": 19.317781448364258, "sampling/sampling_logp_difference/mean": 0.11047627776861191, "step": 709, "step_time": 10.026931442000205 }, { "clip_ratio/high_max": 0.004385964944958687, "clip_ratio/high_mean": 0.0021929824724793434, "clip_ratio/low_mean": 0.015724206576123834, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017917188815772533, "entropy": 1.1147705912590027, "epoch": 0.0071, "grad_norm": 0.541562557220459, "kl": 0.6851187944412231, "learning_rate": 2.3920777761501885e-05, "loss": 0.1193, "step": 710, "step_time": 5.800677265997365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2272.0, "completions/max_terminated_length": 2272.0, "completions/mean_length": 566.78125, "completions/mean_terminated_length": 558.0322265625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0609032325446606, "epoch": 0.00711, "frac_reward_zero_std": 0.25, "grad_norm": 0.739517092704773, "kl": 0.4611767828464508, "learning_rate": 2.3864118182249418e-05, "loss": 0.1537, "num_tokens": 15213718.0, "reward": 0.7754393219947815, "reward_std": 0.46867090463638306, "rewards/rollout_reward_func/mean": 0.7754393219947815, "rewards/rollout_reward_func/std": 0.5567976236343384, "sampling/importance_sampling_ratio/max": 1.138329267501831, "sampling/importance_sampling_ratio/mean": 0.9993600845336914, "sampling/importance_sampling_ratio/min": 0.9122876524925232, "sampling/sampling_logp_difference/max": 0.15241527557373047, "sampling/sampling_logp_difference/mean": 0.022406384348869324, "step": 711, "step_time": 12.047909971999616 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.014362374320626259, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014362374320626259, "entropy": 1.0465088374912739, "epoch": 0.00712, "grad_norm": 0.5708169341087341, "kl": 0.45846296288073063, "learning_rate": 2.380749192328104e-05, "loss": 0.1388, "step": 712, "step_time": 6.601445246999901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 437.8125, "completions/mean_terminated_length": 437.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7007620111107826, "epoch": 0.00713, "frac_reward_zero_std": 0.25, "grad_norm": 0.3127342462539673, "kl": 0.47417138889431953, "learning_rate": 2.3750899422280534e-05, "loss": 0.0536, "num_tokens": 15256846.0, "reward": 0.8408187627792358, "reward_std": 0.3689482510089874, "rewards/rollout_reward_func/mean": 0.8408187627792358, "rewards/rollout_reward_func/std": 0.4517277181148529, "sampling/importance_sampling_ratio/max": 1.1013296842575073, "sampling/importance_sampling_ratio/mean": 0.9902468919754028, "sampling/importance_sampling_ratio/min": 0.8720284104347229, "sampling/sampling_logp_difference/max": 0.12559747695922852, "sampling/sampling_logp_difference/mean": 0.00954488106071949, "step": 713, "step_time": 9.969420861997605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.6865661107003689, "epoch": 0.00714, "grad_norm": 0.23293240368366241, "kl": 0.47219166345894337, "learning_rate": 2.3694341116670735e-05, "loss": 0.0508, "step": 714, "step_time": 5.017174083001009 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1650.0, "completions/max_terminated_length": 1650.0, "completions/mean_length": 488.84375, "completions/mean_terminated_length": 488.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8013762831687927, "epoch": 0.00715, "frac_reward_zero_std": 0.5, "grad_norm": 0.11249834299087524, "kl": 0.4468844747170806, "learning_rate": 2.36378174436102e-05, "loss": 0.0031, "num_tokens": 15300762.0, "reward": 0.837397575378418, "reward_std": 0.31145143508911133, "rewards/rollout_reward_func/mean": 0.837397575378418, "rewards/rollout_reward_func/std": 0.4569096565246582, "sampling/importance_sampling_ratio/max": 1.1596506834030151, "sampling/importance_sampling_ratio/mean": 0.9979619383811951, "sampling/importance_sampling_ratio/min": 0.8728378415107727, "sampling/sampling_logp_difference/max": 0.13688397407531738, "sampling/sampling_logp_difference/mean": 0.014146046712994576, "step": 715, "step_time": 9.844565120998595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.7862136773765087, "epoch": 0.00716, "grad_norm": 0.09925840049982071, "kl": 0.4684479720890522, "learning_rate": 2.358132883998979e-05, "loss": 0.0028, "step": 716, "step_time": 5.938722744000188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1506.0, "completions/max_terminated_length": 1506.0, "completions/mean_length": 555.9375, "completions/mean_terminated_length": 555.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6710281670093536, "epoch": 0.00717, "frac_reward_zero_std": 0.25, "grad_norm": 0.08091665804386139, "kl": 0.5109141729772091, "learning_rate": 2.3524875742429285e-05, "loss": 0.0057, "num_tokens": 15347245.0, "reward": 0.7500265836715698, "reward_std": 0.4931170344352722, "rewards/rollout_reward_func/mean": 0.7500265836715698, "rewards/rollout_reward_func/std": 0.614757776260376, "sampling/importance_sampling_ratio/max": 1.120659351348877, "sampling/importance_sampling_ratio/mean": 0.9946245551109314, "sampling/importance_sampling_ratio/min": 0.8672288656234741, "sampling/sampling_logp_difference/max": 0.1729593276977539, "sampling/sampling_logp_difference/mean": 0.009684127755463123, "step": 717, "step_time": 9.236944370999481 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.6683293245732784, "epoch": 0.00718, "grad_norm": 0.07995332032442093, "kl": 0.5138462036848068, "learning_rate": 2.3468458587274033e-05, "loss": 0.0056, "step": 718, "step_time": 5.061804587000552 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1609.0, "completions/max_terminated_length": 1609.0, "completions/mean_length": 585.65625, "completions/mean_terminated_length": 585.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7083432003855705, "epoch": 0.00719, "frac_reward_zero_std": 0.0, "grad_norm": 0.12019678950309753, "kl": 0.5788338538259268, "learning_rate": 2.3412077810591584e-05, "loss": -0.008, "num_tokens": 15394865.0, "reward": 0.6604063510894775, "reward_std": 0.6974360346794128, "rewards/rollout_reward_func/mean": 0.6604063510894775, "rewards/rollout_reward_func/std": 0.7314324378967285, "sampling/importance_sampling_ratio/max": 1.0932674407958984, "sampling/importance_sampling_ratio/mean": 0.9898531436920166, "sampling/importance_sampling_ratio/min": 0.7285826206207275, "sampling/sampling_logp_difference/max": 0.10959339141845703, "sampling/sampling_logp_difference/mean": 0.010795529931783676, "step": 719, "step_time": 10.216201140998237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.7036913111805916, "epoch": 0.0072, "grad_norm": 0.12064842134714127, "kl": 0.5783365648239851, "learning_rate": 2.3355733848168298e-05, "loss": -0.0083, "step": 720, "step_time": 5.2971426609992704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1437.0, "completions/max_terminated_length": 1437.0, "completions/mean_length": 575.34375, "completions/mean_terminated_length": 575.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7340166047215462, "epoch": 0.00721, "frac_reward_zero_std": 0.25, "grad_norm": 0.12615808844566345, "kl": 0.4044648241251707, "learning_rate": 2.329942713550599e-05, "loss": 0.003, "num_tokens": 15443336.0, "reward": 0.6640375852584839, "reward_std": 0.6308016180992126, "rewards/rollout_reward_func/mean": 0.6640375852584839, "rewards/rollout_reward_func/std": 0.7230803966522217, "sampling/importance_sampling_ratio/max": 1.1760270595550537, "sampling/importance_sampling_ratio/mean": 1.0045270919799805, "sampling/importance_sampling_ratio/min": 0.8813459277153015, "sampling/sampling_logp_difference/max": 0.11063909530639648, "sampling/sampling_logp_difference/mean": 0.014795849099755287, "step": 721, "step_time": 9.11376202000065 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.7347027957439423, "epoch": 0.00722, "grad_norm": 0.1334385871887207, "kl": 0.40482095442712307, "learning_rate": 2.3243158107818542e-05, "loss": 0.0024, "step": 722, "step_time": 4.882413900999381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2128.0, "completions/max_terminated_length": 2128.0, "completions/mean_length": 543.96875, "completions/mean_terminated_length": 535.4515991210938, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9499909225851297, "epoch": 0.00723, "frac_reward_zero_std": 0.75, "grad_norm": 0.6192213892936707, "kl": 0.7481931447982788, "learning_rate": 2.3186927200028583e-05, "loss": 0.0752, "num_tokens": 15488723.0, "reward": 0.8727320432662964, "reward_std": 0.189304918050766, "rewards/rollout_reward_func/mean": 0.8727320432662964, "rewards/rollout_reward_func/std": 0.4238302707672119, "sampling/importance_sampling_ratio/max": 1.1062086820602417, "sampling/importance_sampling_ratio/mean": 0.9462934732437134, "sampling/importance_sampling_ratio/min": 9.088011232449844e-09, "sampling/sampling_logp_difference/max": 17.66103172302246, "sampling/sampling_logp_difference/mean": 0.11813419312238693, "step": 723, "step_time": 11.614902515998438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.01182432472705841, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01182432472705841, "entropy": 0.946502573788166, "epoch": 0.00724, "grad_norm": 0.3111397922039032, "kl": 0.7248288206756115, "learning_rate": 2.3130734846764063e-05, "loss": 0.0693, "step": 724, "step_time": 6.055080137999539 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2495.0, "completions/max_terminated_length": 2495.0, "completions/mean_length": 694.59375, "completions/mean_terminated_length": 694.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8329553753137589, "epoch": 0.00725, "frac_reward_zero_std": 0.0, "grad_norm": 0.2712773382663727, "kl": 0.6895122863352299, "learning_rate": 2.3074581482354976e-05, "loss": 0.0419, "num_tokens": 15539484.0, "reward": 0.7543938159942627, "reward_std": 0.5379430055618286, "rewards/rollout_reward_func/mean": 0.7543938159942627, "rewards/rollout_reward_func/std": 0.552143394947052, "sampling/importance_sampling_ratio/max": 1.1196428537368774, "sampling/importance_sampling_ratio/mean": 0.9734461307525635, "sampling/importance_sampling_ratio/min": 0.5210849046707153, "sampling/sampling_logp_difference/max": 0.17762279510498047, "sampling/sampling_logp_difference/mean": 0.019609807059168816, "step": 725, "step_time": 12.4578166780002 }, { "clip_ratio/high_max": 0.004807692486792803, "clip_ratio/high_mean": 0.0024038462433964014, "clip_ratio/low_mean": 0.01215277798473835, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014556624228134751, "entropy": 0.8173038065433502, "epoch": 0.00726, "grad_norm": 0.21165338158607483, "kl": 0.689838208258152, "learning_rate": 2.301846754082992e-05, "loss": 0.0387, "step": 726, "step_time": 7.286691985000289 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1569.0, "completions/max_terminated_length": 1569.0, "completions/mean_length": 414.875, "completions/mean_terminated_length": 414.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5745225921273232, "epoch": 0.00727, "frac_reward_zero_std": 0.0, "grad_norm": 0.14012043178081512, "kl": 0.4517542514950037, "learning_rate": 2.29623934559128e-05, "loss": 0.0019, "num_tokens": 15579775.0, "reward": 0.6182672381401062, "reward_std": 0.7194052934646606, "rewards/rollout_reward_func/mean": 0.6182672381401062, "rewards/rollout_reward_func/std": 0.7078465223312378, "sampling/importance_sampling_ratio/max": 1.2225805521011353, "sampling/importance_sampling_ratio/mean": 0.9977753162384033, "sampling/importance_sampling_ratio/min": 0.893773078918457, "sampling/sampling_logp_difference/max": 0.10965633392333984, "sampling/sampling_logp_difference/mean": 0.010398373007774353, "step": 727, "step_time": 9.215130231998955 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 0.5657752677798271, "epoch": 0.00728, "grad_norm": 0.1353023499250412, "kl": 0.4489872772246599, "learning_rate": 2.290635966101945e-05, "loss": 0.0019, "step": 728, "step_time": 5.171174640999197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 515.875, "completions/mean_terminated_length": 515.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5705903023481369, "epoch": 0.00729, "frac_reward_zero_std": 0.0, "grad_norm": 0.16632068157196045, "kl": 0.5129135139286518, "learning_rate": 2.2850366589254287e-05, "loss": 0.0031, "num_tokens": 15625285.0, "reward": 0.6591222286224365, "reward_std": 0.6703842878341675, "rewards/rollout_reward_func/mean": 0.6591222286224365, "rewards/rollout_reward_func/std": 0.6887598037719727, "sampling/importance_sampling_ratio/max": 1.1031333208084106, "sampling/importance_sampling_ratio/mean": 0.9915429353713989, "sampling/importance_sampling_ratio/min": 0.8978246450424194, "sampling/sampling_logp_difference/max": 0.1312403678894043, "sampling/sampling_logp_difference/mean": 0.011628316715359688, "step": 729, "step_time": 9.53336210299949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5755409598350525, "epoch": 0.0073, "grad_norm": 0.15392477810382843, "kl": 0.5069199372082949, "learning_rate": 2.279441467340697e-05, "loss": 0.0022, "step": 730, "step_time": 5.79509322100057 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2182.0, "completions/max_terminated_length": 2182.0, "completions/mean_length": 537.0625, "completions/mean_terminated_length": 553.3547973632812, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7665664032101631, "epoch": 0.00731, "frac_reward_zero_std": 0.0, "grad_norm": 0.5539430975914001, "kl": 0.5119128283113241, "learning_rate": 2.2738504345949045e-05, "loss": 0.0673, "num_tokens": 15671418.0, "reward": 0.6501184105873108, "reward_std": 0.6298240423202515, "rewards/rollout_reward_func/mean": 0.6501184105873108, "rewards/rollout_reward_func/std": 0.6582183241844177, "sampling/importance_sampling_ratio/max": 1.1443052291870117, "sampling/importance_sampling_ratio/mean": 0.9973000288009644, "sampling/importance_sampling_ratio/min": 0.8576779365539551, "sampling/sampling_logp_difference/max": 0.14960956573486328, "sampling/sampling_logp_difference/mean": 0.012062406167387962, "step": 731, "step_time": 11.195381771998655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02690972201526165, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02690972201526165, "entropy": 0.766780361533165, "epoch": 0.00732, "grad_norm": 0.3323502242565155, "kl": 0.5272365938872099, "learning_rate": 2.2682636039030608e-05, "loss": 0.0587, "step": 732, "step_time": 6.161286167000071 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1425.0, "completions/max_terminated_length": 1425.0, "completions/mean_length": 333.90625, "completions/mean_terminated_length": 333.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5894388277083635, "epoch": 0.00733, "frac_reward_zero_std": 0.0, "grad_norm": 0.1271638721227646, "kl": 0.6422333009541035, "learning_rate": 2.262681018447697e-05, "loss": -0.0143, "num_tokens": 15709870.0, "reward": 0.4706975221633911, "reward_std": 0.7598009705543518, "rewards/rollout_reward_func/mean": 0.4706975221633911, "rewards/rollout_reward_func/std": 0.8721747994422913, "sampling/importance_sampling_ratio/max": 1.049962043762207, "sampling/importance_sampling_ratio/mean": 0.987941324710846, "sampling/importance_sampling_ratio/min": 0.8791170120239258, "sampling/sampling_logp_difference/max": 0.11716127395629883, "sampling/sampling_logp_difference/mean": 0.008578830398619175, "step": 733, "step_time": 9.263869874000193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5815416574478149, "epoch": 0.00734, "grad_norm": 0.12423069030046463, "kl": 0.6511418223381042, "learning_rate": 2.2571027213785297e-05, "loss": -0.0148, "step": 734, "step_time": 4.83954340299988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1534.0, "completions/max_terminated_length": 1534.0, "completions/mean_length": 368.34375, "completions/mean_terminated_length": 368.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5304556004703045, "epoch": 0.00735, "frac_reward_zero_std": 0.0, "grad_norm": 0.12114448100328445, "kl": 0.5070543540641665, "learning_rate": 2.25152875581213e-05, "loss": -0.0039, "num_tokens": 15749499.0, "reward": 0.6435263156890869, "reward_std": 0.6473668217658997, "rewards/rollout_reward_func/mean": 0.6435263156890869, "rewards/rollout_reward_func/std": 0.6618156433105469, "sampling/importance_sampling_ratio/max": 1.090092420578003, "sampling/importance_sampling_ratio/mean": 1.002851963043213, "sampling/importance_sampling_ratio/min": 0.9340647459030151, "sampling/sampling_logp_difference/max": 0.1209566593170166, "sampling/sampling_logp_difference/mean": 0.008232126012444496, "step": 735, "step_time": 8.918882394998946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5160440765321255, "epoch": 0.00736, "grad_norm": 0.12338762730360031, "kl": 0.506589874625206, "learning_rate": 2.2459591648315917e-05, "loss": -0.0043, "step": 736, "step_time": 4.9700319380008295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2432.0, "completions/max_terminated_length": 2432.0, "completions/mean_length": 559.96875, "completions/mean_terminated_length": 552.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7966158613562584, "epoch": 0.00737, "frac_reward_zero_std": 0.0, "grad_norm": 0.3967040777206421, "kl": 1.09919673204422, "learning_rate": 2.2403939914861917e-05, "loss": 0.0342, "num_tokens": 15795509.0, "reward": 0.5863649845123291, "reward_std": 0.617701530456543, "rewards/rollout_reward_func/mean": 0.5863649845123291, "rewards/rollout_reward_func/std": 0.7153815031051636, "sampling/importance_sampling_ratio/max": 1.3056321144104004, "sampling/importance_sampling_ratio/mean": 1.0059036016464233, "sampling/importance_sampling_ratio/min": 0.8186764121055603, "sampling/sampling_logp_difference/max": 0.18649768829345703, "sampling/sampling_logp_difference/mean": 0.01842072792351246, "step": 737, "step_time": 12.450974117999067 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.013020833488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.023437500465661287, "entropy": 0.7895537726581097, "epoch": 0.00738, "grad_norm": 0.27216005325317383, "kl": 1.0085564814507961, "learning_rate": 2.2348332787910646e-05, "loss": 0.0304, "step": 738, "step_time": 6.623740412999723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 444.03125, "completions/mean_terminated_length": 444.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.49266766011714935, "epoch": 0.00739, "frac_reward_zero_std": 0.0, "grad_norm": 0.12456516176462173, "kl": 0.4611305594444275, "learning_rate": 2.229277069726865e-05, "loss": -0.0036, "num_tokens": 15837796.0, "reward": 0.7225847840309143, "reward_std": 0.703184187412262, "rewards/rollout_reward_func/mean": 0.7225847840309143, "rewards/rollout_reward_func/std": 0.6707859039306641, "sampling/importance_sampling_ratio/max": 1.0470932722091675, "sampling/importance_sampling_ratio/mean": 0.9973845481872559, "sampling/importance_sampling_ratio/min": 0.8633804321289062, "sampling/sampling_logp_difference/max": 0.0991823673248291, "sampling/sampling_logp_difference/mean": 0.008052418008446693, "step": 739, "step_time": 8.877377379998507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4957295022904873, "epoch": 0.0074, "grad_norm": 0.12684746086597443, "kl": 0.4595663044601679, "learning_rate": 2.2237254072394384e-05, "loss": -0.004, "step": 740, "step_time": 5.337046368001211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 813.0, "completions/max_terminated_length": 813.0, "completions/mean_length": 412.0, "completions/mean_terminated_length": 412.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5221765302121639, "epoch": 0.00741, "frac_reward_zero_std": 0.0, "grad_norm": 0.20928019285202026, "kl": 0.7946195639669895, "learning_rate": 2.218178334239489e-05, "loss": 0.0106, "num_tokens": 15879640.0, "reward": 0.6210803985595703, "reward_std": 0.673497200012207, "rewards/rollout_reward_func/mean": 0.6210803985595703, "rewards/rollout_reward_func/std": 0.7037081718444824, "sampling/importance_sampling_ratio/max": 1.11087965965271, "sampling/importance_sampling_ratio/mean": 0.9971277713775635, "sampling/importance_sampling_ratio/min": 0.8520959615707397, "sampling/sampling_logp_difference/max": 0.10575389862060547, "sampling/sampling_logp_difference/mean": 0.009132477454841137, "step": 741, "step_time": 7.098730886999874 }, { "clip_ratio/high_max": 0.0416666679084301, "clip_ratio/high_mean": 0.02083333395421505, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02083333395421505, "entropy": 0.5258191898465157, "epoch": 0.00742, "grad_norm": 0.13571742177009583, "kl": 0.7913757301867008, "learning_rate": 2.2126358936022463e-05, "loss": 0.0095, "step": 742, "step_time": 3.7129049100012708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2337.0, "completions/max_terminated_length": 2337.0, "completions/mean_length": 601.4375, "completions/mean_terminated_length": 601.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6993902809917927, "epoch": 0.00743, "frac_reward_zero_std": 0.25, "grad_norm": 0.5347145795822144, "kl": 0.5996505543589592, "learning_rate": 2.207098128167135e-05, "loss": 0.0632, "num_tokens": 15927409.0, "reward": 0.4983159899711609, "reward_std": 0.6165413856506348, "rewards/rollout_reward_func/mean": 0.4983159899711609, "rewards/rollout_reward_func/std": 0.7546851634979248, "sampling/importance_sampling_ratio/max": 1.1172994375228882, "sampling/importance_sampling_ratio/mean": 1.0153043270111084, "sampling/importance_sampling_ratio/min": 0.9329805970191956, "sampling/sampling_logp_difference/max": 0.13469696044921875, "sampling/sampling_logp_difference/mean": 0.013280182145535946, "step": 743, "step_time": 12.519709928998964 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.016447369009256363, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020353619009256363, "entropy": 0.701530808582902, "epoch": 0.00744, "grad_norm": 0.3183951675891876, "kl": 0.6158079877495766, "learning_rate": 2.2015650807374438e-05, "loss": 0.0562, "step": 744, "step_time": 6.58192009600134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1355.0, "completions/max_terminated_length": 1355.0, "completions/mean_length": 448.46875, "completions/mean_terminated_length": 447.58062744140625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6647972241044044, "epoch": 0.00745, "frac_reward_zero_std": 0.25, "grad_norm": 0.1195279210805893, "kl": 0.5387332290410995, "learning_rate": 2.196036794079993e-05, "loss": -0.0063, "num_tokens": 15971010.0, "reward": 0.740824818611145, "reward_std": 0.45048829913139343, "rewards/rollout_reward_func/mean": 0.740824818611145, "rewards/rollout_reward_func/std": 0.5774456858634949, "sampling/importance_sampling_ratio/max": 1.118504285812378, "sampling/importance_sampling_ratio/mean": 0.9513195753097534, "sampling/importance_sampling_ratio/min": 4.2180568933514184e-12, "sampling/sampling_logp_difference/max": 25.96063804626465, "sampling/sampling_logp_difference/mean": 0.16586388647556305, "step": 745, "step_time": 8.996049018000122 }, { "clip_ratio/high_max": 0.013888888992369175, "clip_ratio/high_mean": 0.0069444444961845875, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0069444444961845875, "entropy": 0.6533864997327328, "epoch": 0.00746, "grad_norm": 0.10886652767658234, "kl": 0.5478470344096422, "learning_rate": 2.1905133109248058e-05, "loss": -0.0071, "step": 746, "step_time": 5.1286653080005635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3349.0, "completions/max_terminated_length": 3349.0, "completions/mean_length": 520.90625, "completions/mean_terminated_length": 520.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5594962518662214, "epoch": 0.00747, "frac_reward_zero_std": 0.0, "grad_norm": 0.26024240255355835, "kl": 0.38421379774808884, "learning_rate": 2.18499467396478e-05, "loss": 0.0004, "num_tokens": 16015405.0, "reward": 0.5612443685531616, "reward_std": 0.8318808078765869, "rewards/rollout_reward_func/mean": 0.5612443685531616, "rewards/rollout_reward_func/std": 0.7959965467453003, "sampling/importance_sampling_ratio/max": 1.1706273555755615, "sampling/importance_sampling_ratio/mean": 1.0074772834777832, "sampling/importance_sampling_ratio/min": 0.9162049889564514, "sampling/sampling_logp_difference/max": 0.11029422283172607, "sampling/sampling_logp_difference/mean": 0.009400291368365288, "step": 747, "step_time": 14.088526815999103 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "entropy": 0.5516011826694012, "epoch": 0.00748, "grad_norm": 0.16481564939022064, "kl": 0.4013940878212452, "learning_rate": 2.1794809258553512e-05, "loss": -0.0003, "step": 748, "step_time": 8.213723355998809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1515.0, "completions/max_terminated_length": 1515.0, "completions/mean_length": 407.0625, "completions/mean_terminated_length": 407.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4780812319368124, "epoch": 0.00749, "frac_reward_zero_std": 0.0, "grad_norm": 0.17578986287117004, "kl": 0.4469014182686806, "learning_rate": 2.1739721092141717e-05, "loss": -0.0003, "num_tokens": 16055828.0, "reward": 0.524856686592102, "reward_std": 0.7148327827453613, "rewards/rollout_reward_func/mean": 0.524856686592102, "rewards/rollout_reward_func/std": 0.7567912340164185, "sampling/importance_sampling_ratio/max": 1.0447918176651, "sampling/importance_sampling_ratio/mean": 0.99384605884552, "sampling/importance_sampling_ratio/min": 0.8485206961631775, "sampling/sampling_logp_difference/max": 0.11223888397216797, "sampling/sampling_logp_difference/mean": 0.0069185541942715645, "step": 749, "step_time": 9.07457764999981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "entropy": 0.4707312434911728, "epoch": 0.0075, "grad_norm": 0.11407463997602463, "kl": 0.44627223163843155, "learning_rate": 2.168468266620773e-05, "loss": -0.0011, "step": 750, "step_time": 5.678211551999993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1382.0, "completions/max_terminated_length": 1382.0, "completions/mean_length": 345.5625, "completions/mean_terminated_length": 345.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4594198577105999, "epoch": 0.00751, "frac_reward_zero_std": 0.25, "grad_norm": 0.16566933691501617, "kl": 0.4662707280367613, "learning_rate": 2.1629694406162433e-05, "loss": 0.0026, "num_tokens": 16094350.0, "reward": 0.5900030136108398, "reward_std": 0.6356229782104492, "rewards/rollout_reward_func/mean": 0.5900030136108398, "rewards/rollout_reward_func/std": 0.753598153591156, "sampling/importance_sampling_ratio/max": 1.0972468852996826, "sampling/importance_sampling_ratio/mean": 0.9904950857162476, "sampling/importance_sampling_ratio/min": 0.8932729959487915, "sampling/sampling_logp_difference/max": 0.11289143562316895, "sampling/sampling_logp_difference/mean": 0.010134734213352203, "step": 751, "step_time": 8.62836948000131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.45295772701501846, "epoch": 0.00752, "grad_norm": 0.1621469408273697, "kl": 0.4654367472976446, "learning_rate": 2.157475673702894e-05, "loss": 0.0022, "step": 752, "step_time": 4.764296872000159 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1623.0, "completions/max_terminated_length": 1623.0, "completions/mean_length": 488.125, "completions/mean_terminated_length": 488.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6613086238503456, "epoch": 0.00753, "frac_reward_zero_std": 0.0, "grad_norm": 0.22410286962985992, "kl": 0.6469302009791136, "learning_rate": 2.1519870083439337e-05, "loss": -0.0226, "num_tokens": 16139425.0, "reward": 0.5979828238487244, "reward_std": 0.7489335536956787, "rewards/rollout_reward_func/mean": 0.5979828238487244, "rewards/rollout_reward_func/std": 0.7367532253265381, "sampling/importance_sampling_ratio/max": 1.1341861486434937, "sampling/importance_sampling_ratio/mean": 0.9752399921417236, "sampling/importance_sampling_ratio/min": 7.732720713748975e-13, "sampling/sampling_logp_difference/max": 21.725269317626953, "sampling/sampling_logp_difference/mean": 0.24878138303756714, "step": 753, "step_time": 10.355116692999218 }, { "clip_ratio/high_max": 0.004166666883975267, "clip_ratio/high_mean": 0.0020833334419876337, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020833334419876337, "entropy": 0.6584298983216286, "epoch": 0.00754, "grad_norm": 0.2274877429008484, "kl": 0.6474000550806522, "learning_rate": 2.1465034869631405e-05, "loss": -0.0236, "step": 754, "step_time": 5.296284117001051 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2432.0, "completions/max_terminated_length": 2432.0, "completions/mean_length": 581.90625, "completions/mean_terminated_length": 581.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5240503177046776, "epoch": 0.00755, "frac_reward_zero_std": 0.25, "grad_norm": 0.17284274101257324, "kl": 0.7330050989985466, "learning_rate": 2.141025151944531e-05, "loss": 0.0004, "num_tokens": 16185854.0, "reward": 0.7229552268981934, "reward_std": 0.5712928771972656, "rewards/rollout_reward_func/mean": 0.7229552268981934, "rewards/rollout_reward_func/std": 0.6690534949302673, "sampling/importance_sampling_ratio/max": 1.1175787448883057, "sampling/importance_sampling_ratio/mean": 1.006455421447754, "sampling/importance_sampling_ratio/min": 0.8729934096336365, "sampling/sampling_logp_difference/max": 0.11373233795166016, "sampling/sampling_logp_difference/mean": 0.010526767931878567, "step": 755, "step_time": 11.738399756000035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.5265414528548717, "epoch": 0.00756, "grad_norm": 0.12660282850265503, "kl": 0.7305229902267456, "learning_rate": 2.135552045632035e-05, "loss": 0.0, "step": 756, "step_time": 6.653512367000076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 568.625, "completions/mean_terminated_length": 568.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6002857647836208, "epoch": 0.00757, "frac_reward_zero_std": 0.25, "grad_norm": 0.12514913082122803, "kl": 0.429178724065423, "learning_rate": 2.1300842103291696e-05, "loss": -0.0063, "num_tokens": 16233426.0, "reward": 0.8454245924949646, "reward_std": 0.43720537424087524, "rewards/rollout_reward_func/mean": 0.8454245924949646, "rewards/rollout_reward_func/std": 0.5044934749603271, "sampling/importance_sampling_ratio/max": 1.119064211845398, "sampling/importance_sampling_ratio/mean": 0.9785807728767395, "sampling/importance_sampling_ratio/min": 0.8162871599197388, "sampling/sampling_logp_difference/max": 0.28817105293273926, "sampling/sampling_logp_difference/mean": 0.013207456097006798, "step": 757, "step_time": 10.360109764998015 }, { "clip_ratio/high_max": 0.01785714365541935, "clip_ratio/high_mean": 0.008928571827709675, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008928571827709675, "entropy": 0.6034286469221115, "epoch": 0.00758, "grad_norm": 0.12175001949071884, "kl": 0.4287486653774977, "learning_rate": 2.1246216882987084e-05, "loss": -0.0068, "step": 758, "step_time": 5.100711741998566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 562.75, "completions/mean_terminated_length": 562.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5310917906463146, "epoch": 0.00759, "frac_reward_zero_std": 0.75, "grad_norm": 0.09859902411699295, "kl": 0.5233961679041386, "learning_rate": 2.1191645217623586e-05, "loss": 0.0018, "num_tokens": 16281363.0, "reward": 0.9672310948371887, "reward_std": 0.09268448501825333, "rewards/rollout_reward_func/mean": 0.9672310948371887, "rewards/rollout_reward_func/std": 0.18536895513534546, "sampling/importance_sampling_ratio/max": 1.1272624731063843, "sampling/importance_sampling_ratio/mean": 0.9888820648193359, "sampling/importance_sampling_ratio/min": 0.6906797289848328, "sampling/sampling_logp_difference/max": 0.16004467010498047, "sampling/sampling_logp_difference/mean": 0.010041951201856136, "step": 759, "step_time": 9.496243087001858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5330731198191643, "epoch": 0.0076, "grad_norm": 0.100838802754879, "kl": 0.522966492921114, "learning_rate": 2.1137127529004302e-05, "loss": 0.0018, "step": 760, "step_time": 5.608611726000163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1686.0, "completions/max_terminated_length": 1686.0, "completions/mean_length": 600.9375, "completions/mean_terminated_length": 600.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5444173887372017, "epoch": 0.00761, "frac_reward_zero_std": 0.25, "grad_norm": 0.19153989851474762, "kl": 0.6688697040081024, "learning_rate": 2.108266423851516e-05, "loss": 0.0117, "num_tokens": 16329645.0, "reward": 0.6862072944641113, "reward_std": 0.5285062193870544, "rewards/rollout_reward_func/mean": 0.6862072944641113, "rewards/rollout_reward_func/std": 0.6329946517944336, "sampling/importance_sampling_ratio/max": 1.1234636306762695, "sampling/importance_sampling_ratio/mean": 1.0117872953414917, "sampling/importance_sampling_ratio/min": 0.9058845639228821, "sampling/sampling_logp_difference/max": 0.11528873443603516, "sampling/sampling_logp_difference/mean": 0.0077851880341768265, "step": 761, "step_time": 9.932099902001937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.5511987544596195, "epoch": 0.00762, "grad_norm": 0.12764781713485718, "kl": 0.6713055334985256, "learning_rate": 2.1028255767121603e-05, "loss": 0.01, "step": 762, "step_time": 5.385755820999293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1670.0, "completions/max_terminated_length": 1670.0, "completions/mean_length": 540.78125, "completions/mean_terminated_length": 540.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5696066804230213, "epoch": 0.00763, "frac_reward_zero_std": 0.25, "grad_norm": 0.13942097127437592, "kl": 0.514889856800437, "learning_rate": 2.0973902535365373e-05, "loss": -0.0051, "num_tokens": 16375032.0, "reward": 0.5685024261474609, "reward_std": 0.5512774586677551, "rewards/rollout_reward_func/mean": 0.5685024261474609, "rewards/rollout_reward_func/std": 0.7808393836021423, "sampling/importance_sampling_ratio/max": 1.2476186752319336, "sampling/importance_sampling_ratio/mean": 1.0139174461364746, "sampling/importance_sampling_ratio/min": 0.9127019047737122, "sampling/sampling_logp_difference/max": 0.1838228702545166, "sampling/sampling_logp_difference/mean": 0.010442761704325676, "step": 763, "step_time": 9.798572337999758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.573801826685667, "epoch": 0.00764, "grad_norm": 0.10675373673439026, "kl": 0.5052819065749645, "learning_rate": 2.0919604963361247e-05, "loss": -0.0059, "step": 764, "step_time": 5.927257370999541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1632.0, "completions/max_terminated_length": 1632.0, "completions/mean_length": 618.9375, "completions/mean_terminated_length": 618.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6789339035749435, "epoch": 0.00765, "frac_reward_zero_std": 0.0, "grad_norm": 0.16562896966934204, "kl": 0.7785225547850132, "learning_rate": 2.0865363470793767e-05, "loss": -0.0024, "num_tokens": 16424291.0, "reward": 0.47491464018821716, "reward_std": 0.7920519113540649, "rewards/rollout_reward_func/mean": 0.47491464018821716, "rewards/rollout_reward_func/std": 0.8271054029464722, "sampling/importance_sampling_ratio/max": 1.196579933166504, "sampling/importance_sampling_ratio/mean": 0.9979792833328247, "sampling/importance_sampling_ratio/min": 0.8828474879264832, "sampling/sampling_logp_difference/max": 0.19106340408325195, "sampling/sampling_logp_difference/mean": 0.015400055795907974, "step": 765, "step_time": 9.882882775000326 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 0.6821189858019352, "epoch": 0.00766, "grad_norm": 0.14900992810726166, "kl": 0.7932510320097208, "learning_rate": 2.0811178476914042e-05, "loss": -0.0037, "step": 766, "step_time": 5.335929766001755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1661.0, "completions/max_terminated_length": 1661.0, "completions/mean_length": 424.9375, "completions/mean_terminated_length": 424.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5214452128857374, "epoch": 0.00767, "frac_reward_zero_std": 0.25, "grad_norm": 0.13749071955680847, "kl": 0.5866287127137184, "learning_rate": 2.0757050400536464e-05, "loss": 0.0131, "num_tokens": 16465959.0, "reward": 0.6345981359481812, "reward_std": 0.6069671511650085, "rewards/rollout_reward_func/mean": 0.6345981359481812, "rewards/rollout_reward_func/std": 0.7730714678764343, "sampling/importance_sampling_ratio/max": 1.0661672353744507, "sampling/importance_sampling_ratio/mean": 0.9956477880477905, "sampling/importance_sampling_ratio/min": 0.842247486114502, "sampling/sampling_logp_difference/max": 0.11826133728027344, "sampling/sampling_logp_difference/mean": 0.011773619800806046, "step": 767, "step_time": 10.23177236599895 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.5325513742864132, "epoch": 0.00768, "grad_norm": 0.13647860288619995, "kl": 0.572475528344512, "learning_rate": 2.0702979660035514e-05, "loss": 0.0125, "step": 768, "step_time": 5.275807662001171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1667.0, "completions/max_terminated_length": 1667.0, "completions/mean_length": 451.875, "completions/mean_terminated_length": 451.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5839571319520473, "epoch": 0.00769, "frac_reward_zero_std": 0.25, "grad_norm": 0.29022952914237976, "kl": 0.48069076240062714, "learning_rate": 2.0648966673342484e-05, "loss": 0.0174, "num_tokens": 16508740.0, "reward": 0.7237550020217896, "reward_std": 0.5827528238296509, "rewards/rollout_reward_func/mean": 0.7237550020217896, "rewards/rollout_reward_func/std": 0.6713195443153381, "sampling/importance_sampling_ratio/max": 1.541393518447876, "sampling/importance_sampling_ratio/mean": 1.007133960723877, "sampling/importance_sampling_ratio/min": 0.7526553869247437, "sampling/sampling_logp_difference/max": 0.17189884185791016, "sampling/sampling_logp_difference/mean": 0.01447431929409504, "step": 769, "step_time": 9.786268728000323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02473958395421505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02473958395421505, "entropy": 0.5875887274742126, "epoch": 0.0077, "grad_norm": 0.22753705084323883, "kl": 0.5084888190031052, "learning_rate": 2.0595011857942273e-05, "loss": 0.0151, "step": 770, "step_time": 5.222036847000709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1587.0, "completions/max_terminated_length": 1587.0, "completions/mean_length": 534.65625, "completions/mean_terminated_length": 534.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6624661013484001, "epoch": 0.00771, "frac_reward_zero_std": 0.0, "grad_norm": 0.1972397118806839, "kl": 0.5126159712672234, "learning_rate": 2.054111563087015e-05, "loss": -0.0029, "num_tokens": 16554868.0, "reward": 0.6892932653427124, "reward_std": 0.6657048463821411, "rewards/rollout_reward_func/mean": 0.6892932653427124, "rewards/rollout_reward_func/std": 0.6842862367630005, "sampling/importance_sampling_ratio/max": 1.1639468669891357, "sampling/importance_sampling_ratio/mean": 1.009112000465393, "sampling/importance_sampling_ratio/min": 0.9344419240951538, "sampling/sampling_logp_difference/max": 0.12900853157043457, "sampling/sampling_logp_difference/mean": 0.014230658300220966, "step": 771, "step_time": 10.315367716000765 }, { "clip_ratio/high_max": 0.004999999888241291, "clip_ratio/high_mean": 0.0024999999441206455, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01291666692122817, "entropy": 0.6679319217801094, "epoch": 0.00772, "grad_norm": 0.14452357590198517, "kl": 0.5006693080067635, "learning_rate": 2.048727840870853e-05, "loss": -0.0044, "step": 772, "step_time": 5.230148661000385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 820.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 300.5625, "completions/mean_terminated_length": 300.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6672230511903763, "epoch": 0.00773, "frac_reward_zero_std": 0.0, "grad_norm": 0.22751149535179138, "kl": 0.6860860269516706, "learning_rate": 2.04335006075838e-05, "loss": -0.0001, "num_tokens": 16592373.0, "reward": 0.5589803457260132, "reward_std": 0.7151007652282715, "rewards/rollout_reward_func/mean": 0.5589803457260132, "rewards/rollout_reward_func/std": 0.7555885910987854, "sampling/importance_sampling_ratio/max": 1.2134286165237427, "sampling/importance_sampling_ratio/mean": 1.0161538124084473, "sampling/importance_sampling_ratio/min": 0.9398910999298096, "sampling/sampling_logp_difference/max": 0.123992919921875, "sampling/sampling_logp_difference/mean": 0.008460241369903088, "step": 773, "step_time": 7.399400340999819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.012500000186264515, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012500000186264515, "entropy": 0.6660268604755402, "epoch": 0.00774, "grad_norm": 0.16957803070545197, "kl": 0.6821988523006439, "learning_rate": 2.0379782643163005e-05, "loss": -0.0021, "step": 774, "step_time": 3.658805596001912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1607.0, "completions/max_terminated_length": 1607.0, "completions/mean_length": 507.78125, "completions/mean_terminated_length": 498.70965576171875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7522969581186771, "epoch": 0.00775, "frac_reward_zero_std": 0.5, "grad_norm": 0.8996609449386597, "kl": 0.4783330522477627, "learning_rate": 2.0326124930650716e-05, "loss": 0.1289, "num_tokens": 16637141.0, "reward": 0.8136776685714722, "reward_std": 0.35976487398147583, "rewards/rollout_reward_func/mean": 0.8136776685714722, "rewards/rollout_reward_func/std": 0.5277752876281738, "sampling/importance_sampling_ratio/max": 1.5971605777740479, "sampling/importance_sampling_ratio/mean": 1.0271286964416504, "sampling/importance_sampling_ratio/min": 0.7333836555480957, "sampling/sampling_logp_difference/max": 0.147200345993042, "sampling/sampling_logp_difference/mean": 0.01674336940050125, "step": 775, "step_time": 9.696669237002425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.01151315774768591, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01151315774768591, "entropy": 0.7483888156712055, "epoch": 0.00776, "grad_norm": 0.613354504108429, "kl": 0.47529664263129234, "learning_rate": 2.0272527884785805e-05, "loss": 0.1155, "step": 776, "step_time": 5.207465431999481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1680.0, "completions/max_terminated_length": 1632.0, "completions/mean_length": 535.5625, "completions/mean_terminated_length": 498.6451416015625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.737381212413311, "epoch": 0.00777, "frac_reward_zero_std": 0.25, "grad_norm": 0.6865932941436768, "kl": 0.8150724954903126, "learning_rate": 2.021899191983822e-05, "loss": 0.0784, "num_tokens": 16683454.0, "reward": 0.5981172323226929, "reward_std": 0.5772075653076172, "rewards/rollout_reward_func/mean": 0.5981172323226929, "rewards/rollout_reward_func/std": 0.6953396797180176, "sampling/importance_sampling_ratio/max": 1.3439704179763794, "sampling/importance_sampling_ratio/mean": 1.010160207748413, "sampling/importance_sampling_ratio/min": 0.901994526386261, "sampling/sampling_logp_difference/max": 0.14528274536132812, "sampling/sampling_logp_difference/mean": 0.01529176626354456, "step": 777, "step_time": 10.336886777000473 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.01406249962747097, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.024479166604578495, "entropy": 0.7298704497516155, "epoch": 0.00778, "grad_norm": 0.4026501476764679, "kl": 0.8018498718738556, "learning_rate": 2.0165517449605785e-05, "loss": 0.0694, "step": 778, "step_time": 5.299853802002872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 562.375, "completions/mean_terminated_length": 554.4838256835938, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8100117705762386, "epoch": 0.00779, "frac_reward_zero_std": 0.5, "grad_norm": 0.3475852608680725, "kl": 0.4543303530663252, "learning_rate": 2.0112104887411033e-05, "loss": 0.016, "num_tokens": 16731175.0, "reward": 0.8763865828514099, "reward_std": 0.2706654369831085, "rewards/rollout_reward_func/mean": 0.8763865828514099, "rewards/rollout_reward_func/std": 0.4157128930091858, "sampling/importance_sampling_ratio/max": 1.0823501348495483, "sampling/importance_sampling_ratio/mean": 0.9895720481872559, "sampling/importance_sampling_ratio/min": 0.8968935608863831, "sampling/sampling_logp_difference/max": 0.10696721076965332, "sampling/sampling_logp_difference/mean": 0.01362180057913065, "step": 779, "step_time": 9.925353042000097 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0069444444961845875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0069444444961845875, "entropy": 0.8032997511327267, "epoch": 0.0078, "grad_norm": 0.18662922084331512, "kl": 0.44547152146697044, "learning_rate": 2.005875464609797e-05, "loss": 0.0146, "step": 780, "step_time": 5.742767375998483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 440.71875, "completions/mean_terminated_length": 440.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5113673694431782, "epoch": 0.00781, "frac_reward_zero_std": 0.25, "grad_norm": 0.1466646045446396, "kl": 0.48409415781497955, "learning_rate": 2.000546713802892e-05, "loss": 0.0133, "num_tokens": 16773333.0, "reward": 0.8115619421005249, "reward_std": 0.46821293234825134, "rewards/rollout_reward_func/mean": 0.8115619421005249, "rewards/rollout_reward_func/std": 0.531319260597229, "sampling/importance_sampling_ratio/max": 1.109130620956421, "sampling/importance_sampling_ratio/mean": 1.0024528503417969, "sampling/importance_sampling_ratio/min": 0.9208766222000122, "sampling/sampling_logp_difference/max": 0.1064291000366211, "sampling/sampling_logp_difference/mean": 0.010031154379248619, "step": 781, "step_time": 9.27749171600044 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "entropy": 0.508708767592907, "epoch": 0.00782, "grad_norm": 0.13709203898906708, "kl": 0.47701284661889076, "learning_rate": 1.9952242775081293e-05, "loss": 0.0124, "step": 782, "step_time": 5.131944311000552 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1722.0, "completions/max_terminated_length": 1722.0, "completions/mean_length": 464.6875, "completions/mean_terminated_length": 464.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5303659196943045, "epoch": 0.00783, "frac_reward_zero_std": 0.25, "grad_norm": 0.1462528109550476, "kl": 0.5986428894102573, "learning_rate": 1.989908196864445e-05, "loss": 0.007, "num_tokens": 16815394.0, "reward": 0.8092198371887207, "reward_std": 0.3970092833042145, "rewards/rollout_reward_func/mean": 0.8092198371887207, "rewards/rollout_reward_func/std": 0.4683467149734497, "sampling/importance_sampling_ratio/max": 1.1354113817214966, "sampling/importance_sampling_ratio/mean": 0.9953715205192566, "sampling/importance_sampling_ratio/min": 0.8307932019233704, "sampling/sampling_logp_difference/max": 0.1238713264465332, "sampling/sampling_logp_difference/mean": 0.011221980676054955, "step": 783, "step_time": 9.907087723000586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5310543701052666, "epoch": 0.00784, "grad_norm": 0.13962578773498535, "kl": 0.59534752368927, "learning_rate": 1.9845985129616507e-05, "loss": 0.0065, "step": 784, "step_time": 5.92839364899919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1382.0, "completions/max_terminated_length": 1382.0, "completions/mean_length": 303.6875, "completions/mean_terminated_length": 303.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.42908582743257284, "epoch": 0.00785, "frac_reward_zero_std": 0.25, "grad_norm": 0.14911535382270813, "kl": 0.5671870987862349, "learning_rate": 1.979295266840113e-05, "loss": -0.0028, "num_tokens": 16851716.0, "reward": 0.7470116019248962, "reward_std": 0.4973726272583008, "rewards/rollout_reward_func/mean": 0.7470116019248962, "rewards/rollout_reward_func/std": 0.6244075298309326, "sampling/importance_sampling_ratio/max": 1.2798868417739868, "sampling/importance_sampling_ratio/mean": 0.9843477010726929, "sampling/importance_sampling_ratio/min": 1.5321598156869953e-10, "sampling/sampling_logp_difference/max": 22.55361557006836, "sampling/sampling_logp_difference/mean": 0.21353571116924286, "step": 785, "step_time": 8.613826126999811 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.43146042712032795, "epoch": 0.00786, "grad_norm": 0.09396333992481232, "kl": 0.5751952240243554, "learning_rate": 1.973998499490441e-05, "loss": -0.0034, "step": 786, "step_time": 4.771237330999611 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 351.28125, "completions/mean_terminated_length": 351.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6364720873534679, "epoch": 0.00787, "frac_reward_zero_std": 0.0, "grad_norm": 0.12547524273395538, "kl": 0.4169987216591835, "learning_rate": 1.968708251853166e-05, "loss": -0.0055, "num_tokens": 16891052.0, "reward": 0.684206485748291, "reward_std": 0.6735973358154297, "rewards/rollout_reward_func/mean": 0.684206485748291, "rewards/rollout_reward_func/std": 0.6942089200019836, "sampling/importance_sampling_ratio/max": 1.0695748329162598, "sampling/importance_sampling_ratio/mean": 0.9667345881462097, "sampling/importance_sampling_ratio/min": 5.7048320060981705e-12, "sampling/sampling_logp_difference/max": 18.964946746826172, "sampling/sampling_logp_difference/mean": 0.15870696306228638, "step": 787, "step_time": 9.377459687000737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6341287679970264, "epoch": 0.00788, "grad_norm": 0.12733979523181915, "kl": 0.4190411679446697, "learning_rate": 1.9634245648184254e-05, "loss": -0.0063, "step": 788, "step_time": 4.833419040000081 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1623.0, "completions/max_terminated_length": 1623.0, "completions/mean_length": 306.40625, "completions/mean_terminated_length": 306.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.43347846530377865, "epoch": 0.00789, "frac_reward_zero_std": 0.0, "grad_norm": 2.0421690940856934, "kl": 10.340774333104491, "learning_rate": 1.9581474792256496e-05, "loss": 0.0372, "num_tokens": 16928065.0, "reward": 0.5902985334396362, "reward_std": 0.7897276282310486, "rewards/rollout_reward_func/mean": 0.5902985334396362, "rewards/rollout_reward_func/std": 0.7560532689094543, "sampling/importance_sampling_ratio/max": 1.054628610610962, "sampling/importance_sampling_ratio/mean": 0.9524479508399963, "sampling/importance_sampling_ratio/min": 2.3292201500879628e-11, "sampling/sampling_logp_difference/max": 24.514511108398438, "sampling/sampling_logp_difference/mean": 0.2604764997959137, "step": 789, "step_time": 9.135332007000216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4379502236843109, "epoch": 0.0079, "grad_norm": 0.46162286400794983, "kl": 4.045585284009576, "learning_rate": 1.952877035863243e-05, "loss": 0.0162, "step": 790, "step_time": 5.130715317000067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2411.0, "completions/max_terminated_length": 2411.0, "completions/mean_length": 573.46875, "completions/mean_terminated_length": 573.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.690522450953722, "epoch": 0.00791, "frac_reward_zero_std": 0.75, "grad_norm": 0.021376624703407288, "kl": 0.42121721245348454, "learning_rate": 1.9476132754682696e-05, "loss": 0.0067, "num_tokens": 16975203.0, "reward": 0.9664958119392395, "reward_std": 0.09476417303085327, "rewards/rollout_reward_func/mean": 0.9664958119392395, "rewards/rollout_reward_func/std": 0.18952834606170654, "sampling/importance_sampling_ratio/max": 1.2538641691207886, "sampling/importance_sampling_ratio/mean": 0.9743049144744873, "sampling/importance_sampling_ratio/min": 7.161660392661557e-16, "sampling/sampling_logp_difference/max": 18.075870513916016, "sampling/sampling_logp_difference/mean": 0.21244964003562927, "step": 791, "step_time": 12.091650839000977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.7023277096450329, "epoch": 0.00792, "grad_norm": 0.021356353536248207, "kl": 0.42030835896730423, "learning_rate": 1.9423562387261394e-05, "loss": 0.0067, "step": 792, "step_time": 6.47401138800069 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 410.4375, "completions/mean_terminated_length": 410.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5492259189486504, "epoch": 0.00793, "frac_reward_zero_std": 0.25, "grad_norm": 0.0802648589015007, "kl": 0.30700498819351196, "learning_rate": 1.937105966270291e-05, "loss": 0.0042, "num_tokens": 17016110.0, "reward": 0.749442994594574, "reward_std": 0.5350698232650757, "rewards/rollout_reward_func/mean": 0.749442994594574, "rewards/rollout_reward_func/std": 0.6156667470932007, "sampling/importance_sampling_ratio/max": 1.067438006401062, "sampling/importance_sampling_ratio/mean": 0.999251663684845, "sampling/importance_sampling_ratio/min": 0.8918675780296326, "sampling/sampling_logp_difference/max": 0.08514666557312012, "sampling/sampling_logp_difference/mean": 0.008120979182422161, "step": 793, "step_time": 8.805995474997871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5600586533546448, "epoch": 0.00794, "grad_norm": 0.0794597789645195, "kl": 0.3130709892138839, "learning_rate": 1.9318624986818822e-05, "loss": 0.0037, "step": 794, "step_time": 5.187039301999903 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 478.34375, "completions/mean_terminated_length": 478.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6781909242272377, "epoch": 0.00795, "frac_reward_zero_std": 0.5, "grad_norm": 0.0765560120344162, "kl": 0.39888735953718424, "learning_rate": 1.9266258764894726e-05, "loss": 0.0005, "num_tokens": 17060141.0, "reward": 0.8166396617889404, "reward_std": 0.39948463439941406, "rewards/rollout_reward_func/mean": 0.8166396617889404, "rewards/rollout_reward_func/std": 0.5792499780654907, "sampling/importance_sampling_ratio/max": 1.2679591178894043, "sampling/importance_sampling_ratio/mean": 1.0047597885131836, "sampling/importance_sampling_ratio/min": 0.8851482272148132, "sampling/sampling_logp_difference/max": 0.12294483184814453, "sampling/sampling_logp_difference/mean": 0.011140218004584312, "step": 795, "step_time": 9.33787944299911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6920997798442841, "epoch": 0.00796, "grad_norm": 0.07917065918445587, "kl": 0.39973900094628334, "learning_rate": 1.92139614016871e-05, "loss": 0.0002, "step": 796, "step_time": 5.045482862999961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1418.0, "completions/max_terminated_length": 1418.0, "completions/mean_length": 554.28125, "completions/mean_terminated_length": 546.1290283203125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9506930708885193, "epoch": 0.00797, "frac_reward_zero_std": 0.0, "grad_norm": 0.93541419506073, "kl": 0.8309726975858212, "learning_rate": 1.9161733301420215e-05, "loss": 0.0336, "num_tokens": 17105988.0, "reward": 0.5935439467430115, "reward_std": 0.758037269115448, "rewards/rollout_reward_func/mean": 0.5935439467430115, "rewards/rollout_reward_func/std": 0.7470954060554504, "sampling/importance_sampling_ratio/max": 1.3436167240142822, "sampling/importance_sampling_ratio/mean": 1.0333904027938843, "sampling/importance_sampling_ratio/min": 0.8595806360244751, "sampling/sampling_logp_difference/max": 0.2496042251586914, "sampling/sampling_logp_difference/mean": 0.026438409462571144, "step": 797, "step_time": 9.212003128997821 }, { "clip_ratio/high_max": 0.0033783784601837397, "clip_ratio/high_mean": 0.0016891892300918698, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033339260844513774, "entropy": 0.9688281826674938, "epoch": 0.00798, "grad_norm": 0.38795265555381775, "kl": 0.7101837731897831, "learning_rate": 1.910957486778295e-05, "loss": 0.027, "step": 798, "step_time": 5.555615301999751 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2325.0, "completions/max_terminated_length": 2325.0, "completions/mean_length": 576.625, "completions/mean_terminated_length": 576.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8124265670776367, "epoch": 0.00799, "frac_reward_zero_std": 0.0, "grad_norm": 0.2044716626405716, "kl": 0.3710605874657631, "learning_rate": 1.9057486503925745e-05, "loss": 0.0172, "num_tokens": 17154247.0, "reward": 0.6181111931800842, "reward_std": 0.6294835805892944, "rewards/rollout_reward_func/mean": 0.6181111931800842, "rewards/rollout_reward_func/std": 0.6619064211845398, "sampling/importance_sampling_ratio/max": 1.129513144493103, "sampling/importance_sampling_ratio/mean": 0.9973589181900024, "sampling/importance_sampling_ratio/min": 0.8863736391067505, "sampling/sampling_logp_difference/max": 0.1708512306213379, "sampling/sampling_logp_difference/mean": 0.015136411413550377, "step": 799, "step_time": 11.601088796001932 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.01197916700039059, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017187500023283064, "entropy": 0.8247725814580917, "epoch": 0.008, "grad_norm": 0.14761404693126678, "kl": 0.37045063078403473, "learning_rate": 1.900546861245743e-05, "loss": 0.0154, "step": 800, "step_time": 6.987468491000072 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1616.0, "completions/max_terminated_length": 1616.0, "completions/mean_length": 505.125, "completions/mean_terminated_length": 494.8064270019531, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7971752882003784, "epoch": 0.00801, "frac_reward_zero_std": 0.25, "grad_norm": 0.2964351773262024, "kl": 0.48048029839992523, "learning_rate": 1.895352159544212e-05, "loss": 0.0374, "num_tokens": 17199407.0, "reward": 0.7199872732162476, "reward_std": 0.5010086894035339, "rewards/rollout_reward_func/mean": 0.7199872732162476, "rewards/rollout_reward_func/std": 0.6224181056022644, "sampling/importance_sampling_ratio/max": 1.1977413892745972, "sampling/importance_sampling_ratio/mean": 1.0013325214385986, "sampling/importance_sampling_ratio/min": 0.7108585834503174, "sampling/sampling_logp_difference/max": 0.1200551986694336, "sampling/sampling_logp_difference/mean": 0.017574399709701538, "step": 801, "step_time": 9.76614030699966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016447368543595076, "entropy": 0.7971008718013763, "epoch": 0.00802, "grad_norm": 0.2718023359775543, "kl": 0.48291922733187675, "learning_rate": 1.8901645854396132e-05, "loss": 0.0353, "step": 802, "step_time": 5.248503959999653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2434.0, "completions/max_terminated_length": 2434.0, "completions/mean_length": 503.84375, "completions/mean_terminated_length": 468.6128845214844, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9500954486429691, "epoch": 0.00803, "frac_reward_zero_std": 0.0, "grad_norm": 0.8508690595626831, "kl": 0.5328507628291845, "learning_rate": 1.8849841790284853e-05, "loss": 0.1209, "num_tokens": 17243066.0, "reward": 0.7234591245651245, "reward_std": 0.6643977761268616, "rewards/rollout_reward_func/mean": 0.7234591245651245, "rewards/rollout_reward_func/std": 0.6661136746406555, "sampling/importance_sampling_ratio/max": 1.06000816822052, "sampling/importance_sampling_ratio/mean": 0.9971582889556885, "sampling/importance_sampling_ratio/min": 0.8888165354728699, "sampling/sampling_logp_difference/max": 0.15231943130493164, "sampling/sampling_logp_difference/mean": 0.02228650450706482, "step": 803, "step_time": 11.816145636998044 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0107421875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0107421875, "entropy": 0.9422279708087444, "epoch": 0.00804, "grad_norm": 0.5960602760314941, "kl": 0.5344425458461046, "learning_rate": 1.8798109803519665e-05, "loss": 0.1086, "step": 804, "step_time": 6.572312853001677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 301.09375, "completions/mean_terminated_length": 301.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5594793818891048, "epoch": 0.00805, "frac_reward_zero_std": 0.25, "grad_norm": 0.10633870959281921, "kl": 0.370867058634758, "learning_rate": 1.874645029395485e-05, "loss": -0.0016, "num_tokens": 17279626.0, "reward": 0.6904571652412415, "reward_std": 0.634080171585083, "rewards/rollout_reward_func/mean": 0.6904571652412415, "rewards/rollout_reward_func/std": 0.730880618095398, "sampling/importance_sampling_ratio/max": 1.1300758123397827, "sampling/importance_sampling_ratio/mean": 1.00905179977417, "sampling/importance_sampling_ratio/min": 0.9394782185554504, "sampling/sampling_logp_difference/max": 0.12191057205200195, "sampling/sampling_logp_difference/mean": 0.010468622669577599, "step": 805, "step_time": 9.349867630998233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5540013313293457, "epoch": 0.00806, "grad_norm": 0.1018148884177208, "kl": 0.38068171590566635, "learning_rate": 1.8694863660884452e-05, "loss": -0.0021, "step": 806, "step_time": 4.850892459000534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2294.0, "completions/max_terminated_length": 2294.0, "completions/mean_length": 556.15625, "completions/mean_terminated_length": 556.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8128577806055546, "epoch": 0.00807, "frac_reward_zero_std": 0.0, "grad_norm": 0.1866709291934967, "kl": 1.0498031359165907, "learning_rate": 1.8643350303039273e-05, "loss": 0.0232, "num_tokens": 17326660.0, "reward": 0.8422096967697144, "reward_std": 0.4462984800338745, "rewards/rollout_reward_func/mean": 0.8422096967697144, "rewards/rollout_reward_func/std": 0.4486958682537079, "sampling/importance_sampling_ratio/max": 1.079418659210205, "sampling/importance_sampling_ratio/mean": 0.969811201095581, "sampling/importance_sampling_ratio/min": 0.48734116554260254, "sampling/sampling_logp_difference/max": 0.2489781379699707, "sampling/sampling_logp_difference/mean": 0.019320998340845108, "step": 807, "step_time": 12.43609609500163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 0.8115962371230125, "epoch": 0.00808, "grad_norm": 0.16909019649028778, "kl": 0.9732256401330233, "learning_rate": 1.8591910618583704e-05, "loss": 0.0221, "step": 808, "step_time": 6.4931173160011895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 504.6875, "completions/mean_terminated_length": 504.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6261922530829906, "epoch": 0.00809, "frac_reward_zero_std": 0.5, "grad_norm": 0.12577398121356964, "kl": 0.39922066405415535, "learning_rate": 1.8540545005112728e-05, "loss": 0.0015, "num_tokens": 17371032.0, "reward": 0.837408721446991, "reward_std": 0.3128599524497986, "rewards/rollout_reward_func/mean": 0.837408721446991, "rewards/rollout_reward_func/std": 0.45759639143943787, "sampling/importance_sampling_ratio/max": 1.100976824760437, "sampling/importance_sampling_ratio/mean": 0.9953517913818359, "sampling/importance_sampling_ratio/min": 0.9003557562828064, "sampling/sampling_logp_difference/max": 0.11590003967285156, "sampling/sampling_logp_difference/mean": 0.010209660045802593, "step": 809, "step_time": 9.598750050999115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6243331134319305, "epoch": 0.0081, "grad_norm": 0.11105905473232269, "kl": 0.3913662824779749, "learning_rate": 1.848925385964876e-05, "loss": 0.0012, "step": 810, "step_time": 5.2525319400001536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 459.125, "completions/mean_terminated_length": 459.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6225273981690407, "epoch": 0.00811, "frac_reward_zero_std": 0.0, "grad_norm": 0.21113751828670502, "kl": 0.4443350825458765, "learning_rate": 1.8438037578638693e-05, "loss": -0.0032, "num_tokens": 17414551.0, "reward": 0.7195037603378296, "reward_std": 0.7108051776885986, "rewards/rollout_reward_func/mean": 0.7195037603378296, "rewards/rollout_reward_func/std": 0.6780244708061218, "sampling/importance_sampling_ratio/max": 1.108416199684143, "sampling/importance_sampling_ratio/mean": 0.9968690872192383, "sampling/importance_sampling_ratio/min": 0.9382011294364929, "sampling/sampling_logp_difference/max": 0.12421655654907227, "sampling/sampling_logp_difference/mean": 0.009783817455172539, "step": 811, "step_time": 10.128396140000405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.016666667070239782, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.016666667070239782, "entropy": 0.6175422817468643, "epoch": 0.00812, "grad_norm": 0.10984614491462708, "kl": 0.4579138122498989, "learning_rate": 1.8386896557950703e-05, "loss": -0.0041, "step": 812, "step_time": 5.199107100000219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1668.0, "completions/max_terminated_length": 1668.0, "completions/mean_length": 490.375, "completions/mean_terminated_length": 490.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9272086024284363, "epoch": 0.00813, "frac_reward_zero_std": 0.0, "grad_norm": 0.3328687846660614, "kl": 0.49560951814055443, "learning_rate": 1.8335831192871282e-05, "loss": 0.0308, "num_tokens": 17459123.0, "reward": 0.4769267439842224, "reward_std": 0.7870758771896362, "rewards/rollout_reward_func/mean": 0.4769267439842224, "rewards/rollout_reward_func/std": 0.8219716548919678, "sampling/importance_sampling_ratio/max": 1.257576584815979, "sampling/importance_sampling_ratio/mean": 1.0080196857452393, "sampling/importance_sampling_ratio/min": 0.8979253768920898, "sampling/sampling_logp_difference/max": 0.15733814239501953, "sampling/sampling_logp_difference/mean": 0.020043808966875076, "step": 813, "step_time": 9.646655776000443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.009868420660495758, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009868420660495758, "entropy": 0.9238025993108749, "epoch": 0.00814, "grad_norm": 0.24926318228244781, "kl": 0.5032154396176338, "learning_rate": 1.828484187810216e-05, "loss": 0.0272, "step": 814, "step_time": 5.792188376000922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 442.375, "completions/mean_terminated_length": 442.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.584303256124258, "epoch": 0.00815, "frac_reward_zero_std": 0.25, "grad_norm": 0.10933994501829147, "kl": 0.2873958293348551, "learning_rate": 1.8233929007757217e-05, "loss": -0.0083, "num_tokens": 17501449.0, "reward": 0.6847631335258484, "reward_std": 0.5920246243476868, "rewards/rollout_reward_func/mean": 0.6847631335258484, "rewards/rollout_reward_func/std": 0.6922931671142578, "sampling/importance_sampling_ratio/max": 1.0633004903793335, "sampling/importance_sampling_ratio/mean": 1.0057899951934814, "sampling/importance_sampling_ratio/min": 0.9231852889060974, "sampling/sampling_logp_difference/max": 0.07829296588897705, "sampling/sampling_logp_difference/mean": 0.007357490714639425, "step": 815, "step_time": 9.209577808999711 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.575192965567112, "epoch": 0.00816, "grad_norm": 0.09995918720960617, "kl": 0.30142701230943203, "learning_rate": 1.8183092975359523e-05, "loss": -0.009, "step": 816, "step_time": 5.048417359002087 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1643.0, "completions/max_terminated_length": 1643.0, "completions/mean_length": 464.0, "completions/mean_terminated_length": 464.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5419336371123791, "epoch": 0.00817, "frac_reward_zero_std": 0.0, "grad_norm": 0.15767812728881836, "kl": 0.3259985875338316, "learning_rate": 1.813233417383818e-05, "loss": 0.0056, "num_tokens": 17544222.0, "reward": 0.7776088714599609, "reward_std": 0.5458219051361084, "rewards/rollout_reward_func/mean": 0.7776088714599609, "rewards/rollout_reward_func/std": 0.5504268407821655, "sampling/importance_sampling_ratio/max": 1.1310229301452637, "sampling/importance_sampling_ratio/mean": 0.9913661479949951, "sampling/importance_sampling_ratio/min": 0.9026395678520203, "sampling/sampling_logp_difference/max": 0.10280752182006836, "sampling/sampling_logp_difference/mean": 0.013562308624386787, "step": 817, "step_time": 9.597462199000802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.020833333488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.020833333488553762, "entropy": 0.5324072502553463, "epoch": 0.00818, "grad_norm": 0.09963635355234146, "kl": 0.32988400012254715, "learning_rate": 1.8081652995525394e-05, "loss": 0.0046, "step": 818, "step_time": 6.013900699997976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 607.8125, "completions/mean_terminated_length": 607.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6902766674757004, "epoch": 0.00819, "frac_reward_zero_std": 0.0, "grad_norm": 0.17526385188102722, "kl": 0.5607752511277795, "learning_rate": 1.803104983215336e-05, "loss": 0.0039, "num_tokens": 17593359.0, "reward": 0.8104056119918823, "reward_std": 0.5362539291381836, "rewards/rollout_reward_func/mean": 0.8104056119918823, "rewards/rollout_reward_func/std": 0.5337495803833008, "sampling/importance_sampling_ratio/max": 1.186792016029358, "sampling/importance_sampling_ratio/mean": 0.9992239475250244, "sampling/importance_sampling_ratio/min": 0.8204982280731201, "sampling/sampling_logp_difference/max": 0.13580632209777832, "sampling/sampling_logp_difference/mean": 0.011159347370266914, "step": 819, "step_time": 9.403958971000066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.688994437456131, "epoch": 0.0082, "grad_norm": 0.09301353991031647, "kl": 0.5652805492281914, "learning_rate": 1.7980525074851282e-05, "loss": 0.003, "step": 820, "step_time": 5.051986125999974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1650.0, "completions/max_terminated_length": 1650.0, "completions/mean_length": 560.59375, "completions/mean_terminated_length": 577.6451416015625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8581799566745758, "epoch": 0.00821, "frac_reward_zero_std": 0.0, "grad_norm": 0.6134741902351379, "kl": 0.4007783271372318, "learning_rate": 1.7930079114142354e-05, "loss": 0.0561, "num_tokens": 17640151.0, "reward": 0.747625470161438, "reward_std": 0.5929499268531799, "rewards/rollout_reward_func/mean": 0.747625470161438, "rewards/rollout_reward_func/std": 0.6231030821800232, "sampling/importance_sampling_ratio/max": 1.3019139766693115, "sampling/importance_sampling_ratio/mean": 1.021460771560669, "sampling/importance_sampling_ratio/min": 0.9059603810310364, "sampling/sampling_logp_difference/max": 0.16656875610351562, "sampling/sampling_logp_difference/mean": 0.018153512850403786, "step": 821, "step_time": 10.340858419000142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.8576586730778217, "epoch": 0.00822, "grad_norm": 0.3965068757534027, "kl": 0.4040851350873709, "learning_rate": 1.7879712339940685e-05, "loss": 0.049, "step": 822, "step_time": 5.416722769000444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1483.0, "completions/max_terminated_length": 1483.0, "completions/mean_length": 430.125, "completions/mean_terminated_length": 430.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.586790032684803, "epoch": 0.00823, "frac_reward_zero_std": 0.0, "grad_norm": 0.1666020303964615, "kl": 0.5309692248702049, "learning_rate": 1.7829425141548378e-05, "loss": 0.0088, "num_tokens": 17681577.0, "reward": 0.7513085603713989, "reward_std": 0.6205500364303589, "rewards/rollout_reward_func/mean": 0.7513085603713989, "rewards/rollout_reward_func/std": 0.6113945245742798, "sampling/importance_sampling_ratio/max": 1.098021388053894, "sampling/importance_sampling_ratio/mean": 0.9892895817756653, "sampling/importance_sampling_ratio/min": 0.8063631057739258, "sampling/sampling_logp_difference/max": 0.10606491565704346, "sampling/sampling_logp_difference/mean": 0.01382577233016491, "step": 823, "step_time": 9.17580976499994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.5847207903862, "epoch": 0.00824, "grad_norm": 0.12842993438243866, "kl": 0.536888238042593, "learning_rate": 1.7779217907652402e-05, "loss": 0.0077, "step": 824, "step_time": 4.961058958000649 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1704.0, "completions/max_terminated_length": 1704.0, "completions/mean_length": 503.625, "completions/mean_terminated_length": 468.45159912109375, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 0.7901991531252861, "epoch": 0.00825, "frac_reward_zero_std": 0.25, "grad_norm": 0.2529870569705963, "kl": 1.2607399094849825, "learning_rate": 1.772909102632172e-05, "loss": -0.006, "num_tokens": 17726951.0, "reward": 0.6626237630844116, "reward_std": 0.6361265182495117, "rewards/rollout_reward_func/mean": 0.6626237630844116, "rewards/rollout_reward_func/std": 0.7305635809898376, "sampling/importance_sampling_ratio/max": 1.1889466047286987, "sampling/importance_sampling_ratio/mean": 0.9888510704040527, "sampling/importance_sampling_ratio/min": 0.8959532976150513, "sampling/sampling_logp_difference/max": 0.1669936180114746, "sampling/sampling_logp_difference/mean": 0.022862080484628677, "step": 825, "step_time": 10.57013264300167 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.012500000186264515, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013920454774051905, "entropy": 0.7865728661417961, "epoch": 0.00826, "grad_norm": 0.21634453535079956, "kl": 1.2954089920967817, "learning_rate": 1.7679044885004162e-05, "loss": -0.0075, "step": 826, "step_time": 5.368305034999139 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 390.5, "completions/mean_terminated_length": 390.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.634742759168148, "epoch": 0.00827, "frac_reward_zero_std": 0.25, "grad_norm": 0.17539262771606445, "kl": 1.0850961543619633, "learning_rate": 1.7629079870523532e-05, "loss": -0.0054, "num_tokens": 17766948.0, "reward": 0.7774483561515808, "reward_std": 0.4658278822898865, "rewards/rollout_reward_func/mean": 0.7774483561515808, "rewards/rollout_reward_func/std": 0.5528864860534668, "sampling/importance_sampling_ratio/max": 1.3723915815353394, "sampling/importance_sampling_ratio/mean": 0.9984395503997803, "sampling/importance_sampling_ratio/min": 0.9045182466506958, "sampling/sampling_logp_difference/max": 0.12258148193359375, "sampling/sampling_logp_difference/mean": 0.015162620693445206, "step": 827, "step_time": 9.761942398999963 }, { "clip_ratio/high_max": 0.018181818537414074, "clip_ratio/high_mean": 0.009090909268707037, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009090909268707037, "entropy": 0.637425571680069, "epoch": 0.00828, "grad_norm": 0.15859375894069672, "kl": 1.090258989483118, "learning_rate": 1.757919636907657e-05, "loss": -0.0063, "step": 828, "step_time": 5.099692557001617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1229.0, "completions/max_terminated_length": 1229.0, "completions/mean_length": 416.75, "completions/mean_terminated_length": 402.1290283203125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6780529981479049, "epoch": 0.00829, "frac_reward_zero_std": 0.0, "grad_norm": 0.4411081373691559, "kl": 0.38134322315454483, "learning_rate": 1.7529394766229953e-05, "loss": 0.0606, "num_tokens": 17808520.0, "reward": 0.5832362771034241, "reward_std": 0.6518243551254272, "rewards/rollout_reward_func/mean": 0.5832362771034241, "rewards/rollout_reward_func/std": 0.6656351089477539, "sampling/importance_sampling_ratio/max": 1.0967344045639038, "sampling/importance_sampling_ratio/mean": 1.0038203001022339, "sampling/importance_sampling_ratio/min": 0.7653203010559082, "sampling/sampling_logp_difference/max": 0.1235346794128418, "sampling/sampling_logp_difference/mean": 0.012947607785463333, "step": 829, "step_time": 8.260996857000464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.01145833358168602, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01145833358168602, "entropy": 0.6753100454807281, "epoch": 0.0083, "grad_norm": 0.31322282552719116, "kl": 0.3816859591752291, "learning_rate": 1.7479675446917368e-05, "loss": 0.0565, "step": 830, "step_time": 4.357579755998813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1607.0, "completions/max_terminated_length": 1607.0, "completions/mean_length": 502.875, "completions/mean_terminated_length": 476.6128845214844, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9902428835630417, "epoch": 0.00831, "frac_reward_zero_std": 0.0, "grad_norm": 0.3884986937046051, "kl": 0.6494214721024036, "learning_rate": 1.7430038795436465e-05, "loss": -0.0716, "num_tokens": 17854019.0, "reward": 0.5717272758483887, "reward_std": 0.6586064100265503, "rewards/rollout_reward_func/mean": 0.5717272758483887, "rewards/rollout_reward_func/std": 0.6334887742996216, "sampling/importance_sampling_ratio/max": 1.403915524482727, "sampling/importance_sampling_ratio/mean": 0.9566042423248291, "sampling/importance_sampling_ratio/min": 2.579555492820962e-14, "sampling/sampling_logp_difference/max": 26.10762596130371, "sampling/sampling_logp_difference/mean": 0.2687641382217407, "step": 831, "step_time": 10.046794472999864 }, { "clip_ratio/high_max": 0.0035714285913854837, "clip_ratio/high_mean": 0.0017857142956927419, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017857142956927419, "entropy": 0.9902498796582222, "epoch": 0.00832, "grad_norm": 0.34308061003685, "kl": 0.6423410270363092, "learning_rate": 1.738048519544596e-05, "loss": -0.0745, "step": 832, "step_time": 5.7773752100010825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 431.0625, "completions/mean_terminated_length": 431.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6437488123774529, "epoch": 0.00833, "frac_reward_zero_std": 0.25, "grad_norm": 0.42441466450691223, "kl": 0.4435964599251747, "learning_rate": 1.7331015029962627e-05, "loss": 0.0407, "num_tokens": 17896351.0, "reward": 0.8406949639320374, "reward_std": 0.385130912065506, "rewards/rollout_reward_func/mean": 0.8406949639320374, "rewards/rollout_reward_func/std": 0.44763463735580444, "sampling/importance_sampling_ratio/max": 1.1095787286758423, "sampling/importance_sampling_ratio/mean": 0.9890403151512146, "sampling/importance_sampling_ratio/min": 0.8491475582122803, "sampling/sampling_logp_difference/max": 0.1259317398071289, "sampling/sampling_logp_difference/mean": 0.01556488312780857, "step": 833, "step_time": 9.07061370899737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.021875000093132257, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.021875000093132257, "entropy": 0.6404992360621691, "epoch": 0.00834, "grad_norm": 0.2741653323173523, "kl": 0.44227669946849346, "learning_rate": 1.728162868135833e-05, "loss": 0.0383, "step": 834, "step_time": 5.263087070999063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1355.0, "completions/max_terminated_length": 1355.0, "completions/mean_length": 395.0, "completions/mean_terminated_length": 406.70965576171875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6134405210614204, "epoch": 0.00835, "frac_reward_zero_std": 0.0, "grad_norm": 0.12271725386381149, "kl": 0.4092550929635763, "learning_rate": 1.723232653135711e-05, "loss": -0.0149, "num_tokens": 17938357.0, "reward": 0.5719802379608154, "reward_std": 0.6296789050102234, "rewards/rollout_reward_func/mean": 0.5719802379608154, "rewards/rollout_reward_func/std": 0.6325890421867371, "sampling/importance_sampling_ratio/max": 1.0957465171813965, "sampling/importance_sampling_ratio/mean": 0.9890608787536621, "sampling/importance_sampling_ratio/min": 0.8510617017745972, "sampling/sampling_logp_difference/max": 0.16722559928894043, "sampling/sampling_logp_difference/mean": 0.01469497848302126, "step": 835, "step_time": 8.575489742000173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6138565242290497, "epoch": 0.00836, "grad_norm": 0.12620605528354645, "kl": 0.41068775206804276, "learning_rate": 1.7183108961032178e-05, "loss": -0.0153, "step": 836, "step_time": 4.675016328998936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 412.96875, "completions/mean_terminated_length": 412.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5650873892009258, "epoch": 0.00837, "frac_reward_zero_std": 0.0, "grad_norm": 0.19490109384059906, "kl": 0.29362941160798073, "learning_rate": 1.7133976350803022e-05, "loss": 0.0016, "num_tokens": 17979528.0, "reward": 0.6757820844650269, "reward_std": 0.624184787273407, "rewards/rollout_reward_func/mean": 0.6757820844650269, "rewards/rollout_reward_func/std": 0.604274570941925, "sampling/importance_sampling_ratio/max": 1.144953966140747, "sampling/importance_sampling_ratio/mean": 1.009097933769226, "sampling/importance_sampling_ratio/min": 0.8418552875518799, "sampling/sampling_logp_difference/max": 0.11423516273498535, "sampling/sampling_logp_difference/mean": 0.011158619076013565, "step": 837, "step_time": 9.432566783000766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.02500000037252903, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02500000037252903, "entropy": 0.5747268386185169, "epoch": 0.00838, "grad_norm": 0.07830584794282913, "kl": 0.28987837582826614, "learning_rate": 1.7084929080432442e-05, "loss": 0.0005, "step": 838, "step_time": 5.198808381999697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 835.0, "completions/max_terminated_length": 835.0, "completions/mean_length": 324.40625, "completions/mean_terminated_length": 324.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7351261451840401, "epoch": 0.00839, "frac_reward_zero_std": 0.0, "grad_norm": 0.41943031549453735, "kl": 0.47842644341289997, "learning_rate": 1.703596752902361e-05, "loss": 0.0227, "num_tokens": 18018176.0, "reward": 0.4856956899166107, "reward_std": 0.7516698241233826, "rewards/rollout_reward_func/mean": 0.4856956899166107, "rewards/rollout_reward_func/std": 0.7652511596679688, "sampling/importance_sampling_ratio/max": 1.4193511009216309, "sampling/importance_sampling_ratio/mean": 1.0077663660049438, "sampling/importance_sampling_ratio/min": 0.8719865679740906, "sampling/sampling_logp_difference/max": 0.17667102813720703, "sampling/sampling_logp_difference/mean": 0.018227115273475647, "step": 839, "step_time": 7.744714663001105 }, { "clip_ratio/high_max": 0.0052083334885537624, "clip_ratio/high_mean": 0.0026041667442768812, "clip_ratio/low_mean": 0.0069444444961845875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009548611240461469, "entropy": 0.7387723624706268, "epoch": 0.0084, "grad_norm": 0.32749274373054504, "kl": 0.47597021237015724, "learning_rate": 1.6987092075017148e-05, "loss": 0.018, "step": 840, "step_time": 3.728832244998557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1632.0, "completions/max_terminated_length": 1632.0, "completions/mean_length": 430.625, "completions/mean_terminated_length": 430.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6658781282603741, "epoch": 0.00841, "frac_reward_zero_std": 0.0, "grad_norm": 0.13846859335899353, "kl": 0.6501933448016644, "learning_rate": 1.6938303096188206e-05, "loss": 0.018, "num_tokens": 18060525.0, "reward": 0.5305547714233398, "reward_std": 0.7915467023849487, "rewards/rollout_reward_func/mean": 0.5305547714233398, "rewards/rollout_reward_func/std": 0.793770432472229, "sampling/importance_sampling_ratio/max": 1.1066031455993652, "sampling/importance_sampling_ratio/mean": 0.9680131673812866, "sampling/importance_sampling_ratio/min": 1.6649147338565484e-10, "sampling/sampling_logp_difference/max": 22.423511505126953, "sampling/sampling_logp_difference/mean": 0.19516561925411224, "step": 841, "step_time": 9.972299451999788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6714333221316338, "epoch": 0.00842, "grad_norm": 0.13609547913074493, "kl": 0.6524445451796055, "learning_rate": 1.6889600969643523e-05, "loss": 0.0172, "step": 842, "step_time": 5.208898222001153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 474.34375, "completions/mean_terminated_length": 443.6773986816406, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8974368795752525, "epoch": 0.00843, "frac_reward_zero_std": 0.0, "grad_norm": 0.29948922991752625, "kl": 1.184892512857914, "learning_rate": 1.6840986071818553e-05, "loss": -0.0371, "num_tokens": 18105254.0, "reward": 0.495932012796402, "reward_std": 0.7852356433868408, "rewards/rollout_reward_func/mean": 0.495932012796402, "rewards/rollout_reward_func/std": 0.7977966070175171, "sampling/importance_sampling_ratio/max": 1.6861335039138794, "sampling/importance_sampling_ratio/mean": 1.0419996976852417, "sampling/importance_sampling_ratio/min": 0.9120880365371704, "sampling/sampling_logp_difference/max": 0.21128582954406738, "sampling/sampling_logp_difference/mean": 0.02362579107284546, "step": 843, "step_time": 9.776801540999259 }, { "clip_ratio/high_max": 0.022857143077999353, "clip_ratio/high_mean": 0.011428571538999677, "clip_ratio/low_mean": 0.0027173913549631834, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01414596289396286, "entropy": 0.9065736830234528, "epoch": 0.00844, "grad_norm": 0.24236471951007843, "kl": 1.2188309766352177, "learning_rate": 1.6792458778474518e-05, "loss": -0.0391, "step": 844, "step_time": 4.977362583997092 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1557.0, "completions/max_terminated_length": 1557.0, "completions/mean_length": 345.625, "completions/mean_terminated_length": 345.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6952201388776302, "epoch": 0.00845, "frac_reward_zero_std": 0.25, "grad_norm": 0.20126445591449738, "kl": 0.7193928547203541, "learning_rate": 1.6744019464695502e-05, "loss": -0.0187, "num_tokens": 18143405.0, "reward": 0.6283847689628601, "reward_std": 0.6259229183197021, "rewards/rollout_reward_func/mean": 0.6283847689628601, "rewards/rollout_reward_func/std": 0.7403914332389832, "sampling/importance_sampling_ratio/max": 1.1176772117614746, "sampling/importance_sampling_ratio/mean": 0.9794078469276428, "sampling/importance_sampling_ratio/min": 0.6988672614097595, "sampling/sampling_logp_difference/max": 0.19925332069396973, "sampling/sampling_logp_difference/mean": 0.018284859135746956, "step": 845, "step_time": 9.257862693000789 }, { "clip_ratio/high_max": 0.012239583767950535, "clip_ratio/high_mean": 0.006119791883975267, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006119791883975267, "entropy": 0.7107310555875301, "epoch": 0.00846, "grad_norm": 0.1811259388923645, "kl": 0.7275427840650082, "learning_rate": 1.6695668504885568e-05, "loss": -0.019, "step": 846, "step_time": 5.368709393999779 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1625.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 467.90625, "completions/mean_terminated_length": 416.70001220703125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.022416815161705, "epoch": 0.00847, "frac_reward_zero_std": 0.25, "grad_norm": 0.6596115231513977, "kl": 0.9140776917338371, "learning_rate": 1.6647406272765875e-05, "loss": -0.0023, "num_tokens": 18186875.0, "reward": 0.5934839248657227, "reward_std": 0.6354612708091736, "rewards/rollout_reward_func/mean": 0.5934839248657227, "rewards/rollout_reward_func/std": 0.7512715458869934, "sampling/importance_sampling_ratio/max": 1.489582896232605, "sampling/importance_sampling_ratio/mean": 0.9964454174041748, "sampling/importance_sampling_ratio/min": 0.01039195992052555, "sampling/sampling_logp_difference/max": 1.7582601308822632, "sampling/sampling_logp_difference/mean": 0.04449235647916794, "step": 847, "step_time": 9.90133637299914 }, { "clip_ratio/high_max": 0.025657895021140575, "clip_ratio/high_mean": 0.012828947510570288, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012828947510570288, "entropy": 1.0304998457431793, "epoch": 0.00848, "grad_norm": 0.5858210921287537, "kl": 0.9378763698041439, "learning_rate": 1.6599233141371773e-05, "loss": -0.0115, "step": 848, "step_time": 5.749863464001464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 797.0, "completions/max_terminated_length": 797.0, "completions/mean_length": 247.0, "completions/mean_terminated_length": 247.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6923670396208763, "epoch": 0.00849, "frac_reward_zero_std": 0.0, "grad_norm": 0.16954372823238373, "kl": 0.8596753366291523, "learning_rate": 1.655114948304992e-05, "loss": 0.0068, "num_tokens": 18221213.0, "reward": 0.5878082513809204, "reward_std": 0.7735468149185181, "rewards/rollout_reward_func/mean": 0.5878082513809204, "rewards/rollout_reward_func/std": 0.7573027610778809, "sampling/importance_sampling_ratio/max": 1.0716062784194946, "sampling/importance_sampling_ratio/mean": 0.9940816164016724, "sampling/importance_sampling_ratio/min": 0.8688737154006958, "sampling/sampling_logp_difference/max": 0.12072563171386719, "sampling/sampling_logp_difference/mean": 0.009906920604407787, "step": 849, "step_time": 6.786763114997484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.7071281149983406, "epoch": 0.0085, "grad_norm": 0.1539735049009323, "kl": 0.9315300434827805, "learning_rate": 1.650315566945539e-05, "loss": 0.0057, "step": 850, "step_time": 3.623146545000054 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2127.0, "completions/max_terminated_length": 2127.0, "completions/mean_length": 413.96875, "completions/mean_terminated_length": 400.70965576171875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0453823953866959, "epoch": 0.00851, "frac_reward_zero_std": 0.25, "grad_norm": 0.7330817580223083, "kl": 0.74776766076684, "learning_rate": 1.6455252071548836e-05, "loss": 0.084, "num_tokens": 18262125.0, "reward": 0.6599608659744263, "reward_std": 0.6278988718986511, "rewards/rollout_reward_func/mean": 0.6599608659744263, "rewards/rollout_reward_func/std": 0.7350403070449829, "sampling/importance_sampling_ratio/max": 1.5863049030303955, "sampling/importance_sampling_ratio/mean": 1.054802656173706, "sampling/importance_sampling_ratio/min": 0.8288826942443848, "sampling/sampling_logp_difference/max": 0.1862635612487793, "sampling/sampling_logp_difference/mean": 0.02565549686551094, "step": 851, "step_time": 10.876475875000324 }, { "clip_ratio/high_max": 0.025493420660495758, "clip_ratio/high_mean": 0.012746710330247879, "clip_ratio/low_mean": 0.007708333432674408, "clip_ratio/low_min": 0.004999999888241291, "clip_ratio/region_mean": 0.020455043762922287, "entropy": 1.057716403156519, "epoch": 0.00852, "grad_norm": 0.5790297985076904, "kl": 0.7421135678887367, "learning_rate": 1.6407439059593587e-05, "loss": 0.0758, "step": 852, "step_time": 5.954823398999906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1659.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 486.15625, "completions/mean_terminated_length": 483.1612854003906, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9388038665056229, "epoch": 0.00853, "frac_reward_zero_std": 0.25, "grad_norm": 0.3183837831020355, "kl": 0.964560117572546, "learning_rate": 1.6359717003152818e-05, "loss": -0.011, "num_tokens": 18306480.0, "reward": 0.6313971877098083, "reward_std": 0.4259287714958191, "rewards/rollout_reward_func/mean": 0.6313971877098083, "rewards/rollout_reward_func/std": 0.5179029703140259, "sampling/importance_sampling_ratio/max": 1.3218410015106201, "sampling/importance_sampling_ratio/mean": 0.9848277568817139, "sampling/importance_sampling_ratio/min": 0.6494459509849548, "sampling/sampling_logp_difference/max": 0.15744495391845703, "sampling/sampling_logp_difference/mean": 0.018823347985744476, "step": 853, "step_time": 10.56415467000079 }, { "clip_ratio/high_max": 0.013620689511299133, "clip_ratio/high_mean": 0.006810344755649567, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006810344755649567, "entropy": 0.9482126235961914, "epoch": 0.00854, "grad_norm": 0.2846826910972595, "kl": 0.9811736717820168, "learning_rate": 1.631208627108668e-05, "loss": -0.0131, "step": 854, "step_time": 5.814450474000296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1659.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 490.90625, "completions/mean_terminated_length": 509.7930908203125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1209965869784355, "epoch": 0.00855, "frac_reward_zero_std": 0.0, "grad_norm": 0.7990384101867676, "kl": 0.783106192946434, "learning_rate": 1.626454723154942e-05, "loss": 0.0883, "num_tokens": 18350732.0, "reward": 0.7140394449234009, "reward_std": 0.6336410045623779, "rewards/rollout_reward_func/mean": 0.7140394449234009, "rewards/rollout_reward_func/std": 0.6386069655418396, "sampling/importance_sampling_ratio/max": 1.545379877090454, "sampling/importance_sampling_ratio/mean": 1.0035662651062012, "sampling/importance_sampling_ratio/min": 0.5765411257743835, "sampling/sampling_logp_difference/max": 0.2723813056945801, "sampling/sampling_logp_difference/mean": 0.026247646659612656, "step": 855, "step_time": 10.378807953000432 }, { "clip_ratio/high_max": 0.010347985429689288, "clip_ratio/high_mean": 0.005173992714844644, "clip_ratio/low_mean": 0.00657894741743803, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011752940132282674, "entropy": 1.121246598660946, "epoch": 0.00856, "grad_norm": 0.6105203628540039, "kl": 0.7985721901059151, "learning_rate": 1.6217100251986594e-05, "loss": 0.0801, "step": 856, "step_time": 5.30280498600041 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 1580.0, "completions/max_terminated_length": 1475.0, "completions/mean_length": 416.34375, "completions/mean_terminated_length": 385.239990234375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.444454200565815, "epoch": 0.00857, "frac_reward_zero_std": 0.0, "grad_norm": 0.6609808802604675, "kl": 1.174719586968422, "learning_rate": 1.6169745699132167e-05, "loss": 0.0143, "num_tokens": 18391342.0, "reward": 0.7995470762252808, "reward_std": 0.43618226051330566, "rewards/rollout_reward_func/mean": 0.7995470762252808, "rewards/rollout_reward_func/std": 0.42450666427612305, "sampling/importance_sampling_ratio/max": 2.239652633666992, "sampling/importance_sampling_ratio/mean": 0.9717288613319397, "sampling/importance_sampling_ratio/min": 6.437855890745192e-12, "sampling/sampling_logp_difference/max": 20.622623443603516, "sampling/sampling_logp_difference/mean": 0.09853711724281311, "step": 857, "step_time": 10.94152129899885 }, { "clip_ratio/high_max": 0.025887957075610757, "clip_ratio/high_mean": 0.015494999126531184, "clip_ratio/low_mean": 0.006619458319619298, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02211445732973516, "entropy": 1.4435777589678764, "epoch": 0.00858, "grad_norm": 0.5645499229431152, "kl": 1.1509734317660332, "learning_rate": 1.6122483939005726e-05, "loss": 0.0066, "step": 858, "step_time": 5.184616157998789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 849.0, "completions/max_terminated_length": 849.0, "completions/mean_length": 387.125, "completions/mean_terminated_length": 387.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8111044615507126, "epoch": 0.00859, "frac_reward_zero_std": 0.25, "grad_norm": 0.13186568021774292, "kl": 0.9172727223485708, "learning_rate": 1.607531533690963e-05, "loss": -0.023, "num_tokens": 18432552.0, "reward": 0.6862204074859619, "reward_std": 0.5706055164337158, "rewards/rollout_reward_func/mean": 0.6862204074859619, "rewards/rollout_reward_func/std": 0.6895962953567505, "sampling/importance_sampling_ratio/max": 1.1211779117584229, "sampling/importance_sampling_ratio/mean": 0.9930081963539124, "sampling/importance_sampling_ratio/min": 0.7542689442634583, "sampling/sampling_logp_difference/max": 0.16616439819335938, "sampling/sampling_logp_difference/mean": 0.01965990662574768, "step": 859, "step_time": 8.04133363800156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.8003969714045525, "epoch": 0.0086, "grad_norm": 0.12774905562400818, "kl": 0.9017507284879684, "learning_rate": 1.602824025742617e-05, "loss": -0.0231, "step": 860, "step_time": 3.7860343740003373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1506.0, "completions/max_terminated_length": 1506.0, "completions/mean_length": 572.375, "completions/mean_terminated_length": 572.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0884819775819778, "epoch": 0.00861, "frac_reward_zero_std": 0.25, "grad_norm": 0.4803817570209503, "kl": 1.2070488557219505, "learning_rate": 1.5981259064414796e-05, "loss": 0.0537, "num_tokens": 18480464.0, "reward": 0.711367130279541, "reward_std": 0.4952605068683624, "rewards/rollout_reward_func/mean": 0.711367130279541, "rewards/rollout_reward_func/std": 0.5859953761100769, "sampling/importance_sampling_ratio/max": 1.1102077960968018, "sampling/importance_sampling_ratio/mean": 0.9766106009483337, "sampling/importance_sampling_ratio/min": 0.7214534878730774, "sampling/sampling_logp_difference/max": 0.11936235427856445, "sampling/sampling_logp_difference/mean": 0.021583009511232376, "step": 861, "step_time": 9.630083940998702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.018863636534661055, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.018863636534661055, "entropy": 1.0748531445860863, "epoch": 0.00862, "grad_norm": 0.33699744939804077, "kl": 1.1914907954633236, "learning_rate": 1.5934372121009246e-05, "loss": 0.0488, "step": 862, "step_time": 5.0061141770002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1621.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 361.34375, "completions/mean_terminated_length": 320.70965576171875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8081045895814896, "epoch": 0.00863, "frac_reward_zero_std": 0.25, "grad_norm": 0.07906447350978851, "kl": 1.0259510092437267, "learning_rate": 1.5887579789614792e-05, "loss": 0.0014, "num_tokens": 18520213.0, "reward": 0.8359108567237854, "reward_std": 0.37988027930259705, "rewards/rollout_reward_func/mean": 0.8359108567237854, "rewards/rollout_reward_func/std": 0.4617302417755127, "sampling/importance_sampling_ratio/max": 1.097791075706482, "sampling/importance_sampling_ratio/mean": 0.997450590133667, "sampling/importance_sampling_ratio/min": 0.807340681552887, "sampling/sampling_logp_difference/max": 0.2024672031402588, "sampling/sampling_logp_difference/mean": 0.018279684707522392, "step": 863, "step_time": 9.554206809998504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.7878925427794456, "epoch": 0.00864, "grad_norm": 0.0697057917714119, "kl": 1.0233884565532207, "learning_rate": 1.5840882431905427e-05, "loss": 0.0009, "step": 864, "step_time": 5.187417306000498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 446.53125, "completions/mean_terminated_length": 446.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7566735446453094, "epoch": 0.00865, "frac_reward_zero_std": 0.75, "grad_norm": 0.338090717792511, "kl": 0.5756008718162775, "learning_rate": 1.579428040882102e-05, "loss": -0.0196, "num_tokens": 18562599.0, "reward": 0.7858991622924805, "reward_std": 0.23909883201122284, "rewards/rollout_reward_func/mean": 0.7858991622924805, "rewards/rollout_reward_func/std": 0.590336263179779, "sampling/importance_sampling_ratio/max": 1.1258666515350342, "sampling/importance_sampling_ratio/mean": 0.987260103225708, "sampling/importance_sampling_ratio/min": 0.7034458518028259, "sampling/sampling_logp_difference/max": 0.280519962310791, "sampling/sampling_logp_difference/mean": 0.016454819589853287, "step": 865, "step_time": 8.955013684998448 }, { "clip_ratio/high_max": 0.014285714365541935, "clip_ratio/high_mean": 0.0071428571827709675, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0071428571827709675, "entropy": 0.7540821693837643, "epoch": 0.00866, "grad_norm": 0.2520107924938202, "kl": 0.5686697289347649, "learning_rate": 1.5747774080564608e-05, "loss": -0.021, "step": 866, "step_time": 5.401126023999495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1364.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 590.3125, "completions/mean_terminated_length": 590.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6986459866166115, "epoch": 0.00867, "frac_reward_zero_std": 0.25, "grad_norm": 0.15174484252929688, "kl": 0.5670888070017099, "learning_rate": 1.5701363806599542e-05, "loss": -0.0066, "num_tokens": 18611339.0, "reward": 0.8644874095916748, "reward_std": 0.3178345561027527, "rewards/rollout_reward_func/mean": 0.8644874095916748, "rewards/rollout_reward_func/std": 0.3643455505371094, "sampling/importance_sampling_ratio/max": 1.1023398637771606, "sampling/importance_sampling_ratio/mean": 1.0043296813964844, "sampling/importance_sampling_ratio/min": 0.8480501174926758, "sampling/sampling_logp_difference/max": 0.12624680995941162, "sampling/sampling_logp_difference/mean": 0.015189290046691895, "step": 867, "step_time": 9.180908829000145 }, { "clip_ratio/high_max": 0.008333333767950535, "clip_ratio/high_mean": 0.004166666883975267, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004166666883975267, "entropy": 0.6985111944377422, "epoch": 0.00868, "grad_norm": 0.1529342085123062, "kl": 0.566157627850771, "learning_rate": 1.5655049945646755e-05, "loss": -0.007, "step": 868, "step_time": 5.361744148000071 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1436.0, "completions/max_terminated_length": 1436.0, "completions/mean_length": 368.6875, "completions/mean_terminated_length": 368.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8493942096829414, "epoch": 0.00869, "frac_reward_zero_std": 0.0, "grad_norm": 0.3290322422981262, "kl": 0.6456970199942589, "learning_rate": 1.5608832855681963e-05, "loss": 0.0529, "num_tokens": 18651170.0, "reward": 0.6361719369888306, "reward_std": 0.6349842548370361, "rewards/rollout_reward_func/mean": 0.6361719369888306, "rewards/rollout_reward_func/std": 0.6217851042747498, "sampling/importance_sampling_ratio/max": 1.1429613828659058, "sampling/importance_sampling_ratio/mean": 0.9896537065505981, "sampling/importance_sampling_ratio/min": 0.7881940007209778, "sampling/sampling_logp_difference/max": 0.20986557006835938, "sampling/sampling_logp_difference/mean": 0.018529575318098068, "step": 869, "step_time": 8.847045590000562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.005244755418971181, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005244755418971181, "entropy": 0.8335891515016556, "epoch": 0.0087, "grad_norm": 0.2744773030281067, "kl": 0.6367924120277166, "learning_rate": 1.5562712893932896e-05, "loss": 0.0503, "step": 870, "step_time": 4.761697363001076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 774.0, "completions/max_terminated_length": 774.0, "completions/mean_length": 271.1875, "completions/mean_terminated_length": 271.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.49326252192258835, "epoch": 0.00871, "frac_reward_zero_std": 0.25, "grad_norm": 0.1685665249824524, "kl": 0.7639194261282682, "learning_rate": 1.551669041687657e-05, "loss": 0.0098, "num_tokens": 18686919.0, "reward": 0.7978600263595581, "reward_std": 0.3598341941833496, "rewards/rollout_reward_func/mean": 0.7978600263595581, "rewards/rollout_reward_func/std": 0.42771151661872864, "sampling/importance_sampling_ratio/max": 1.1322752237319946, "sampling/importance_sampling_ratio/mean": 1.015150785446167, "sampling/importance_sampling_ratio/min": 0.9368591904640198, "sampling/sampling_logp_difference/max": 0.1186833381652832, "sampling/sampling_logp_difference/mean": 0.012482985854148865, "step": 871, "step_time": 7.132426797001244 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.01953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02734375, "entropy": 0.47973334044218063, "epoch": 0.00872, "grad_norm": 0.12259134650230408, "kl": 0.807329261675477, "learning_rate": 1.5470765780236483e-05, "loss": 0.009, "step": 872, "step_time": 3.66186314100014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1686.0, "completions/max_terminated_length": 1686.0, "completions/mean_length": 413.875, "completions/mean_terminated_length": 413.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7262389622628689, "epoch": 0.00873, "frac_reward_zero_std": 0.0, "grad_norm": 0.18846438825130463, "kl": 0.8126604929566383, "learning_rate": 1.542493933897991e-05, "loss": -0.0004, "num_tokens": 18729033.0, "reward": 0.6246352195739746, "reward_std": 0.7746298313140869, "rewards/rollout_reward_func/mean": 0.6246352195739746, "rewards/rollout_reward_func/std": 0.7454275488853455, "sampling/importance_sampling_ratio/max": 1.1169346570968628, "sampling/importance_sampling_ratio/mean": 1.0012295246124268, "sampling/importance_sampling_ratio/min": 0.8859577775001526, "sampling/sampling_logp_difference/max": 0.15108489990234375, "sampling/sampling_logp_difference/mean": 0.013166850432753563, "step": 873, "step_time": 10.191556227000547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.7316882386803627, "epoch": 0.00874, "grad_norm": 0.18278147280216217, "kl": 0.8084335662424564, "learning_rate": 1.537921144731512e-05, "loss": -0.0012, "step": 874, "step_time": 5.54612314500082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2317.0, "completions/max_terminated_length": 2317.0, "completions/mean_length": 615.75, "completions/mean_terminated_length": 615.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8608829155564308, "epoch": 0.00875, "frac_reward_zero_std": 0.0, "grad_norm": 0.31795039772987366, "kl": 0.6611455790698528, "learning_rate": 1.5333582458688696e-05, "loss": 0.014, "num_tokens": 18779311.0, "reward": 0.7506624460220337, "reward_std": 0.5866923332214355, "rewards/rollout_reward_func/mean": 0.7506624460220337, "rewards/rollout_reward_func/std": 0.6139788627624512, "sampling/importance_sampling_ratio/max": 1.2056176662445068, "sampling/importance_sampling_ratio/mean": 1.0135629177093506, "sampling/importance_sampling_ratio/min": 0.7205096483230591, "sampling/sampling_logp_difference/max": 0.16257786750793457, "sampling/sampling_logp_difference/mean": 0.02034096047282219, "step": 875, "step_time": 11.763852608000889 }, { "clip_ratio/high_max": 0.005681818351149559, "clip_ratio/high_mean": 0.0028409091755747795, "clip_ratio/low_mean": 0.0034722222480922937, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006313131423667073, "entropy": 0.8629271388053894, "epoch": 0.00876, "grad_norm": 0.28402289748191833, "kl": 0.6535944864153862, "learning_rate": 1.5288052725782717e-05, "loss": 0.012, "step": 876, "step_time": 6.555237004999981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 509.375, "completions/mean_terminated_length": 509.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.569519642740488, "epoch": 0.00877, "frac_reward_zero_std": 0.25, "grad_norm": 0.140402153134346, "kl": 0.5266147404909134, "learning_rate": 1.5242622600512118e-05, "loss": 0.0057, "num_tokens": 18824269.0, "reward": 0.8721362352371216, "reward_std": 0.36165332794189453, "rewards/rollout_reward_func/mean": 0.8721362352371216, "rewards/rollout_reward_func/std": 0.42485931515693665, "sampling/importance_sampling_ratio/max": 1.0734738111495972, "sampling/importance_sampling_ratio/mean": 0.9927952289581299, "sampling/importance_sampling_ratio/min": 0.8254187703132629, "sampling/sampling_logp_difference/max": 0.1213071346282959, "sampling/sampling_logp_difference/mean": 0.011409029364585876, "step": 877, "step_time": 9.57096757599993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5613879784941673, "epoch": 0.00878, "grad_norm": 0.14779220521450043, "kl": 0.5232104752212763, "learning_rate": 1.5197292434021916e-05, "loss": 0.0059, "step": 878, "step_time": 5.133185744998627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1364.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 438.03125, "completions/mean_terminated_length": 438.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5892351195216179, "epoch": 0.00879, "frac_reward_zero_std": 0.25, "grad_norm": 0.18478725850582123, "kl": 0.567435584962368, "learning_rate": 1.5152062576684497e-05, "loss": -0.0053, "num_tokens": 18867099.0, "reward": 0.7123662233352661, "reward_std": 0.5113439559936523, "rewards/rollout_reward_func/mean": 0.7123662233352661, "rewards/rollout_reward_func/std": 0.638980507850647, "sampling/importance_sampling_ratio/max": 1.1068183183670044, "sampling/importance_sampling_ratio/mean": 1.0023932456970215, "sampling/importance_sampling_ratio/min": 0.8845301270484924, "sampling/sampling_logp_difference/max": 0.12382149696350098, "sampling/sampling_logp_difference/mean": 0.010962624102830887, "step": 879, "step_time": 9.288684688002832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.584502574056387, "epoch": 0.0088, "grad_norm": 0.13640554249286652, "kl": 0.564317224547267, "learning_rate": 1.5106933378096969e-05, "loss": -0.0059, "step": 880, "step_time": 4.709965163000561 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1650.0, "completions/max_terminated_length": 1650.0, "completions/mean_length": 614.15625, "completions/mean_terminated_length": 614.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.703591275960207, "epoch": 0.00881, "frac_reward_zero_std": 0.0, "grad_norm": 0.17928127944469452, "kl": 0.5808097142726183, "learning_rate": 1.5061905187078368e-05, "loss": -0.0077, "num_tokens": 18915565.0, "reward": 0.8698825240135193, "reward_std": 0.36802783608436584, "rewards/rollout_reward_func/mean": 0.8698825240135193, "rewards/rollout_reward_func/std": 0.3501339256763458, "sampling/importance_sampling_ratio/max": 1.4752169847488403, "sampling/importance_sampling_ratio/mean": 1.0261328220367432, "sampling/importance_sampling_ratio/min": 0.7929633855819702, "sampling/sampling_logp_difference/max": 0.19882500171661377, "sampling/sampling_logp_difference/mean": 0.015989772975444794, "step": 881, "step_time": 10.481904042001588 }, { "clip_ratio/high_max": 0.011363636702299118, "clip_ratio/high_mean": 0.005681818351149559, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005681818351149559, "entropy": 0.7025555819272995, "epoch": 0.00882, "grad_norm": 0.16898435354232788, "kl": 0.583396889269352, "learning_rate": 1.501697835166704e-05, "loss": -0.0084, "step": 882, "step_time": 5.416424221999478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 409.71875, "completions/mean_terminated_length": 409.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5925387814640999, "epoch": 0.00883, "frac_reward_zero_std": 0.0, "grad_norm": 0.31778085231781006, "kl": 0.6217529810965061, "learning_rate": 1.4972153219117886e-05, "loss": 0.0331, "num_tokens": 18957002.0, "reward": 0.6767298579216003, "reward_std": 0.6195150017738342, "rewards/rollout_reward_func/mean": 0.6767298579216003, "rewards/rollout_reward_func/std": 0.6007736921310425, "sampling/importance_sampling_ratio/max": 1.2297755479812622, "sampling/importance_sampling_ratio/mean": 1.0327175855636597, "sampling/importance_sampling_ratio/min": 0.9108384251594543, "sampling/sampling_logp_difference/max": 0.17028093338012695, "sampling/sampling_logp_difference/mean": 0.015625737607479095, "step": 883, "step_time": 9.033325273999253 }, { "clip_ratio/high_max": 0.008928571827709675, "clip_ratio/high_mean": 0.004464285913854837, "clip_ratio/low_mean": 0.012828947510570288, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.017293233424425125, "entropy": 0.6013257093727589, "epoch": 0.00884, "grad_norm": 0.21472938358783722, "kl": 0.6555830631405115, "learning_rate": 1.4927430135899739e-05, "loss": 0.031, "step": 884, "step_time": 5.026770877999297 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2450.0, "completions/max_terminated_length": 2450.0, "completions/mean_length": 369.40625, "completions/mean_terminated_length": 369.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6251321202144027, "epoch": 0.00885, "frac_reward_zero_std": 0.25, "grad_norm": 0.20218425989151, "kl": 0.5776144713163376, "learning_rate": 1.4882809447692655e-05, "loss": -0.0389, "num_tokens": 18997145.0, "reward": 0.7191775441169739, "reward_std": 0.5534111261367798, "rewards/rollout_reward_func/mean": 0.7191775441169739, "rewards/rollout_reward_func/std": 0.6778753995895386, "sampling/importance_sampling_ratio/max": 2.064908027648926, "sampling/importance_sampling_ratio/mean": 1.0328989028930664, "sampling/importance_sampling_ratio/min": 0.9367380738258362, "sampling/sampling_logp_difference/max": 0.17393970489501953, "sampling/sampling_logp_difference/mean": 0.012498501688241959, "step": 885, "step_time": 11.607143905000157 }, { "clip_ratio/high_max": 0.015625000465661287, "clip_ratio/high_mean": 0.007812500232830644, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007812500232830644, "entropy": 0.6373481601476669, "epoch": 0.00886, "grad_norm": 0.18595680594444275, "kl": 0.5911002308130264, "learning_rate": 1.4838291499385229e-05, "loss": -0.04, "step": 886, "step_time": 6.722624666000229 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1613.0, "completions/max_terminated_length": 1613.0, "completions/mean_length": 420.125, "completions/mean_terminated_length": 420.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7886236906051636, "epoch": 0.00887, "frac_reward_zero_std": 0.0, "grad_norm": 0.18888171017169952, "kl": 0.8297282010316849, "learning_rate": 1.4793876635071955e-05, "loss": -0.0248, "num_tokens": 19038004.0, "reward": 0.7390589714050293, "reward_std": 0.5195685625076294, "rewards/rollout_reward_func/mean": 0.7390589714050293, "rewards/rollout_reward_func/std": 0.5832902193069458, "sampling/importance_sampling_ratio/max": 1.4934158325195312, "sampling/importance_sampling_ratio/mean": 0.9952533841133118, "sampling/importance_sampling_ratio/min": 0.6267028450965881, "sampling/sampling_logp_difference/max": 0.20305776596069336, "sampling/sampling_logp_difference/mean": 0.023380234837532043, "step": 887, "step_time": 9.377170385000682 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014880952658131719, "entropy": 0.8015890084207058, "epoch": 0.00888, "grad_norm": 0.19203440845012665, "kl": 0.8374641332775354, "learning_rate": 1.4749565198050545e-05, "loss": -0.0254, "step": 888, "step_time": 5.62644756500049 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2574.0, "completions/max_terminated_length": 2574.0, "completions/mean_length": 626.96875, "completions/mean_terminated_length": 626.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9495255053043365, "epoch": 0.00889, "frac_reward_zero_std": 0.0, "grad_norm": 0.3341214656829834, "kl": 0.6389265656471252, "learning_rate": 1.470535753081931e-05, "loss": 0.0278, "num_tokens": 19086815.0, "reward": 0.7392309308052063, "reward_std": 0.4977114200592041, "rewards/rollout_reward_func/mean": 0.7392309308052063, "rewards/rollout_reward_func/std": 0.5213115215301514, "sampling/importance_sampling_ratio/max": 1.0981543064117432, "sampling/importance_sampling_ratio/mean": 0.9612611532211304, "sampling/importance_sampling_ratio/min": 0.18696726858615875, "sampling/sampling_logp_difference/max": 0.4639568328857422, "sampling/sampling_logp_difference/mean": 0.027722379192709923, "step": 889, "step_time": 12.517737879999913 }, { "clip_ratio/high_max": 0.019256757106631994, "clip_ratio/high_mean": 0.0144360710401088, "clip_ratio/low_mean": 0.0016891892300918698, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.016125259920954704, "entropy": 0.9684466570615768, "epoch": 0.0089, "grad_norm": 0.30486035346984863, "kl": 0.6481194980442524, "learning_rate": 1.4661253975074448e-05, "loss": 0.025, "step": 890, "step_time": 6.901370447998488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1423.0, "completions/max_terminated_length": 1423.0, "completions/mean_length": 554.96875, "completions/mean_terminated_length": 554.96875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9151100069284439, "epoch": 0.00891, "frac_reward_zero_std": 0.0, "grad_norm": 0.37380412220954895, "kl": 1.105292171239853, "learning_rate": 1.4617254871707484e-05, "loss": -0.0325, "num_tokens": 19133733.0, "reward": 0.6226399540901184, "reward_std": 0.6693781018257141, "rewards/rollout_reward_func/mean": 0.6226399540901184, "rewards/rollout_reward_func/std": 0.7026206851005554, "sampling/importance_sampling_ratio/max": 1.2240642309188843, "sampling/importance_sampling_ratio/mean": 0.9928103685379028, "sampling/importance_sampling_ratio/min": 0.8187813758850098, "sampling/sampling_logp_difference/max": 0.4274716377258301, "sampling/sampling_logp_difference/mean": 0.024201523512601852, "step": 891, "step_time": 9.814234282001962 }, { "clip_ratio/high_max": 0.04534706426784396, "clip_ratio/high_mean": 0.025464218808338046, "clip_ratio/low_mean": 0.003289473708719015, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02875369251705706, "entropy": 0.9279188401997089, "epoch": 0.00892, "grad_norm": 0.3209657669067383, "kl": 1.1580773368477821, "learning_rate": 1.457336056080258e-05, "loss": -0.0354, "step": 892, "step_time": 4.882079926000188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1704.0, "completions/max_terminated_length": 1704.0, "completions/mean_length": 411.40625, "completions/mean_terminated_length": 411.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8853077106177807, "epoch": 0.00893, "frac_reward_zero_std": 0.25, "grad_norm": 0.22818654775619507, "kl": 1.0768856592476368, "learning_rate": 1.4529571381633913e-05, "loss": 0.0265, "num_tokens": 19174567.0, "reward": 0.8075370788574219, "reward_std": 0.3988041877746582, "rewards/rollout_reward_func/mean": 0.8075370788574219, "rewards/rollout_reward_func/std": 0.47213396430015564, "sampling/importance_sampling_ratio/max": 1.2196884155273438, "sampling/importance_sampling_ratio/mean": 0.9882597923278809, "sampling/importance_sampling_ratio/min": 0.682083249092102, "sampling/sampling_logp_difference/max": 0.3601105213165283, "sampling/sampling_logp_difference/mean": 0.020059166476130486, "step": 893, "step_time": 10.377068737999252 }, { "clip_ratio/high_max": 0.019852941390126944, "clip_ratio/high_mean": 0.009926470695063472, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01439075660891831, "entropy": 0.9017242714762688, "epoch": 0.00894, "grad_norm": 0.20453551411628723, "kl": 1.1236139237880707, "learning_rate": 1.448588767266307e-05, "loss": 0.0257, "step": 894, "step_time": 5.267049112998393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 436.78125, "completions/mean_terminated_length": 427.45159912109375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3398346081376076, "epoch": 0.00895, "frac_reward_zero_std": 0.5, "grad_norm": 0.44678154587745667, "kl": 1.4838778972625732, "learning_rate": 1.444230977153642e-05, "loss": 0.0087, "num_tokens": 19216689.0, "reward": 0.7471040487289429, "reward_std": 0.41959625482559204, "rewards/rollout_reward_func/mean": 0.7471040487289429, "rewards/rollout_reward_func/std": 0.621358335018158, "sampling/importance_sampling_ratio/max": 1.252846598625183, "sampling/importance_sampling_ratio/mean": 0.9788070917129517, "sampling/importance_sampling_ratio/min": 1.4182928377744208e-13, "sampling/sampling_logp_difference/max": 10.59520149230957, "sampling/sampling_logp_difference/mean": 0.12258455157279968, "step": 895, "step_time": 10.97535960100231 }, { "clip_ratio/high_max": 0.006083202664740384, "clip_ratio/high_mean": 0.003041601332370192, "clip_ratio/low_mean": 0.008928571827709675, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011970173160079867, "entropy": 1.3490037843585014, "epoch": 0.00896, "grad_norm": 0.393466591835022, "kl": 1.4868921414017677, "learning_rate": 1.4398838015082503e-05, "loss": 0.004, "step": 896, "step_time": 5.198559564999414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1545.0, "completions/max_terminated_length": 1545.0, "completions/mean_length": 351.46875, "completions/mean_terminated_length": 335.36669921875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3004648610949516, "epoch": 0.00897, "frac_reward_zero_std": 0.25, "grad_norm": 0.5258046984672546, "kl": 1.1572134234011173, "learning_rate": 1.4355472739309444e-05, "loss": -0.0564, "num_tokens": 19255364.0, "reward": 0.7120358943939209, "reward_std": 0.5285786986351013, "rewards/rollout_reward_func/mean": 0.7120358943939209, "rewards/rollout_reward_func/std": 0.6380960941314697, "sampling/importance_sampling_ratio/max": 1.7227544784545898, "sampling/importance_sampling_ratio/mean": 0.9675562381744385, "sampling/importance_sampling_ratio/min": 0.3070182502269745, "sampling/sampling_logp_difference/max": 0.3074531555175781, "sampling/sampling_logp_difference/mean": 0.030075371265411377, "step": 897, "step_time": 10.086332388999836 }, { "clip_ratio/high_max": 0.03385416674427688, "clip_ratio/high_mean": 0.01692708337213844, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01692708337213844, "entropy": 1.303014099597931, "epoch": 0.00898, "grad_norm": 0.3788618743419647, "kl": 1.154137521982193, "learning_rate": 1.431221427940232e-05, "loss": -0.0619, "step": 898, "step_time": 5.073179503001484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1318.0, "completions/max_terminated_length": 1318.0, "completions/mean_length": 318.28125, "completions/mean_terminated_length": 301.3548278808594, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8870170153677464, "epoch": 0.00899, "frac_reward_zero_std": 0.0, "grad_norm": 0.6976280808448792, "kl": 1.0137833431363106, "learning_rate": 1.4269062969720616e-05, "loss": 0.0082, "num_tokens": 19292794.0, "reward": 0.6712517738342285, "reward_std": 0.5415976643562317, "rewards/rollout_reward_func/mean": 0.6712517738342285, "rewards/rollout_reward_func/std": 0.6121626496315002, "sampling/importance_sampling_ratio/max": 1.6587809324264526, "sampling/importance_sampling_ratio/mean": 1.0184481143951416, "sampling/importance_sampling_ratio/min": 0.6546981334686279, "sampling/sampling_logp_difference/max": 0.18648815155029297, "sampling/sampling_logp_difference/mean": 0.024006936699151993, "step": 899, "step_time": 8.758504049000294 }, { "clip_ratio/high_max": 0.02116815559566021, "clip_ratio/high_mean": 0.014654456172138453, "clip_ratio/low_mean": 0.006414473755285144, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.021068929694592953, "entropy": 0.8855181951075792, "epoch": 0.009, "grad_norm": 0.5189555287361145, "kl": 0.99655532091856, "learning_rate": 1.4226019143795585e-05, "loss": 0.0016, "step": 900, "step_time": 5.1973118300029455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1452.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 341.125, "completions/mean_terminated_length": 297.20001220703125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8186965100467205, "epoch": 0.00901, "frac_reward_zero_std": 0.5, "grad_norm": 0.16836076974868774, "kl": 0.9909631758928299, "learning_rate": 1.4183083134327752e-05, "loss": 0.0061, "num_tokens": 19330914.0, "reward": 0.8991353511810303, "reward_std": 0.21864262223243713, "rewards/rollout_reward_func/mean": 0.8991353511810303, "rewards/rollout_reward_func/std": 0.3187394440174103, "sampling/importance_sampling_ratio/max": 1.3480370044708252, "sampling/importance_sampling_ratio/mean": 0.9729267358779907, "sampling/importance_sampling_ratio/min": 0.5110671520233154, "sampling/sampling_logp_difference/max": 0.4383993148803711, "sampling/sampling_logp_difference/mean": 0.03168601915240288, "step": 901, "step_time": 9.57579247600006 }, { "clip_ratio/high_max": 0.03030303120613098, "clip_ratio/high_mean": 0.01515151560306549, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01515151560306549, "entropy": 0.8267133049666882, "epoch": 0.00902, "grad_norm": 0.13023042678833008, "kl": 0.9832303076982498, "learning_rate": 1.4140255273184235e-05, "loss": 0.0049, "step": 902, "step_time": 4.711257022000609 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 1560.0, "completions/max_terminated_length": 1560.0, "completions/mean_length": 631.78125, "completions/mean_terminated_length": 600.4815063476562, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.4803330674767494, "epoch": 0.00903, "frac_reward_zero_std": 0.25, "grad_norm": 0.8166508078575134, "kl": 1.342128500342369, "learning_rate": 1.409753589139627e-05, "loss": 0.1118, "num_tokens": 19381876.0, "reward": 0.6926263570785522, "reward_std": 0.5240026116371155, "rewards/rollout_reward_func/mean": 0.6926263570785522, "rewards/rollout_reward_func/std": 0.6796518564224243, "sampling/importance_sampling_ratio/max": 1.201751470565796, "sampling/importance_sampling_ratio/mean": 0.9595880508422852, "sampling/importance_sampling_ratio/min": 0.5506630539894104, "sampling/sampling_logp_difference/max": 0.26019787788391113, "sampling/sampling_logp_difference/mean": 0.032299090176820755, "step": 903, "step_time": 10.936319309000282 }, { "clip_ratio/high_max": 0.026893940288573503, "clip_ratio/high_mean": 0.013446970144286752, "clip_ratio/low_mean": 0.009679118520580232, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02312608843203634, "entropy": 1.4765670448541641, "epoch": 0.00904, "grad_norm": 0.6883198022842407, "kl": 1.338652603328228, "learning_rate": 1.4054925319156615e-05, "loss": 0.1039, "step": 904, "step_time": 5.121666465000089 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 2169.0, "completions/max_terminated_length": 2169.0, "completions/mean_length": 320.09375, "completions/mean_terminated_length": 271.03448486328125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1347242603078485, "epoch": 0.00905, "frac_reward_zero_std": 0.25, "grad_norm": 0.6689638495445251, "kl": 1.0591287389397621, "learning_rate": 1.4012423885816984e-05, "loss": 0.062, "num_tokens": 19419188.0, "reward": 0.8405416011810303, "reward_std": 0.45101648569107056, "rewards/rollout_reward_func/mean": 0.8405416011810303, "rewards/rollout_reward_func/std": 0.5200932621955872, "sampling/importance_sampling_ratio/max": 1.1879938840866089, "sampling/importance_sampling_ratio/mean": 0.9853551387786865, "sampling/importance_sampling_ratio/min": 2.276049726057781e-37, "sampling/sampling_logp_difference/max": 14.540526390075684, "sampling/sampling_logp_difference/mean": 0.2557131052017212, "step": 905, "step_time": 11.566998657002841 }, { "clip_ratio/high_max": 0.0016891892300918698, "clip_ratio/high_mean": 0.0008445946150459349, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01126126159215346, "entropy": 1.122502263635397, "epoch": 0.00906, "grad_norm": 0.43124711513519287, "kl": 1.0619174018502235, "learning_rate": 1.3970031919885536e-05, "loss": 0.0559, "step": 906, "step_time": 6.207367547001013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1655.0, "completions/max_terminated_length": 1655.0, "completions/mean_length": 424.0625, "completions/mean_terminated_length": 370.9000244140625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.028804282657802, "epoch": 0.00907, "frac_reward_zero_std": 0.0, "grad_norm": 0.6307170391082764, "kl": 1.4359404016286135, "learning_rate": 1.3927749749024323e-05, "loss": 0.0588, "num_tokens": 19460998.0, "reward": 0.7741609811782837, "reward_std": 0.571847140789032, "rewards/rollout_reward_func/mean": 0.7741609811782837, "rewards/rollout_reward_func/std": 0.5610021948814392, "sampling/importance_sampling_ratio/max": 1.187340497970581, "sampling/importance_sampling_ratio/mean": 0.944592297077179, "sampling/importance_sampling_ratio/min": 1.2551363990916274e-17, "sampling/sampling_logp_difference/max": 25.68596839904785, "sampling/sampling_logp_difference/mean": 0.1285315752029419, "step": 907, "step_time": 10.637883688001239 }, { "clip_ratio/high_max": 0.051104329992085695, "clip_ratio/high_mean": 0.026754088001325727, "clip_ratio/low_mean": 0.006578947184607387, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03333303518593311, "entropy": 1.0196869056671858, "epoch": 0.00908, "grad_norm": 0.5160375237464905, "kl": 1.4219324048608541, "learning_rate": 1.3885577700046747e-05, "loss": 0.0531, "step": 908, "step_time": 5.826804016001915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1548.0, "completions/max_terminated_length": 1548.0, "completions/mean_length": 396.40625, "completions/mean_terminated_length": 381.86669921875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.999930776655674, "epoch": 0.00909, "frac_reward_zero_std": 0.0, "grad_norm": 1.0008039474487305, "kl": 1.8505331724882126, "learning_rate": 1.3843516098915028e-05, "loss": 0.0823, "num_tokens": 19501694.0, "reward": 0.7118622660636902, "reward_std": 0.6480968594551086, "rewards/rollout_reward_func/mean": 0.7118622660636902, "rewards/rollout_reward_func/std": 0.6424738764762878, "sampling/importance_sampling_ratio/max": 1.395294427871704, "sampling/importance_sampling_ratio/mean": 0.955879807472229, "sampling/importance_sampling_ratio/min": 1.101191945789004e-12, "sampling/sampling_logp_difference/max": 25.083162307739258, "sampling/sampling_logp_difference/mean": 0.10337551683187485, "step": 909, "step_time": 9.810913141000128 }, { "clip_ratio/high_max": 0.011363636702299118, "clip_ratio/high_mean": 0.005681818351149559, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010890151839703321, "entropy": 0.9846896789968014, "epoch": 0.0091, "grad_norm": 0.8008623123168945, "kl": 1.8132791966199875, "learning_rate": 1.3801565270737715e-05, "loss": 0.0729, "step": 910, "step_time": 5.04313638199892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 1624.0, "completions/max_terminated_length": 1624.0, "completions/mean_length": 432.1875, "completions/mean_terminated_length": 393.13336181640625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.9647465199232101, "epoch": 0.00911, "frac_reward_zero_std": 0.25, "grad_norm": 0.42573827505111694, "kl": 1.2140538096427917, "learning_rate": 1.3759725539767143e-05, "loss": 0.0368, "num_tokens": 19544526.0, "reward": 0.8058000802993774, "reward_std": 0.40434062480926514, "rewards/rollout_reward_func/mean": 0.8058000802993774, "rewards/rollout_reward_func/std": 0.48310473561286926, "sampling/importance_sampling_ratio/max": 1.3775608539581299, "sampling/importance_sampling_ratio/mean": 0.9939539432525635, "sampling/importance_sampling_ratio/min": 0.21709918975830078, "sampling/sampling_logp_difference/max": 0.3659176826477051, "sampling/sampling_logp_difference/mean": 0.02704102359712124, "step": 911, "step_time": 10.264843196999209 }, { "clip_ratio/high_max": 0.008333333767950535, "clip_ratio/high_mean": 0.004166666883975267, "clip_ratio/low_mean": 0.010879629757255316, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015046296641230583, "entropy": 0.951503574848175, "epoch": 0.00912, "grad_norm": 0.35902339220046997, "kl": 1.2154381424188614, "learning_rate": 1.3717997229396939e-05, "loss": 0.0339, "step": 912, "step_time": 5.157214723000834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1515.0, "completions/max_terminated_length": 1515.0, "completions/mean_length": 367.75, "completions/mean_terminated_length": 367.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8225609324872494, "epoch": 0.00913, "frac_reward_zero_std": 0.0, "grad_norm": 0.5569309592247009, "kl": 1.0344653576612473, "learning_rate": 1.367638066215954e-05, "loss": 0.0198, "num_tokens": 19583312.0, "reward": 0.6194392442703247, "reward_std": 0.7150931358337402, "rewards/rollout_reward_func/mean": 0.6194392442703247, "rewards/rollout_reward_func/std": 0.7078050374984741, "sampling/importance_sampling_ratio/max": 1.1648399829864502, "sampling/importance_sampling_ratio/mean": 0.974712610244751, "sampling/importance_sampling_ratio/min": 0.4295209050178528, "sampling/sampling_logp_difference/max": 0.23790431022644043, "sampling/sampling_logp_difference/mean": 0.021725203841924667, "step": 913, "step_time": 10.0680979509998 }, { "clip_ratio/high_max": 0.02000000001862645, "clip_ratio/high_mean": 0.010000000009313226, "clip_ratio/low_mean": 0.004166666883975267, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014166666893288493, "entropy": 0.81624785810709, "epoch": 0.00914, "grad_norm": 0.42635345458984375, "kl": 1.025850087404251, "learning_rate": 1.3634876159723646e-05, "loss": 0.0156, "step": 914, "step_time": 4.9958559220003735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 861.0, "completions/max_terminated_length": 822.0, "completions/mean_length": 356.59375, "completions/mean_terminated_length": 325.66668701171875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8617268316447735, "epoch": 0.00915, "frac_reward_zero_std": 0.0, "grad_norm": 1.2400151491165161, "kl": 1.2119342759251595, "learning_rate": 1.3593484042891815e-05, "loss": 0.1627, "num_tokens": 19622166.0, "reward": 0.7444303035736084, "reward_std": 0.6382617950439453, "rewards/rollout_reward_func/mean": 0.7444303035736084, "rewards/rollout_reward_func/std": 0.629130482673645, "sampling/importance_sampling_ratio/max": 1.567893624305725, "sampling/importance_sampling_ratio/mean": 1.0042717456817627, "sampling/importance_sampling_ratio/min": 0.6546186208724976, "sampling/sampling_logp_difference/max": 0.29393911361694336, "sampling/sampling_logp_difference/mean": 0.023556377738714218, "step": 915, "step_time": 8.094753528002002 }, { "clip_ratio/high_max": 0.0062500000931322575, "clip_ratio/high_mean": 0.0031250000465661287, "clip_ratio/low_mean": 0.01052449969574809, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01364949974231422, "entropy": 0.8548881411552429, "epoch": 0.00916, "grad_norm": 0.9452363848686218, "kl": 1.2029014378786087, "learning_rate": 1.3552204631597894e-05, "loss": 0.1521, "step": 916, "step_time": 3.8094419700009894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1620.0, "completions/max_terminated_length": 1620.0, "completions/mean_length": 487.59375, "completions/mean_terminated_length": 487.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.0018601194024086, "epoch": 0.00917, "frac_reward_zero_std": 0.0, "grad_norm": 0.6296290755271912, "kl": 1.0079364702105522, "learning_rate": 1.3511038244904619e-05, "loss": 0.0213, "num_tokens": 19666928.0, "reward": 0.5622223615646362, "reward_std": 0.8180723190307617, "rewards/rollout_reward_func/mean": 0.5622223615646362, "rewards/rollout_reward_func/std": 0.7964748740196228, "sampling/importance_sampling_ratio/max": 1.7816869020462036, "sampling/importance_sampling_ratio/mean": 1.0129313468933105, "sampling/importance_sampling_ratio/min": 0.5851631164550781, "sampling/sampling_logp_difference/max": 0.2428436279296875, "sampling/sampling_logp_difference/mean": 0.02708526886999607, "step": 917, "step_time": 10.137193173000924 }, { "clip_ratio/high_max": 0.026041666977107525, "clip_ratio/high_mean": 0.013020833488553762, "clip_ratio/low_mean": 0.005681818351149559, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01870265183970332, "entropy": 1.002453126013279, "epoch": 0.00918, "grad_norm": 0.4915623962879181, "kl": 0.9907336011528969, "learning_rate": 1.3469985201001119e-05, "loss": 0.0152, "step": 918, "step_time": 5.213651440998547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2106.0, "completions/max_terminated_length": 2106.0, "completions/mean_length": 417.625, "completions/mean_terminated_length": 417.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5546293221414089, "epoch": 0.00919, "frac_reward_zero_std": 0.75, "grad_norm": 0.10774257779121399, "kl": 1.0035732463002205, "learning_rate": 1.3429045817200436e-05, "loss": 0.0106, "num_tokens": 19708094.0, "reward": 0.9332024455070496, "reward_std": 0.12373406440019608, "rewards/rollout_reward_func/mean": 0.9332024455070496, "rewards/rollout_reward_func/std": 0.26292872428894043, "sampling/importance_sampling_ratio/max": 1.314341425895691, "sampling/importance_sampling_ratio/mean": 1.0075041055679321, "sampling/importance_sampling_ratio/min": 0.8086802363395691, "sampling/sampling_logp_difference/max": 0.17506790161132812, "sampling/sampling_logp_difference/mean": 0.015276599675416946, "step": 919, "step_time": 10.823218554998675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5524364374577999, "epoch": 0.0092, "grad_norm": 0.10323173552751541, "kl": 0.98249451816082, "learning_rate": 1.3388220409937122e-05, "loss": 0.0101, "step": 920, "step_time": 6.680326164999315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 342.8125, "completions/mean_terminated_length": 342.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5804933719336987, "epoch": 0.00921, "frac_reward_zero_std": 0.75, "grad_norm": 0.31219470500946045, "kl": 0.7039822414517403, "learning_rate": 1.3347509294764742e-05, "loss": 0.0216, "num_tokens": 19745516.0, "reward": 0.9664487838745117, "reward_std": 0.09489725530147552, "rewards/rollout_reward_func/mean": 0.9664487838745117, "rewards/rollout_reward_func/std": 0.18979451060295105, "sampling/importance_sampling_ratio/max": 1.3121366500854492, "sampling/importance_sampling_ratio/mean": 1.0182291269302368, "sampling/importance_sampling_ratio/min": 0.8242549896240234, "sampling/sampling_logp_difference/max": 0.14029908180236816, "sampling/sampling_logp_difference/mean": 0.01727868989109993, "step": 921, "step_time": 9.248926790999576 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "entropy": 0.5737351812422276, "epoch": 0.00922, "grad_norm": 0.2419624924659729, "kl": 0.6910670083016157, "learning_rate": 1.3306912786353471e-05, "loss": 0.0196, "step": 922, "step_time": 5.604180439999254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 398.6875, "completions/mean_terminated_length": 398.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5972808748483658, "epoch": 0.00923, "frac_reward_zero_std": 0.5, "grad_norm": 0.15227347612380981, "kl": 0.762324869632721, "learning_rate": 1.3266431198487653e-05, "loss": -0.0085, "num_tokens": 19787019.0, "reward": 0.8423977494239807, "reward_std": 0.3624948561191559, "rewards/rollout_reward_func/mean": 0.8423977494239807, "rewards/rollout_reward_func/std": 0.5152022838592529, "sampling/importance_sampling_ratio/max": 1.1430150270462036, "sampling/importance_sampling_ratio/mean": 0.9989451169967651, "sampling/importance_sampling_ratio/min": 0.8214864134788513, "sampling/sampling_logp_difference/max": 0.1990821361541748, "sampling/sampling_logp_difference/mean": 0.01640549674630165, "step": 923, "step_time": 9.284290067000256 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017361111240461469, "entropy": 0.5889649167656898, "epoch": 0.00924, "grad_norm": 0.13739897310733795, "kl": 0.7451160773634911, "learning_rate": 1.322606484406335e-05, "loss": -0.0088, "step": 924, "step_time": 5.050075746999937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1337.0, "completions/max_terminated_length": 1337.0, "completions/mean_length": 326.78125, "completions/mean_terminated_length": 326.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6010366752743721, "epoch": 0.00925, "frac_reward_zero_std": 0.0, "grad_norm": 0.11046310514211655, "kl": 0.5415563434362411, "learning_rate": 1.3185814035085978e-05, "loss": -0.0208, "num_tokens": 19825980.0, "reward": 0.7431850433349609, "reward_std": 0.6044584512710571, "rewards/rollout_reward_func/mean": 0.7431850433349609, "rewards/rollout_reward_func/std": 0.6305996775627136, "sampling/importance_sampling_ratio/max": 1.1572211980819702, "sampling/importance_sampling_ratio/mean": 1.0051610469818115, "sampling/importance_sampling_ratio/min": 0.7530088424682617, "sampling/sampling_logp_difference/max": 0.23156213760375977, "sampling/sampling_logp_difference/mean": 0.01496659591794014, "step": 925, "step_time": 8.388105859998177 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.5931185185909271, "epoch": 0.00926, "grad_norm": 0.10242599993944168, "kl": 0.5338663794100285, "learning_rate": 1.3145679082667827e-05, "loss": -0.0214, "step": 926, "step_time": 5.091232685996147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1650.0, "completions/max_terminated_length": 1650.0, "completions/mean_length": 377.84375, "completions/mean_terminated_length": 377.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5984072610735893, "epoch": 0.00927, "frac_reward_zero_std": 0.0, "grad_norm": 0.10900222510099411, "kl": 0.5688784476369619, "learning_rate": 1.310566029702572e-05, "loss": 0.0057, "num_tokens": 19865869.0, "reward": 0.6499605178833008, "reward_std": 0.6068187952041626, "rewards/rollout_reward_func/mean": 0.6499605178833008, "rewards/rollout_reward_func/std": 0.6497580409049988, "sampling/importance_sampling_ratio/max": 1.1453371047973633, "sampling/importance_sampling_ratio/mean": 0.9829075932502747, "sampling/importance_sampling_ratio/min": 0.7109342813491821, "sampling/sampling_logp_difference/max": 0.24466824531555176, "sampling/sampling_logp_difference/mean": 0.016273261979222298, "step": 927, "step_time": 9.372850574998665 }, { "clip_ratio/high_max": 0.007352941203862429, "clip_ratio/high_mean": 0.0036764706019312143, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036764706019312143, "entropy": 0.5922271199524403, "epoch": 0.00928, "grad_norm": 0.10470639169216156, "kl": 0.5629836954176426, "learning_rate": 1.3065757987478581e-05, "loss": 0.0052, "step": 928, "step_time": 5.676103736996083 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1538.0, "completions/max_terminated_length": 1538.0, "completions/mean_length": 653.40625, "completions/mean_terminated_length": 653.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7248523533344269, "epoch": 0.00929, "frac_reward_zero_std": 0.25, "grad_norm": 0.10154635459184647, "kl": 0.4743854086846113, "learning_rate": 1.3025972462445029e-05, "loss": -0.006, "num_tokens": 19916542.0, "reward": 0.8384662866592407, "reward_std": 0.3912229537963867, "rewards/rollout_reward_func/mean": 0.8384662866592407, "rewards/rollout_reward_func/std": 0.4579237997531891, "sampling/importance_sampling_ratio/max": 1.3906432390213013, "sampling/importance_sampling_ratio/mean": 0.9928574562072754, "sampling/importance_sampling_ratio/min": 0.4497111141681671, "sampling/sampling_logp_difference/max": 0.37200355529785156, "sampling/sampling_logp_difference/mean": 0.02413804829120636, "step": 929, "step_time": 9.734551034995093 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "entropy": 0.7185246646404266, "epoch": 0.0093, "grad_norm": 0.0969448909163475, "kl": 0.4706295616924763, "learning_rate": 1.2986304029441053e-05, "loss": -0.0062, "step": 930, "step_time": 5.1571548500014615 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 386.1875, "completions/mean_terminated_length": 386.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.642318669706583, "epoch": 0.00931, "frac_reward_zero_std": 0.0, "grad_norm": 0.20911338925361633, "kl": 0.3517751358449459, "learning_rate": 1.2946752995077557e-05, "loss": -0.0372, "num_tokens": 19956576.0, "reward": 0.7111146450042725, "reward_std": 0.6496675610542297, "rewards/rollout_reward_func/mean": 0.7111146450042725, "rewards/rollout_reward_func/std": 0.641621470451355, "sampling/importance_sampling_ratio/max": 1.4463814496994019, "sampling/importance_sampling_ratio/mean": 1.023248314857483, "sampling/importance_sampling_ratio/min": 0.8859100937843323, "sampling/sampling_logp_difference/max": 0.23401641845703125, "sampling/sampling_logp_difference/mean": 0.02139062061905861, "step": 931, "step_time": 8.820501777001482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6389771066606045, "epoch": 0.00932, "grad_norm": 0.21272233128547668, "kl": 0.35406558588147163, "learning_rate": 1.2907319665058062e-05, "loss": -0.0383, "step": 932, "step_time": 4.804709095998987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1641.0, "completions/max_terminated_length": 1641.0, "completions/mean_length": 485.59375, "completions/mean_terminated_length": 485.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6138236075639725, "epoch": 0.00933, "frac_reward_zero_std": 0.25, "grad_norm": 0.12994380295276642, "kl": 0.391917010769248, "learning_rate": 1.28680043441763e-05, "loss": -0.0081, "num_tokens": 20000622.0, "reward": 0.8450874090194702, "reward_std": 0.43815892934799194, "rewards/rollout_reward_func/mean": 0.8450874090194702, "rewards/rollout_reward_func/std": 0.5066381692886353, "sampling/importance_sampling_ratio/max": 1.1135456562042236, "sampling/importance_sampling_ratio/mean": 1.0112038850784302, "sampling/importance_sampling_ratio/min": 0.8910341262817383, "sampling/sampling_logp_difference/max": 0.15945303440093994, "sampling/sampling_logp_difference/mean": 0.013962144032120705, "step": 933, "step_time": 10.31484723199901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6136013567447662, "epoch": 0.00934, "grad_norm": 0.13647614419460297, "kl": 0.3988507464528084, "learning_rate": 1.2828807336313877e-05, "loss": -0.008, "step": 934, "step_time": 5.260238584005492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 840.0, "completions/max_terminated_length": 840.0, "completions/mean_length": 326.375, "completions/mean_terminated_length": 326.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5490365028381348, "epoch": 0.00935, "frac_reward_zero_std": 0.25, "grad_norm": 0.23938950896263123, "kl": 0.2500773295760155, "learning_rate": 1.2789728944437891e-05, "loss": 0.0102, "num_tokens": 20039508.0, "reward": 0.6466432809829712, "reward_std": 0.5098671913146973, "rewards/rollout_reward_func/mean": 0.6466432809829712, "rewards/rollout_reward_func/std": 0.6602655649185181, "sampling/importance_sampling_ratio/max": 1.1896501779556274, "sampling/importance_sampling_ratio/mean": 1.0045974254608154, "sampling/importance_sampling_ratio/min": 0.9160425662994385, "sampling/sampling_logp_difference/max": 0.18305587768554688, "sampling/sampling_logp_difference/mean": 0.012355027720332146, "step": 935, "step_time": 7.549860265997268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.548802200704813, "epoch": 0.00936, "grad_norm": 0.2182071954011917, "kl": 0.25363109819591045, "learning_rate": 1.2750769470598654e-05, "loss": 0.009, "step": 936, "step_time": 3.740384904998791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2369.0, "completions/max_terminated_length": 2369.0, "completions/mean_length": 514.6875, "completions/mean_terminated_length": 514.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6325517385266721, "epoch": 0.00937, "frac_reward_zero_std": 0.0, "grad_norm": 0.24595637619495392, "kl": 0.5316681992262602, "learning_rate": 1.2711929215927296e-05, "loss": 0.0137, "num_tokens": 20084625.0, "reward": 0.7469850182533264, "reward_std": 0.5302813053131104, "rewards/rollout_reward_func/mean": 0.7469850182533264, "rewards/rollout_reward_func/std": 0.5661178827285767, "sampling/importance_sampling_ratio/max": 1.1973047256469727, "sampling/importance_sampling_ratio/mean": 1.018237829208374, "sampling/importance_sampling_ratio/min": 0.880012035369873, "sampling/sampling_logp_difference/max": 0.15623998641967773, "sampling/sampling_logp_difference/mean": 0.017047423869371414, "step": 937, "step_time": 11.492145243002597 }, { "clip_ratio/high_max": 0.017769608180969954, "clip_ratio/high_mean": 0.008884804090484977, "clip_ratio/low_mean": 0.005681818351149559, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014566622441634536, "entropy": 0.6392473354935646, "epoch": 0.00938, "grad_norm": 0.19884808361530304, "kl": 0.5318039264529943, "learning_rate": 1.2673208480633451e-05, "loss": 0.0121, "step": 938, "step_time": 6.5441955840015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 433.0, "completions/mean_terminated_length": 433.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5098544657230377, "epoch": 0.00939, "frac_reward_zero_std": 0.0, "grad_norm": 0.07696344703435898, "kl": 0.30798182636499405, "learning_rate": 1.263460756400299e-05, "loss": -0.0109, "num_tokens": 20126239.0, "reward": 0.7470180988311768, "reward_std": 0.630828320980072, "rewards/rollout_reward_func/mean": 0.7470180988311768, "rewards/rollout_reward_func/std": 0.6213452816009521, "sampling/importance_sampling_ratio/max": 1.1549711227416992, "sampling/importance_sampling_ratio/mean": 1.0034105777740479, "sampling/importance_sampling_ratio/min": 0.8754769563674927, "sampling/sampling_logp_difference/max": 0.13326096534729004, "sampling/sampling_logp_difference/mean": 0.011408618651330471, "step": 939, "step_time": 9.142194223002662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5160748325288296, "epoch": 0.0094, "grad_norm": 0.07724633067846298, "kl": 0.3096331963315606, "learning_rate": 1.2596126764395617e-05, "loss": -0.0109, "step": 940, "step_time": 5.654097067001203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1506.0, "completions/max_terminated_length": 1506.0, "completions/mean_length": 388.75, "completions/mean_terminated_length": 388.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6485850363969803, "epoch": 0.00941, "frac_reward_zero_std": 0.25, "grad_norm": 0.30072101950645447, "kl": 0.5822693891823292, "learning_rate": 1.2557766379242641e-05, "loss": 0.0246, "num_tokens": 20166365.0, "reward": 0.7762606143951416, "reward_std": 0.46023768186569214, "rewards/rollout_reward_func/mean": 0.7762606143951416, "rewards/rollout_reward_func/std": 0.5544412136077881, "sampling/importance_sampling_ratio/max": 1.142449140548706, "sampling/importance_sampling_ratio/mean": 0.9816929697990417, "sampling/importance_sampling_ratio/min": 0.7961292862892151, "sampling/sampling_logp_difference/max": 0.2052241563796997, "sampling/sampling_logp_difference/mean": 0.01846553385257721, "step": 941, "step_time": 9.048621275000187 }, { "clip_ratio/high_max": 0.009615384973585606, "clip_ratio/high_mean": 0.004807692486792803, "clip_ratio/low_mean": 0.0034722222480922937, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008279914734885097, "entropy": 0.6522770375013351, "epoch": 0.00942, "grad_norm": 0.25966429710388184, "kl": 0.5848300084471703, "learning_rate": 1.2519526705044622e-05, "loss": 0.0233, "step": 942, "step_time": 5.464671143001397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1479.0, "completions/max_terminated_length": 1479.0, "completions/mean_length": 432.84375, "completions/mean_terminated_length": 432.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5989007242023945, "epoch": 0.00943, "frac_reward_zero_std": 0.25, "grad_norm": 0.14613907039165497, "kl": 0.4244496012106538, "learning_rate": 1.248140803736913e-05, "loss": -0.0117, "num_tokens": 20208456.0, "reward": 0.8028090000152588, "reward_std": 0.40934187173843384, "rewards/rollout_reward_func/mean": 0.8028090000152588, "rewards/rollout_reward_func/std": 0.48831579089164734, "sampling/importance_sampling_ratio/max": 1.2358894348144531, "sampling/importance_sampling_ratio/mean": 1.0095431804656982, "sampling/importance_sampling_ratio/min": 0.903816282749176, "sampling/sampling_logp_difference/max": 0.12485885620117188, "sampling/sampling_logp_difference/mean": 0.014524579048156738, "step": 943, "step_time": 9.079464411997833 }, { "clip_ratio/high_max": 0.016559829469770193, "clip_ratio/high_mean": 0.008279914734885097, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008279914734885097, "entropy": 0.6083627417683601, "epoch": 0.00944, "grad_norm": 0.12870843708515167, "kl": 0.4292076453566551, "learning_rate": 1.244341067084842e-05, "loss": -0.0121, "step": 944, "step_time": 4.964025936000326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1690.0, "completions/max_terminated_length": 1690.0, "completions/mean_length": 430.53125, "completions/mean_terminated_length": 430.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7338519808836281, "epoch": 0.00945, "frac_reward_zero_std": 0.25, "grad_norm": 0.46470800042152405, "kl": 0.8172099068760872, "learning_rate": 1.2405534899177164e-05, "loss": 0.0624, "num_tokens": 20249414.0, "reward": 0.7736040353775024, "reward_std": 0.4282618761062622, "rewards/rollout_reward_func/mean": 0.7736040353775024, "rewards/rollout_reward_func/std": 0.5012399554252625, "sampling/importance_sampling_ratio/max": 1.165823221206665, "sampling/importance_sampling_ratio/mean": 0.989198625087738, "sampling/importance_sampling_ratio/min": 0.543742299079895, "sampling/sampling_logp_difference/max": 0.36765193939208984, "sampling/sampling_logp_difference/mean": 0.027260273694992065, "step": 945, "step_time": 9.842427260000477 }, { "clip_ratio/high_max": 0.02500000037252903, "clip_ratio/high_mean": 0.012500000186264515, "clip_ratio/low_mean": 0.0011160714784637094, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013616071664728224, "entropy": 0.7394917262718081, "epoch": 0.00946, "grad_norm": 0.41595184803009033, "kl": 0.8153543472290039, "learning_rate": 1.2367781015110199e-05, "loss": 0.0595, "step": 946, "step_time": 5.370025382999302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1587.0, "completions/max_terminated_length": 1587.0, "completions/mean_length": 359.15625, "completions/mean_terminated_length": 359.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5761920846998692, "epoch": 0.00947, "frac_reward_zero_std": 0.0, "grad_norm": 0.38128578662872314, "kl": 0.6216224804520607, "learning_rate": 1.2330149310460231e-05, "loss": 0.0334, "num_tokens": 20288215.0, "reward": 0.3990500867366791, "reward_std": 0.8377212882041931, "rewards/rollout_reward_func/mean": 0.3990500867366791, "rewards/rollout_reward_func/std": 0.8353340029716492, "sampling/importance_sampling_ratio/max": 1.166190266609192, "sampling/importance_sampling_ratio/mean": 1.0230776071548462, "sampling/importance_sampling_ratio/min": 0.8825439810752869, "sampling/sampling_logp_difference/max": 0.1249847412109375, "sampling/sampling_logp_difference/mean": 0.015551837161183357, "step": 947, "step_time": 9.92367477699554 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.007499999832361937, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007499999832361937, "entropy": 0.5789626305922866, "epoch": 0.00948, "grad_norm": 0.24403701722621918, "kl": 0.626840140670538, "learning_rate": 1.2292640076095618e-05, "loss": 0.0304, "step": 948, "step_time": 5.125175586999831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1486.0, "completions/max_terminated_length": 1486.0, "completions/mean_length": 416.625, "completions/mean_terminated_length": 405.4838562011719, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7711132764816284, "epoch": 0.00949, "frac_reward_zero_std": 0.0, "grad_norm": 0.42357757687568665, "kl": 1.1930756140500307, "learning_rate": 1.2255253601938112e-05, "loss": 0.0143, "num_tokens": 20330150.0, "reward": 0.6821012496948242, "reward_std": 0.6674561500549316, "rewards/rollout_reward_func/mean": 0.6821012496948242, "rewards/rollout_reward_func/std": 0.6427432894706726, "sampling/importance_sampling_ratio/max": 1.3136794567108154, "sampling/importance_sampling_ratio/mean": 0.9868835210800171, "sampling/importance_sampling_ratio/min": 0.6073076128959656, "sampling/sampling_logp_difference/max": 0.2304825782775879, "sampling/sampling_logp_difference/mean": 0.02453944832086563, "step": 949, "step_time": 9.742518303999532 }, { "clip_ratio/high_max": 0.013523391913622618, "clip_ratio/high_mean": 0.006761695956811309, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006761695956811309, "entropy": 0.772755466401577, "epoch": 0.0095, "grad_norm": 0.39618587493896484, "kl": 1.160571150481701, "learning_rate": 1.2217990176960571e-05, "loss": 0.0112, "step": 950, "step_time": 4.9396544019964495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 471.15625, "completions/mean_terminated_length": 460.9031982421875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7098592482507229, "epoch": 0.00951, "frac_reward_zero_std": 0.0, "grad_norm": 0.7081959843635559, "kl": 0.5925844497978687, "learning_rate": 1.2180850089184804e-05, "loss": 0.0734, "num_tokens": 20373652.0, "reward": 0.6803485155105591, "reward_std": 0.5738992094993591, "rewards/rollout_reward_func/mean": 0.6803485155105591, "rewards/rollout_reward_func/std": 0.6463130712509155, "sampling/importance_sampling_ratio/max": 1.4373992681503296, "sampling/importance_sampling_ratio/mean": 1.0205450057983398, "sampling/importance_sampling_ratio/min": 0.8960070610046387, "sampling/sampling_logp_difference/max": 0.19791150093078613, "sampling/sampling_logp_difference/mean": 0.021115044131875038, "step": 951, "step_time": 8.981161191006322 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.01260567654389888, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01260567654389888, "entropy": 0.7059501446783543, "epoch": 0.00952, "grad_norm": 0.5503758192062378, "kl": 0.5924301408231258, "learning_rate": 1.2143833625679275e-05, "loss": 0.0681, "step": 952, "step_time": 4.826683572999173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1520.0, "completions/max_terminated_length": 1520.0, "completions/mean_length": 453.28125, "completions/mean_terminated_length": 453.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7237689048051834, "epoch": 0.00953, "frac_reward_zero_std": 0.0, "grad_norm": 0.8760866522789001, "kl": 0.7339475378394127, "learning_rate": 1.2106941072556937e-05, "loss": -0.0169, "num_tokens": 20417253.0, "reward": 0.6248522996902466, "reward_std": 0.7641909122467041, "rewards/rollout_reward_func/mean": 0.6248522996902466, "rewards/rollout_reward_func/std": 0.7433812022209167, "sampling/importance_sampling_ratio/max": 1.3532229661941528, "sampling/importance_sampling_ratio/mean": 1.007156252861023, "sampling/importance_sampling_ratio/min": 0.6579049825668335, "sampling/sampling_logp_difference/max": 0.34986352920532227, "sampling/sampling_logp_difference/mean": 0.024877287447452545, "step": 953, "step_time": 9.320470163002028 }, { "clip_ratio/high_max": 0.016194332391023636, "clip_ratio/high_mean": 0.008097166195511818, "clip_ratio/low_mean": 0.014204545877873898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.022301712073385715, "entropy": 0.7149962894618511, "epoch": 0.00954, "grad_norm": 0.2539048194885254, "kl": 0.7324937246739864, "learning_rate": 1.2070172714972977e-05, "loss": -0.0203, "step": 954, "step_time": 5.608977730998959 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 572.6875, "completions/mean_terminated_length": 572.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7098382115364075, "epoch": 0.00955, "frac_reward_zero_std": 0.25, "grad_norm": 0.21430717408657074, "kl": 0.8878435492515564, "learning_rate": 1.2033528837122646e-05, "loss": -0.0145, "num_tokens": 20466334.0, "reward": 0.8062988519668579, "reward_std": 0.4642435908317566, "rewards/rollout_reward_func/mean": 0.8062988519668579, "rewards/rollout_reward_func/std": 0.5452664494514465, "sampling/importance_sampling_ratio/max": 1.2025996446609497, "sampling/importance_sampling_ratio/mean": 1.0087074041366577, "sampling/importance_sampling_ratio/min": 0.8446065187454224, "sampling/sampling_logp_difference/max": 0.16995906829833984, "sampling/sampling_logp_difference/mean": 0.014604023657739162, "step": 955, "step_time": 10.138533475997974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "entropy": 0.7058281898498535, "epoch": 0.00956, "grad_norm": 0.20371052622795105, "kl": 0.8876973763108253, "learning_rate": 1.1997009722239048e-05, "loss": -0.0148, "step": 956, "step_time": 5.093139287997474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2128.0, "completions/max_terminated_length": 2128.0, "completions/mean_length": 571.3125, "completions/mean_terminated_length": 571.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7546515762805939, "epoch": 0.00957, "frac_reward_zero_std": 0.5, "grad_norm": 0.3961486518383026, "kl": 0.6762099452316761, "learning_rate": 1.1960615652590938e-05, "loss": 0.0276, "num_tokens": 20513393.0, "reward": 0.8107627630233765, "reward_std": 0.36436697840690613, "rewards/rollout_reward_func/mean": 0.8107627630233765, "rewards/rollout_reward_func/std": 0.5343469381332397, "sampling/importance_sampling_ratio/max": 1.6090381145477295, "sampling/importance_sampling_ratio/mean": 1.0067158937454224, "sampling/importance_sampling_ratio/min": 0.683687150478363, "sampling/sampling_logp_difference/max": 0.202315092086792, "sampling/sampling_logp_difference/mean": 0.020762551575899124, "step": 957, "step_time": 11.488979167001162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0014880952658131719, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014880952658131719, "entropy": 0.7523226998746395, "epoch": 0.00958, "grad_norm": 0.37463849782943726, "kl": 0.6754303500056267, "learning_rate": 1.1924346909480573e-05, "loss": 0.0254, "step": 958, "step_time": 6.189443924002262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 465.84375, "completions/mean_terminated_length": 465.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6263924613595009, "epoch": 0.00959, "frac_reward_zero_std": 0.0, "grad_norm": 0.2259327471256256, "kl": 0.4208654426038265, "learning_rate": 1.1888203773241489e-05, "loss": 0.0066, "num_tokens": 20556787.0, "reward": 0.6297733187675476, "reward_std": 0.7241827249526978, "rewards/rollout_reward_func/mean": 0.6297733187675476, "rewards/rollout_reward_func/std": 0.7388657927513123, "sampling/importance_sampling_ratio/max": 1.2083022594451904, "sampling/importance_sampling_ratio/mean": 0.9909488558769226, "sampling/importance_sampling_ratio/min": 0.7208747267723083, "sampling/sampling_logp_difference/max": 0.21308410167694092, "sampling/sampling_logp_difference/mean": 0.019373219460248947, "step": 959, "step_time": 9.071953953001866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6188591308891773, "epoch": 0.0096, "grad_norm": 0.22413580119609833, "kl": 0.417814202606678, "learning_rate": 1.1852186523236407e-05, "loss": 0.006, "step": 960, "step_time": 4.935072893998949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2450.0, "completions/max_terminated_length": 2450.0, "completions/mean_length": 534.84375, "completions/mean_terminated_length": 534.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.593387870118022, "epoch": 0.00961, "frac_reward_zero_std": 0.0, "grad_norm": 0.35440564155578613, "kl": 0.540638230741024, "learning_rate": 1.1816295437854995e-05, "loss": 0.0106, "num_tokens": 20601654.0, "reward": 0.7737923264503479, "reward_std": 0.5117642879486084, "rewards/rollout_reward_func/mean": 0.7737923264503479, "rewards/rollout_reward_func/std": 0.4993864893913269, "sampling/importance_sampling_ratio/max": 1.1664905548095703, "sampling/importance_sampling_ratio/mean": 1.0082738399505615, "sampling/importance_sampling_ratio/min": 0.7024635672569275, "sampling/sampling_logp_difference/max": 0.2001490592956543, "sampling/sampling_logp_difference/mean": 0.016536466777324677, "step": 961, "step_time": 12.102719810000053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 0.5851026345044374, "epoch": 0.00962, "grad_norm": 0.26489266753196716, "kl": 0.5337911248207092, "learning_rate": 1.1780530794511753e-05, "loss": 0.0093, "step": 962, "step_time": 7.231515153000146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1623.0, "completions/max_terminated_length": 1623.0, "completions/mean_length": 598.625, "completions/mean_terminated_length": 598.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5565116852521896, "epoch": 0.00963, "frac_reward_zero_std": 0.0, "grad_norm": 0.19876131415367126, "kl": 0.5533840078860521, "learning_rate": 1.1744892869643888e-05, "loss": -0.0065, "num_tokens": 20648794.0, "reward": 0.7723438143730164, "reward_std": 0.49440255761146545, "rewards/rollout_reward_func/mean": 0.7723438143730164, "rewards/rollout_reward_func/std": 0.5037213563919067, "sampling/importance_sampling_ratio/max": 1.2647037506103516, "sampling/importance_sampling_ratio/mean": 0.9964163303375244, "sampling/importance_sampling_ratio/min": 0.5947914123535156, "sampling/sampling_logp_difference/max": 0.24295473098754883, "sampling/sampling_logp_difference/mean": 0.017594123259186745, "step": 963, "step_time": 9.827856417001385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5552741251885891, "epoch": 0.00964, "grad_norm": 0.19680649042129517, "kl": 0.5544369351118803, "learning_rate": 1.1709381938709129e-05, "loss": -0.0065, "step": 964, "step_time": 5.34796028600249 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1623.0, "completions/max_terminated_length": 1623.0, "completions/mean_length": 410.25, "completions/mean_terminated_length": 410.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5155835710465908, "epoch": 0.00965, "frac_reward_zero_std": 0.0, "grad_norm": 0.1717834621667862, "kl": 0.5527202971279621, "learning_rate": 1.1673998276183663e-05, "loss": -0.0003, "num_tokens": 20690026.0, "reward": 0.6892563104629517, "reward_std": 0.675182580947876, "rewards/rollout_reward_func/mean": 0.6892563104629517, "rewards/rollout_reward_func/std": 0.6833781003952026, "sampling/importance_sampling_ratio/max": 1.157230019569397, "sampling/importance_sampling_ratio/mean": 0.9856898784637451, "sampling/importance_sampling_ratio/min": 0.6644821166992188, "sampling/sampling_logp_difference/max": 0.16551876068115234, "sampling/sampling_logp_difference/mean": 0.016309145838022232, "step": 965, "step_time": 9.424971371998254 }, { "clip_ratio/high_max": 0.026871155947446823, "clip_ratio/high_mean": 0.013435577973723412, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02906057797372341, "entropy": 0.5204752199351788, "epoch": 0.00966, "grad_norm": 0.15783967077732086, "kl": 0.5925788916647434, "learning_rate": 1.1638742155559933e-05, "loss": -0.0018, "step": 966, "step_time": 5.212074605002272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2416.0, "completions/max_terminated_length": 2416.0, "completions/mean_length": 419.875, "completions/mean_terminated_length": 416.32257080078125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6727216155268252, "epoch": 0.00967, "frac_reward_zero_std": 0.25, "grad_norm": 0.15307800471782684, "kl": 0.6441838853061199, "learning_rate": 1.16036138493446e-05, "loss": -0.0123, "num_tokens": 20732166.0, "reward": 0.7114918231964111, "reward_std": 0.5304636359214783, "rewards/rollout_reward_func/mean": 0.7114918231964111, "rewards/rollout_reward_func/std": 0.6399064064025879, "sampling/importance_sampling_ratio/max": 1.1550410985946655, "sampling/importance_sampling_ratio/mean": 0.9961885213851929, "sampling/importance_sampling_ratio/min": 0.6892120838165283, "sampling/sampling_logp_difference/max": 0.1644296646118164, "sampling/sampling_logp_difference/mean": 0.017685312777757645, "step": 967, "step_time": 12.055042191001121 }, { "clip_ratio/high_max": 0.0193452388048172, "clip_ratio/high_mean": 0.0096726194024086, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0096726194024086, "entropy": 0.6799985682591796, "epoch": 0.00968, "grad_norm": 0.15479567646980286, "kl": 0.6480487585067749, "learning_rate": 1.1568613629056383e-05, "loss": -0.0128, "step": 968, "step_time": 6.47863611700086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1645.0, "completions/max_terminated_length": 1645.0, "completions/mean_length": 608.0, "completions/mean_terminated_length": 608.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.613163024187088, "epoch": 0.00969, "frac_reward_zero_std": 0.25, "grad_norm": 0.21213789284229279, "kl": 0.7066199667751789, "learning_rate": 1.1533741765223996e-05, "loss": -0.0054, "num_tokens": 20780431.0, "reward": 0.5929798483848572, "reward_std": 0.5909035801887512, "rewards/rollout_reward_func/mean": 0.5929798483848572, "rewards/rollout_reward_func/std": 0.7055425047874451, "sampling/importance_sampling_ratio/max": 1.2399989366531372, "sampling/importance_sampling_ratio/mean": 1.0178338289260864, "sampling/importance_sampling_ratio/min": 0.8546574711799622, "sampling/sampling_logp_difference/max": 0.16325902938842773, "sampling/sampling_logp_difference/mean": 0.01667625829577446, "step": 969, "step_time": 10.462587306998103 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.6220342442393303, "epoch": 0.0097, "grad_norm": 0.20054271817207336, "kl": 0.7096292488276958, "learning_rate": 1.1498998527384023e-05, "loss": -0.0057, "step": 970, "step_time": 5.3421229389987275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1402.0, "completions/max_terminated_length": 1402.0, "completions/mean_length": 553.3125, "completions/mean_terminated_length": 553.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6715928092598915, "epoch": 0.00971, "frac_reward_zero_std": 0.25, "grad_norm": 0.2740989029407501, "kl": 0.4697668179869652, "learning_rate": 1.1464384184078873e-05, "loss": -0.0123, "num_tokens": 20827856.0, "reward": 0.753308892250061, "reward_std": 0.5776990652084351, "rewards/rollout_reward_func/mean": 0.753308892250061, "rewards/rollout_reward_func/std": 0.6631731986999512, "sampling/importance_sampling_ratio/max": 1.1281051635742188, "sampling/importance_sampling_ratio/mean": 0.9998327493667603, "sampling/importance_sampling_ratio/min": 0.9089019894599915, "sampling/sampling_logp_difference/max": 0.11581170558929443, "sampling/sampling_logp_difference/mean": 0.016350723803043365, "step": 971, "step_time": 9.029696061998038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6723648011684418, "epoch": 0.00972, "grad_norm": 0.2619990110397339, "kl": 0.4704776518046856, "learning_rate": 1.1429899002854685e-05, "loss": -0.0128, "step": 972, "step_time": 4.876181745001304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1713.0, "completions/max_terminated_length": 1713.0, "completions/mean_length": 522.3125, "completions/mean_terminated_length": 522.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7473876401782036, "epoch": 0.00973, "frac_reward_zero_std": 0.0, "grad_norm": 0.31801778078079224, "kl": 0.8789380937814713, "learning_rate": 1.1395543250259237e-05, "loss": -0.011, "num_tokens": 20873914.0, "reward": 0.602938711643219, "reward_std": 0.754192590713501, "rewards/rollout_reward_func/mean": 0.602938711643219, "rewards/rollout_reward_func/std": 0.7773758172988892, "sampling/importance_sampling_ratio/max": 1.237606167793274, "sampling/importance_sampling_ratio/mean": 0.9965600967407227, "sampling/importance_sampling_ratio/min": 0.7737995386123657, "sampling/sampling_logp_difference/max": 0.29820966720581055, "sampling/sampling_logp_difference/mean": 0.024679582566022873, "step": 973, "step_time": 10.068741014001716 }, { "clip_ratio/high_max": 0.013523391913622618, "clip_ratio/high_mean": 0.006761695956811309, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013011696049943566, "entropy": 0.7449947595596313, "epoch": 0.00974, "grad_norm": 0.22487394511699677, "kl": 0.8746978864073753, "learning_rate": 1.136131719183994e-05, "loss": -0.0124, "step": 974, "step_time": 6.0423812539957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2020.0, "completions/max_terminated_length": 2020.0, "completions/mean_length": 546.0, "completions/mean_terminated_length": 546.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5890642777085304, "epoch": 0.00975, "frac_reward_zero_std": 0.25, "grad_norm": 0.15011189877986908, "kl": 0.7579064406454563, "learning_rate": 1.132722109214172e-05, "loss": -0.0019, "num_tokens": 20921236.0, "reward": 0.7263515591621399, "reward_std": 0.5762779712677002, "rewards/rollout_reward_func/mean": 0.7263515591621399, "rewards/rollout_reward_func/std": 0.6611599326133728, "sampling/importance_sampling_ratio/max": 1.1249052286148071, "sampling/importance_sampling_ratio/mean": 0.9857171773910522, "sampling/importance_sampling_ratio/min": 0.7748233675956726, "sampling/sampling_logp_difference/max": 0.15702486038208008, "sampling/sampling_logp_difference/mean": 0.019140135496854782, "step": 975, "step_time": 10.73327176499879 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.590458657592535, "epoch": 0.00976, "grad_norm": 0.1495809257030487, "kl": 0.7798984423279762, "learning_rate": 1.1293255214705037e-05, "loss": -0.0025, "step": 976, "step_time": 6.372265113002868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1573.0, "completions/max_terminated_length": 1573.0, "completions/mean_length": 476.375, "completions/mean_terminated_length": 476.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7987535819411278, "epoch": 0.00977, "frac_reward_zero_std": 0.25, "grad_norm": 0.2622486650943756, "kl": 0.7429801672697067, "learning_rate": 1.1259419822063808e-05, "loss": -0.0128, "num_tokens": 20965290.0, "reward": 0.7870660424232483, "reward_std": 0.4845156669616699, "rewards/rollout_reward_func/mean": 0.7870660424232483, "rewards/rollout_reward_func/std": 0.5880776047706604, "sampling/importance_sampling_ratio/max": 1.4819337129592896, "sampling/importance_sampling_ratio/mean": 1.0097072124481201, "sampling/importance_sampling_ratio/min": 0.7578192949295044, "sampling/sampling_logp_difference/max": 0.38143491744995117, "sampling/sampling_logp_difference/mean": 0.025681886821985245, "step": 977, "step_time": 9.789797789000659 }, { "clip_ratio/high_max": 0.022433036006987095, "clip_ratio/high_mean": 0.011216518003493547, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011216518003493547, "entropy": 0.801960326731205, "epoch": 0.00978, "grad_norm": 0.2203306257724762, "kl": 0.7497713677585125, "learning_rate": 1.1225715175743381e-05, "loss": -0.0133, "step": 978, "step_time": 5.201020932003303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2317.0, "completions/max_terminated_length": 2317.0, "completions/mean_length": 552.4375, "completions/mean_terminated_length": 552.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5939412713050842, "epoch": 0.00979, "frac_reward_zero_std": 0.25, "grad_norm": 0.2831692397594452, "kl": 0.5684366784989834, "learning_rate": 1.1192141536258534e-05, "loss": 0.0087, "num_tokens": 21011627.0, "reward": 0.7111430168151855, "reward_std": 0.5556213855743408, "rewards/rollout_reward_func/mean": 0.7111430168151855, "rewards/rollout_reward_func/std": 0.6408747434616089, "sampling/importance_sampling_ratio/max": 1.1085776090621948, "sampling/importance_sampling_ratio/mean": 1.0170464515686035, "sampling/importance_sampling_ratio/min": 0.9138630628585815, "sampling/sampling_logp_difference/max": 0.09495902061462402, "sampling/sampling_logp_difference/mean": 0.014575033448636532, "step": 979, "step_time": 11.854134782999608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0034722222480922937, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034722222480922937, "entropy": 0.5911850742995739, "epoch": 0.0098, "grad_norm": 0.23228906095027924, "kl": 0.5682556927204132, "learning_rate": 1.1158699163111437e-05, "loss": 0.0079, "step": 980, "step_time": 6.621495973000492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 395.21875, "completions/mean_terminated_length": 395.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5883529596030712, "epoch": 0.00981, "frac_reward_zero_std": 0.0, "grad_norm": 0.1849799007177353, "kl": 0.9245569556951523, "learning_rate": 1.1125388314789658e-05, "loss": -0.0214, "num_tokens": 21053099.0, "reward": 0.7024519443511963, "reward_std": 0.6064046025276184, "rewards/rollout_reward_func/mean": 0.7024519443511963, "rewards/rollout_reward_func/std": 0.6022577881813049, "sampling/importance_sampling_ratio/max": 1.1349246501922607, "sampling/importance_sampling_ratio/mean": 0.9944777488708496, "sampling/importance_sampling_ratio/min": 0.770402193069458, "sampling/sampling_logp_difference/max": 0.22839701175689697, "sampling/sampling_logp_difference/mean": 0.016515091061592102, "step": 981, "step_time": 9.677836957002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5877589099109173, "epoch": 0.00982, "grad_norm": 0.1703585833311081, "kl": 0.9270560052245855, "learning_rate": 1.1092209248764181e-05, "loss": -0.0219, "step": 982, "step_time": 5.4282114150028065 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 1535.0, "completions/max_terminated_length": 1535.0, "completions/mean_length": 454.96875, "completions/mean_terminated_length": 434.5517272949219, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7266339659690857, "epoch": 0.00983, "frac_reward_zero_std": 0.0, "grad_norm": 2.984471082687378, "kl": 1.2757287882268429, "learning_rate": 1.1059162221487378e-05, "loss": 0.0419, "num_tokens": 21096620.0, "reward": 0.8044461011886597, "reward_std": 0.48882439732551575, "rewards/rollout_reward_func/mean": 0.8044461011886597, "rewards/rollout_reward_func/std": 0.48301416635513306, "sampling/importance_sampling_ratio/max": 1.586457371711731, "sampling/importance_sampling_ratio/mean": 1.0196688175201416, "sampling/importance_sampling_ratio/min": 0.7141357064247131, "sampling/sampling_logp_difference/max": 0.3996460437774658, "sampling/sampling_logp_difference/mean": 0.029103947803378105, "step": 983, "step_time": 9.803404108002724 }, { "clip_ratio/high_max": 0.012561274692416191, "clip_ratio/high_mean": 0.0062806373462080956, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062806373462080956, "entropy": 0.7275388613343239, "epoch": 0.00984, "grad_norm": 1.0316613912582397, "kl": 0.818458154797554, "learning_rate": 1.1026247488391059e-05, "loss": 0.0279, "step": 984, "step_time": 5.125906021001356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 413.84375, "completions/mean_terminated_length": 413.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.71357337012887, "epoch": 0.00985, "frac_reward_zero_std": 0.25, "grad_norm": 0.31603100895881653, "kl": 0.6194660812616348, "learning_rate": 1.0993465303884481e-05, "loss": 0.015, "num_tokens": 21138415.0, "reward": 0.6938085556030273, "reward_std": 0.6267021894454956, "rewards/rollout_reward_func/mean": 0.6938085556030273, "rewards/rollout_reward_func/std": 0.7230144143104553, "sampling/importance_sampling_ratio/max": 1.31226646900177, "sampling/importance_sampling_ratio/mean": 1.025134563446045, "sampling/importance_sampling_ratio/min": 0.8313618302345276, "sampling/sampling_logp_difference/max": 0.1704096794128418, "sampling/sampling_logp_difference/mean": 0.018993636593222618, "step": 985, "step_time": 9.124928966002699 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004166666883975267, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004166666883975267, "entropy": 0.7091538719832897, "epoch": 0.00986, "grad_norm": 0.2505190074443817, "kl": 0.6159311719238758, "learning_rate": 1.0960815921352395e-05, "loss": 0.0138, "step": 986, "step_time": 4.959962249000455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 463.21875, "completions/mean_terminated_length": 463.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.8046802468597889, "epoch": 0.00987, "frac_reward_zero_std": 0.25, "grad_norm": 0.20287886261940002, "kl": 0.6137843430042267, "learning_rate": 1.0928299593153084e-05, "loss": 0.0, "num_tokens": 21181892.0, "reward": 0.7092876434326172, "reward_std": 0.5018919706344604, "rewards/rollout_reward_func/mean": 0.7092876434326172, "rewards/rollout_reward_func/std": 0.5885053873062134, "sampling/importance_sampling_ratio/max": 1.3397150039672852, "sampling/importance_sampling_ratio/mean": 1.0113149881362915, "sampling/importance_sampling_ratio/min": 0.7561540007591248, "sampling/sampling_logp_difference/max": 0.19648754596710205, "sampling/sampling_logp_difference/mean": 0.024435672909021378, "step": 987, "step_time": 10.221130845999141 }, { "clip_ratio/high_max": 0.0078125, "clip_ratio/high_mean": 0.00390625, "clip_ratio/low_mean": 0.004807692486792803, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008713942486792803, "entropy": 0.8001845106482506, "epoch": 0.00988, "grad_norm": 0.17389728128910065, "kl": 0.6115817688405514, "learning_rate": 1.0895916570616387e-05, "loss": -0.0009, "step": 988, "step_time": 5.220471128002828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 774.0, "completions/max_terminated_length": 774.0, "completions/mean_length": 264.90625, "completions/mean_terminated_length": 264.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.45081356167793274, "epoch": 0.00989, "frac_reward_zero_std": 0.25, "grad_norm": 0.13283143937587738, "kl": 0.5012103375047445, "learning_rate": 1.0863667104041801e-05, "loss": 0.0061, "num_tokens": 21217356.0, "reward": 0.7684144973754883, "reward_std": 0.41275596618652344, "rewards/rollout_reward_func/mean": 0.7684144973754883, "rewards/rollout_reward_func/std": 0.51243656873703, "sampling/importance_sampling_ratio/max": 1.1044576168060303, "sampling/importance_sampling_ratio/mean": 1.001554012298584, "sampling/importance_sampling_ratio/min": 0.9171801209449768, "sampling/sampling_logp_difference/max": 0.12628483772277832, "sampling/sampling_logp_difference/mean": 0.008486952632665634, "step": 989, "step_time": 7.261413804997574 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.44735639728605747, "epoch": 0.0099, "grad_norm": 0.13252650201320648, "kl": 0.4993058256804943, "learning_rate": 1.0831551442696507e-05, "loss": 0.0062, "step": 990, "step_time": 3.575318111003071 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 518.59375, "completions/mean_terminated_length": 518.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5926754958927631, "epoch": 0.00991, "frac_reward_zero_std": 0.0, "grad_norm": 0.2870299518108368, "kl": 1.3952560052275658, "learning_rate": 1.079956983481347e-05, "loss": 0.0045, "num_tokens": 21263040.0, "reward": 0.6800887584686279, "reward_std": 0.6100099682807922, "rewards/rollout_reward_func/mean": 0.6800887584686279, "rewards/rollout_reward_func/std": 0.6472162008285522, "sampling/importance_sampling_ratio/max": 1.2492672204971313, "sampling/importance_sampling_ratio/mean": 1.0167007446289062, "sampling/importance_sampling_ratio/min": 0.9024344086647034, "sampling/sampling_logp_difference/max": 0.2265181541442871, "sampling/sampling_logp_difference/mean": 0.014334943145513535, "step": 991, "step_time": 9.067427521000354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.5933578088879585, "epoch": 0.00992, "grad_norm": 0.19786512851715088, "kl": 1.3917904682457447, "learning_rate": 1.076772252758951e-05, "loss": 0.0039, "step": 992, "step_time": 4.849596490001204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 363.0, "completions/mean_terminated_length": 363.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37603453546762466, "epoch": 0.00993, "frac_reward_zero_std": 0.0, "grad_norm": 0.11323463171720505, "kl": 0.34967522136867046, "learning_rate": 1.0736009767183388e-05, "loss": -0.0045, "num_tokens": 21301585.0, "reward": 0.7083573341369629, "reward_std": 0.5654629468917847, "rewards/rollout_reward_func/mean": 0.7083573341369629, "rewards/rollout_reward_func/std": 0.5883799195289612, "sampling/importance_sampling_ratio/max": 1.0851120948791504, "sampling/importance_sampling_ratio/mean": 0.9964020252227783, "sampling/importance_sampling_ratio/min": 0.9287146329879761, "sampling/sampling_logp_difference/max": 0.09263503551483154, "sampling/sampling_logp_difference/mean": 0.008700299076735973, "step": 993, "step_time": 9.081178067002838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.37407981511205435, "epoch": 0.00994, "grad_norm": 0.11439129710197449, "kl": 0.34922363236546516, "learning_rate": 1.07044317987139e-05, "loss": -0.0045, "step": 994, "step_time": 5.3409024180018605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1641.0, "completions/max_terminated_length": 1641.0, "completions/mean_length": 443.9375, "completions/mean_terminated_length": 443.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5830949731171131, "epoch": 0.00995, "frac_reward_zero_std": 0.25, "grad_norm": 0.1900939792394638, "kl": 0.7769625373184681, "learning_rate": 1.0672988866257991e-05, "loss": -0.0002, "num_tokens": 21344400.0, "reward": 0.7163920998573303, "reward_std": 0.5212464332580566, "rewards/rollout_reward_func/mean": 0.7163920998573303, "rewards/rollout_reward_func/std": 0.6287806034088135, "sampling/importance_sampling_ratio/max": 1.0984570980072021, "sampling/importance_sampling_ratio/mean": 0.9928399324417114, "sampling/importance_sampling_ratio/min": 0.6669690012931824, "sampling/sampling_logp_difference/max": 0.24840545654296875, "sampling/sampling_logp_difference/mean": 0.016099480912089348, "step": 995, "step_time": 9.503003522999279 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5776410214602947, "epoch": 0.00996, "grad_norm": 0.1872236430644989, "kl": 0.7480480037629604, "learning_rate": 1.064168121284888e-05, "loss": -0.0014, "step": 996, "step_time": 5.756135918994914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1364.0, "completions/max_terminated_length": 1364.0, "completions/mean_length": 339.125, "completions/mean_terminated_length": 339.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5383289009332657, "epoch": 0.00997, "frac_reward_zero_std": 0.0, "grad_norm": 0.1668912023305893, "kl": 0.7754412442445755, "learning_rate": 1.0610509080474148e-05, "loss": 0.0003, "num_tokens": 21383226.0, "reward": 0.7774429321289062, "reward_std": 0.5448153018951416, "rewards/rollout_reward_func/mean": 0.7774429321289062, "rewards/rollout_reward_func/std": 0.5533227324485779, "sampling/importance_sampling_ratio/max": 1.1296665668487549, "sampling/importance_sampling_ratio/mean": 1.0029466152191162, "sampling/importance_sampling_ratio/min": 0.7819855213165283, "sampling/sampling_logp_difference/max": 0.12191915512084961, "sampling/sampling_logp_difference/mean": 0.012560204602777958, "step": 997, "step_time": 8.601045473997146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0069444444961845875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0069444444961845875, "entropy": 0.5308678913861513, "epoch": 0.00998, "grad_norm": 0.1181015893816948, "kl": 0.7733523175120354, "learning_rate": 1.0579472710073913e-05, "loss": 0.0, "step": 998, "step_time": 4.635775755001305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2558.0, "completions/max_terminated_length": 2558.0, "completions/mean_length": 536.28125, "completions/mean_terminated_length": 536.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5296676196157932, "epoch": 0.00999, "frac_reward_zero_std": 0.25, "grad_norm": 0.08343272656202316, "kl": 0.539655014872551, "learning_rate": 1.0548572341538908e-05, "loss": -0.0132, "num_tokens": 21429757.0, "reward": 0.8004733324050903, "reward_std": 0.3884080648422241, "rewards/rollout_reward_func/mean": 0.8004733324050903, "rewards/rollout_reward_func/std": 0.49080902338027954, "sampling/importance_sampling_ratio/max": 1.3245229721069336, "sampling/importance_sampling_ratio/mean": 1.0126227140426636, "sampling/importance_sampling_ratio/min": 0.928109884262085, "sampling/sampling_logp_difference/max": 0.16009950637817383, "sampling/sampling_logp_difference/mean": 0.011026384308934212, "step": 999, "step_time": 12.380395457001214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5264818612486124, "epoch": 0.01, "grad_norm": 0.07741794735193253, "kl": 0.5352959595620632, "learning_rate": 1.0517808213708696e-05, "loss": -0.0133, "step": 1000, "step_time": 6.976574604001144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 447.28125, "completions/mean_terminated_length": 447.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4914293196052313, "epoch": 0.01001, "frac_reward_zero_std": 0.25, "grad_norm": 0.11579907685518265, "kl": 0.40161895006895065, "learning_rate": 1.0487180564369756e-05, "loss": 0.0025, "num_tokens": 21472017.0, "reward": 0.7718047499656677, "reward_std": 0.4350782036781311, "rewards/rollout_reward_func/mean": 0.7718047499656677, "rewards/rollout_reward_func/std": 0.5048362612724304, "sampling/importance_sampling_ratio/max": 1.117655873298645, "sampling/importance_sampling_ratio/mean": 0.9992102384567261, "sampling/importance_sampling_ratio/min": 0.9014904499053955, "sampling/sampling_logp_difference/max": 0.18353629112243652, "sampling/sampling_logp_difference/mean": 0.015456365421414375, "step": 1001, "step_time": 10.080351811999208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.49022736586630344, "epoch": 0.01002, "grad_norm": 0.10883476585149765, "kl": 0.401247451081872, "learning_rate": 1.0456689630253705e-05, "loss": 0.0023, "step": 1002, "step_time": 5.690466237001601 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2290.0, "completions/max_terminated_length": 2290.0, "completions/mean_length": 547.21875, "completions/mean_terminated_length": 540.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6975128501653671, "epoch": 0.01003, "frac_reward_zero_std": 0.25, "grad_norm": 0.3065994083881378, "kl": 0.480899915099144, "learning_rate": 1.0426335647035434e-05, "loss": 0.0398, "num_tokens": 21518455.0, "reward": 0.8429891467094421, "reward_std": 0.38066035509109497, "rewards/rollout_reward_func/mean": 0.8429891467094421, "rewards/rollout_reward_func/std": 0.4427889883518219, "sampling/importance_sampling_ratio/max": 1.116559624671936, "sampling/importance_sampling_ratio/mean": 0.9894228577613831, "sampling/importance_sampling_ratio/min": 0.5909058451652527, "sampling/sampling_logp_difference/max": 0.31023502349853516, "sampling/sampling_logp_difference/mean": 0.01902964897453785, "step": 1003, "step_time": 11.80069648400422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.6995600163936615, "epoch": 0.01004, "grad_norm": 0.307674765586853, "kl": 0.4770371448248625, "learning_rate": 1.0396118849331274e-05, "loss": 0.038, "step": 1004, "step_time": 6.4858678319997125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 750.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 198.5625, "completions/mean_terminated_length": 198.5625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24669605633243918, "epoch": 0.01005, "frac_reward_zero_std": 0.5, "grad_norm": 0.13201157748699188, "kl": 0.3990416210144758, "learning_rate": 1.0366039470697236e-05, "loss": 0.004, "num_tokens": 21550966.0, "reward": 0.7808796167373657, "reward_std": 0.41383564472198486, "rewards/rollout_reward_func/mean": 0.7808796167373657, "rewards/rollout_reward_func/std": 0.6036065816879272, "sampling/importance_sampling_ratio/max": 1.1027251482009888, "sampling/importance_sampling_ratio/mean": 1.006719708442688, "sampling/importance_sampling_ratio/min": 0.9599050283432007, "sampling/sampling_logp_difference/max": 0.09093284606933594, "sampling/sampling_logp_difference/mean": 0.0062361592426896095, "step": 1005, "step_time": 6.617177439997249 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.2459342998918146, "epoch": 0.01006, "grad_norm": 0.12161234766244888, "kl": 0.3999837413430214, "learning_rate": 1.0336097743627142e-05, "loss": 0.004, "step": 1006, "step_time": 3.5524333699995623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2308.0, "completions/max_terminated_length": 2308.0, "completions/mean_length": 586.40625, "completions/mean_terminated_length": 586.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5316614396870136, "epoch": 0.01007, "frac_reward_zero_std": 0.0, "grad_norm": 0.1796700656414032, "kl": 0.6666673421859741, "learning_rate": 1.0306293899550873e-05, "loss": -0.0021, "num_tokens": 21599353.0, "reward": 0.5702401399612427, "reward_std": 0.8267173767089844, "rewards/rollout_reward_func/mean": 0.5702401399612427, "rewards/rollout_reward_func/std": 0.8253659605979919, "sampling/importance_sampling_ratio/max": 1.1322613954544067, "sampling/importance_sampling_ratio/mean": 0.9916743636131287, "sampling/importance_sampling_ratio/min": 0.802109956741333, "sampling/sampling_logp_difference/max": 0.21565532684326172, "sampling/sampling_logp_difference/mean": 0.013351948000490665, "step": 1007, "step_time": 12.016780326999651 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "entropy": 0.5230284370481968, "epoch": 0.01008, "grad_norm": 0.14266645908355713, "kl": 0.769706504419446, "learning_rate": 1.0276628168832564e-05, "loss": -0.0026, "step": 1008, "step_time": 7.245633361002547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 329.21875, "completions/mean_terminated_length": 329.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.46584872528910637, "epoch": 0.01009, "frac_reward_zero_std": 0.0, "grad_norm": 0.1586562693119049, "kl": 0.346783846616745, "learning_rate": 1.024710078076882e-05, "loss": -0.0078, "num_tokens": 21637547.0, "reward": 0.5920392274856567, "reward_std": 0.7534056901931763, "rewards/rollout_reward_func/mean": 0.5920392274856567, "rewards/rollout_reward_func/std": 0.7495269775390625, "sampling/importance_sampling_ratio/max": 1.135900855064392, "sampling/importance_sampling_ratio/mean": 1.0028547048568726, "sampling/importance_sampling_ratio/min": 0.8966779708862305, "sampling/sampling_logp_difference/max": 0.10907697677612305, "sampling/sampling_logp_difference/mean": 0.008851851336658001, "step": 1009, "step_time": 8.91410225999789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.4604951608926058, "epoch": 0.0101, "grad_norm": 0.09813772886991501, "kl": 0.34727771766483784, "learning_rate": 1.0217711963586953e-05, "loss": -0.0081, "step": 1010, "step_time": 4.839938847999292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2245.0, "completions/max_terminated_length": 2245.0, "completions/mean_length": 549.875, "completions/mean_terminated_length": 541.8709716796875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.593243446201086, "epoch": 0.01011, "frac_reward_zero_std": 0.0, "grad_norm": 0.2445245385169983, "kl": 0.438238762319088, "learning_rate": 1.0188461944443206e-05, "loss": 0.0253, "num_tokens": 21684069.0, "reward": 0.6588027477264404, "reward_std": 0.7035728693008423, "rewards/rollout_reward_func/mean": 0.6588027477264404, "rewards/rollout_reward_func/std": 0.6881868839263916, "sampling/importance_sampling_ratio/max": 1.0810052156448364, "sampling/importance_sampling_ratio/mean": 0.9734106063842773, "sampling/importance_sampling_ratio/min": 0.46971386671066284, "sampling/sampling_logp_difference/max": 0.2126784324645996, "sampling/sampling_logp_difference/mean": 0.01700485125184059, "step": 1011, "step_time": 11.473193002999324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5888897720724344, "epoch": 0.01012, "grad_norm": 0.226095512509346, "kl": 0.43963494151830673, "learning_rate": 1.015935094942101e-05, "loss": 0.0244, "step": 1012, "step_time": 6.409343313001955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2450.0, "completions/max_terminated_length": 2450.0, "completions/mean_length": 381.84375, "completions/mean_terminated_length": 373.8709716796875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5470472734887153, "epoch": 0.01013, "frac_reward_zero_std": 0.0, "grad_norm": 0.19228895008563995, "kl": 0.3854017127305269, "learning_rate": 1.0130379203529239e-05, "loss": -0.016, "num_tokens": 21723443.0, "reward": 0.5847376585006714, "reward_std": 0.6994093060493469, "rewards/rollout_reward_func/mean": 0.5847376585006714, "rewards/rollout_reward_func/std": 0.7166255116462708, "sampling/importance_sampling_ratio/max": 1.0983854532241821, "sampling/importance_sampling_ratio/mean": 0.9982234835624695, "sampling/importance_sampling_ratio/min": 0.6481137871742249, "sampling/sampling_logp_difference/max": 0.18914508819580078, "sampling/sampling_logp_difference/mean": 0.013183013536036015, "step": 1013, "step_time": 11.54831219800144 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017361111240461469, "entropy": 0.5440634777769446, "epoch": 0.01014, "grad_norm": 0.1600855588912964, "kl": 0.39482682570815086, "learning_rate": 1.010154693070044e-05, "loss": -0.017, "step": 1014, "step_time": 6.66611549299887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2185.0, "completions/max_terminated_length": 2185.0, "completions/mean_length": 541.125, "completions/mean_terminated_length": 541.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7028886303305626, "epoch": 0.01015, "frac_reward_zero_std": 0.5, "grad_norm": 0.30199557542800903, "kl": 0.6629322171211243, "learning_rate": 1.0072854353789147e-05, "loss": -0.0172, "num_tokens": 21769283.0, "reward": 0.9004249572753906, "reward_std": 0.21712011098861694, "rewards/rollout_reward_func/mean": 0.9004249572753906, "rewards/rollout_reward_func/std": 0.31457605957984924, "sampling/importance_sampling_ratio/max": 1.3967996835708618, "sampling/importance_sampling_ratio/mean": 1.0158970355987549, "sampling/importance_sampling_ratio/min": 0.8384286165237427, "sampling/sampling_logp_difference/max": 0.15924954414367676, "sampling/sampling_logp_difference/mean": 0.018900323659181595, "step": 1015, "step_time": 12.37755463800022 }, { "clip_ratio/high_max": 0.004310344811528921, "clip_ratio/high_mean": 0.0021551724057644606, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021551724057644606, "entropy": 0.7010127454996109, "epoch": 0.01016, "grad_norm": 0.1743122935295105, "kl": 0.6282603591680527, "learning_rate": 1.0044301694570124e-05, "loss": -0.0179, "step": 1016, "step_time": 6.676713963001021 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 704.8125, "completions/mean_terminated_length": 704.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.7022220566868782, "epoch": 0.01017, "frac_reward_zero_std": 0.5, "grad_norm": 0.2953015863895416, "kl": 0.32056297920644283, "learning_rate": 1.0015889173736671e-05, "loss": 0.032, "num_tokens": 21821748.0, "reward": 0.8457168340682983, "reward_std": 0.35481828451156616, "rewards/rollout_reward_func/mean": 0.8457168340682983, "rewards/rollout_reward_func/std": 0.5042836666107178, "sampling/importance_sampling_ratio/max": 1.138466477394104, "sampling/importance_sampling_ratio/mean": 0.9902911186218262, "sampling/importance_sampling_ratio/min": 0.5356103181838989, "sampling/sampling_logp_difference/max": 0.12558555603027344, "sampling/sampling_logp_difference/mean": 0.013790419325232506, "step": 1017, "step_time": 9.94542436699885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0017361111240461469, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017361111240461469, "entropy": 0.7022295314818621, "epoch": 0.01018, "grad_norm": 0.259756475687027, "kl": 0.3219415694475174, "learning_rate": 9.987617010898895e-06, "loss": 0.0302, "step": 1018, "step_time": 5.198512519998985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1580.0, "completions/max_terminated_length": 1580.0, "completions/mean_length": 574.46875, "completions/mean_terminated_length": 574.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4910347666591406, "epoch": 0.01019, "frac_reward_zero_std": 0.0, "grad_norm": 0.1254168599843979, "kl": 0.3421036805957556, "learning_rate": 9.959485424582055e-06, "loss": 0.004, "num_tokens": 21869150.0, "reward": 0.7775778770446777, "reward_std": 0.5092911720275879, "rewards/rollout_reward_func/mean": 0.7775778770446777, "rewards/rollout_reward_func/std": 0.554548442363739, "sampling/importance_sampling_ratio/max": 1.1316401958465576, "sampling/importance_sampling_ratio/mean": 0.9951803088188171, "sampling/importance_sampling_ratio/min": 0.9286341071128845, "sampling/sampling_logp_difference/max": 0.10189807415008545, "sampling/sampling_logp_difference/mean": 0.009157387539744377, "step": 1019, "step_time": 9.666193981998731 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4903473053127527, "epoch": 0.0102, "grad_norm": 0.12077417224645615, "kl": 0.3428781908005476, "learning_rate": 9.931494632224824e-06, "loss": 0.004, "step": 1020, "step_time": 5.166378983001778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1641.0, "completions/max_terminated_length": 1641.0, "completions/mean_length": 518.65625, "completions/mean_terminated_length": 518.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4370726700872183, "epoch": 0.01021, "frac_reward_zero_std": 0.0, "grad_norm": 0.1593223214149475, "kl": 0.4788453448563814, "learning_rate": 9.903644850177633e-06, "loss": 0.0103, "num_tokens": 21914537.0, "reward": 0.6248436570167542, "reward_std": 0.6816990375518799, "rewards/rollout_reward_func/mean": 0.6248436570167542, "rewards/rollout_reward_func/std": 0.696792721748352, "sampling/importance_sampling_ratio/max": 1.0962443351745605, "sampling/importance_sampling_ratio/mean": 1.0124809741973877, "sampling/importance_sampling_ratio/min": 0.883277177810669, "sampling/sampling_logp_difference/max": 0.12398624420166016, "sampling/sampling_logp_difference/mean": 0.011572595685720444, "step": 1021, "step_time": 9.822020622998025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.43532359786331654, "epoch": 0.01022, "grad_norm": 0.12433409690856934, "kl": 0.47351264394819736, "learning_rate": 9.875936293701005e-06, "loss": 0.0094, "step": 1022, "step_time": 5.918977433995678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2459.0, "completions/max_terminated_length": 2459.0, "completions/mean_length": 587.75, "completions/mean_terminated_length": 587.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6369164623320103, "epoch": 0.01023, "frac_reward_zero_std": 0.0, "grad_norm": 0.4366070032119751, "kl": 0.741073127835989, "learning_rate": 9.84836917696386e-06, "loss": 0.0229, "num_tokens": 21962302.0, "reward": 0.692715048789978, "reward_std": 0.6661700010299683, "rewards/rollout_reward_func/mean": 0.692715048789978, "rewards/rollout_reward_func/std": 0.6742614507675171, "sampling/importance_sampling_ratio/max": 1.1442193984985352, "sampling/importance_sampling_ratio/mean": 0.9968219995498657, "sampling/importance_sampling_ratio/min": 0.7654553055763245, "sampling/sampling_logp_difference/max": 0.24736833572387695, "sampling/sampling_logp_difference/mean": 0.015831233933568, "step": 1023, "step_time": 12.050224390002768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00657894741743803, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00657894741743803, "entropy": 0.6346926828846335, "epoch": 0.01024, "grad_norm": 0.3247493505477905, "kl": 0.7396286092698574, "learning_rate": 9.820943713041922e-06, "loss": 0.0214, "step": 1024, "step_time": 6.684730614999353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2369.0, "completions/max_terminated_length": 2369.0, "completions/mean_length": 653.4375, "completions/mean_terminated_length": 653.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.46774946711957455, "epoch": 0.01025, "frac_reward_zero_std": 0.25, "grad_norm": 0.09735317528247833, "kl": 0.30769453197717667, "learning_rate": 9.793660113915994e-06, "loss": -0.0074, "num_tokens": 22012193.0, "reward": 0.8727589845657349, "reward_std": 0.35989201068878174, "rewards/rollout_reward_func/mean": 0.8727589845657349, "rewards/rollout_reward_func/std": 0.42374876141548157, "sampling/importance_sampling_ratio/max": 1.1559783220291138, "sampling/importance_sampling_ratio/mean": 0.9992760419845581, "sampling/importance_sampling_ratio/min": 0.8976107835769653, "sampling/sampling_logp_difference/max": 0.11546683311462402, "sampling/sampling_logp_difference/mean": 0.008942363783717155, "step": 1025, "step_time": 11.76986500899693 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.4602246396243572, "epoch": 0.01026, "grad_norm": 0.08489783108234406, "kl": 0.31504392996430397, "learning_rate": 9.766518590470375e-06, "loss": -0.0077, "step": 1026, "step_time": 6.586232000001473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1276.0, "completions/max_terminated_length": 1276.0, "completions/mean_length": 354.21875, "completions/mean_terminated_length": 354.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3986867945641279, "epoch": 0.01027, "frac_reward_zero_std": 0.25, "grad_norm": 0.1133754774928093, "kl": 0.47745393589138985, "learning_rate": 9.73951935249121e-06, "loss": 0.0018, "num_tokens": 22051259.0, "reward": 0.7422391176223755, "reward_std": 0.4483627378940582, "rewards/rollout_reward_func/mean": 0.7422391176223755, "rewards/rollout_reward_func/std": 0.5745885372161865, "sampling/importance_sampling_ratio/max": 1.1503615379333496, "sampling/importance_sampling_ratio/mean": 1.0037922859191895, "sampling/importance_sampling_ratio/min": 0.9335223436355591, "sampling/sampling_logp_difference/max": 0.09452223777770996, "sampling/sampling_logp_difference/mean": 0.006473022513091564, "step": 1027, "step_time": 8.146411860998342 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3934101425111294, "epoch": 0.01028, "grad_norm": 0.10808484256267548, "kl": 0.4793460350483656, "learning_rate": 9.712662608664856e-06, "loss": 0.0011, "step": 1028, "step_time": 5.111616647998744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1625.0, "completions/max_terminated_length": 1625.0, "completions/mean_length": 453.0625, "completions/mean_terminated_length": 453.4838562011719, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.6623525582253933, "epoch": 0.01029, "frac_reward_zero_std": 0.0, "grad_norm": 0.15594729781150818, "kl": 0.7383751515299082, "learning_rate": 9.685948566576318e-06, "loss": -0.0061, "num_tokens": 22094481.0, "reward": 0.5017399787902832, "reward_std": 0.8410198092460632, "rewards/rollout_reward_func/mean": 0.5017399787902832, "rewards/rollout_reward_func/std": 0.8322726488113403, "sampling/importance_sampling_ratio/max": 1.1706925630569458, "sampling/importance_sampling_ratio/mean": 1.0162675380706787, "sampling/importance_sampling_ratio/min": 0.8440130949020386, "sampling/sampling_logp_difference/max": 0.17393779754638672, "sampling/sampling_logp_difference/mean": 0.018884843215346336, "step": 1029, "step_time": 9.812366748998102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.658606193959713, "epoch": 0.0103, "grad_norm": 0.15453693270683289, "kl": 0.7548535279929638, "learning_rate": 9.659377432707572e-06, "loss": -0.0061, "step": 1030, "step_time": 5.2710400840005605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1668.0, "completions/max_terminated_length": 1668.0, "completions/mean_length": 424.34375, "completions/mean_terminated_length": 424.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36648413725197315, "epoch": 0.01031, "frac_reward_zero_std": 0.25, "grad_norm": 0.10050114244222641, "kl": 0.7081953622400761, "learning_rate": 9.63294941243604e-06, "loss": 0.0071, "num_tokens": 22135935.0, "reward": 0.6562961339950562, "reward_std": 0.5979263186454773, "rewards/rollout_reward_func/mean": 0.6562961339950562, "rewards/rollout_reward_func/std": 0.6908497214317322, "sampling/importance_sampling_ratio/max": 1.1452503204345703, "sampling/importance_sampling_ratio/mean": 1.0047991275787354, "sampling/importance_sampling_ratio/min": 0.8460845351219177, "sampling/sampling_logp_difference/max": 0.23418235778808594, "sampling/sampling_logp_difference/mean": 0.011096430942416191, "step": 1031, "step_time": 9.490794337998523 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.36396322399377823, "epoch": 0.01032, "grad_norm": 0.09895316511392593, "kl": 0.6818673200905323, "learning_rate": 9.606664710032943e-06, "loss": 0.0066, "step": 1032, "step_time": 5.260927755996818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 417.46875, "completions/mean_terminated_length": 417.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.30049862805753946, "epoch": 0.01033, "frac_reward_zero_std": 0.5, "grad_norm": 0.13094981014728546, "kl": 0.31912973150610924, "learning_rate": 9.580523528661767e-06, "loss": 0.0102, "num_tokens": 22175836.0, "reward": 0.9093714356422424, "reward_std": 0.2563363015651703, "rewards/rollout_reward_func/mean": 0.9093714356422424, "rewards/rollout_reward_func/std": 0.3766235113143921, "sampling/importance_sampling_ratio/max": 1.089363932609558, "sampling/importance_sampling_ratio/mean": 1.0080684423446655, "sampling/importance_sampling_ratio/min": 0.9743208885192871, "sampling/sampling_logp_difference/max": 0.08610254526138306, "sampling/sampling_logp_difference/mean": 0.0042306953109800816, "step": 1033, "step_time": 9.239370470000722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.29890530556440353, "epoch": 0.01034, "grad_norm": 0.11352907866239548, "kl": 0.3242962472140789, "learning_rate": 9.554526070376667e-06, "loss": 0.0098, "step": 1034, "step_time": 5.138542888002121 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1634.0, "completions/max_terminated_length": 1634.0, "completions/mean_length": 544.53125, "completions/mean_terminated_length": 544.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36228748597204685, "epoch": 0.01035, "frac_reward_zero_std": 0.25, "grad_norm": 0.12168285995721817, "kl": 0.6433392204344273, "learning_rate": 9.528672536120909e-06, "loss": 0.0189, "num_tokens": 22221012.0, "reward": 0.8692169785499573, "reward_std": 0.30567270517349243, "rewards/rollout_reward_func/mean": 0.8692169785499573, "rewards/rollout_reward_func/std": 0.3516973853111267, "sampling/importance_sampling_ratio/max": 1.0411931276321411, "sampling/importance_sampling_ratio/mean": 1.0005693435668945, "sampling/importance_sampling_ratio/min": 0.9205164313316345, "sampling/sampling_logp_difference/max": 0.11850786209106445, "sampling/sampling_logp_difference/mean": 0.006305104121565819, "step": 1035, "step_time": 10.071717794999131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3590582646429539, "epoch": 0.01036, "grad_norm": 0.12216056138277054, "kl": 0.6372812539339066, "learning_rate": 9.502963125725337e-06, "loss": 0.0189, "step": 1036, "step_time": 5.716095920000953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1445.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 331.375, "completions/mean_terminated_length": 331.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2783354597631842, "epoch": 0.01037, "frac_reward_zero_std": 0.75, "grad_norm": 0.06139644235372543, "kl": 0.47838329896330833, "learning_rate": 9.477398037906792e-06, "loss": 0.0054, "num_tokens": 22258688.0, "reward": 0.8097238540649414, "reward_std": 0.21870829164981842, "rewards/rollout_reward_func/mean": 0.8097238540649414, "rewards/rollout_reward_func/std": 0.5337938666343689, "sampling/importance_sampling_ratio/max": 1.103014349937439, "sampling/importance_sampling_ratio/mean": 0.9916536808013916, "sampling/importance_sampling_ratio/min": 0.8437854051589966, "sampling/sampling_logp_difference/max": 0.14616680145263672, "sampling/sampling_logp_difference/mean": 0.006127624772489071, "step": 1037, "step_time": 8.845594116997745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.27546369784977287, "epoch": 0.01038, "grad_norm": 0.05287647619843483, "kl": 0.4931228645145893, "learning_rate": 9.451977470266615e-06, "loss": 0.0052, "step": 1038, "step_time": 4.883453409995127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 406.0, "completions/mean_terminated_length": 406.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3091341368854046, "epoch": 0.01039, "frac_reward_zero_std": 0.25, "grad_norm": 0.08766607940196991, "kl": 0.4087150041013956, "learning_rate": 9.426701619289081e-06, "loss": -0.0011, "num_tokens": 22300203.0, "reward": 0.8034917116165161, "reward_std": 0.38535329699516296, "rewards/rollout_reward_func/mean": 0.8034917116165161, "rewards/rollout_reward_func/std": 0.48639652132987976, "sampling/importance_sampling_ratio/max": 1.1560227870941162, "sampling/importance_sampling_ratio/mean": 1.0087804794311523, "sampling/importance_sampling_ratio/min": 0.897472620010376, "sampling/sampling_logp_difference/max": 0.10797572135925293, "sampling/sampling_logp_difference/mean": 0.006655991077423096, "step": 1039, "step_time": 9.177404760999707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3076555021107197, "epoch": 0.0104, "grad_norm": 0.08617821335792542, "kl": 0.40829532966017723, "learning_rate": 9.401570680339922e-06, "loss": -0.0013, "step": 1040, "step_time": 4.920905541999673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1623.0, "completions/max_terminated_length": 1623.0, "completions/mean_length": 444.15625, "completions/mean_terminated_length": 444.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3612256608903408, "epoch": 0.01041, "frac_reward_zero_std": 0.25, "grad_norm": 0.09702389687299728, "kl": 0.34913814812898636, "learning_rate": 9.37658484766478e-06, "loss": -0.0131, "num_tokens": 22343363.0, "reward": 0.816626787185669, "reward_std": 0.5186577439308167, "rewards/rollout_reward_func/mean": 0.816626787185669, "rewards/rollout_reward_func/std": 0.5792884826660156, "sampling/importance_sampling_ratio/max": 1.2302777767181396, "sampling/importance_sampling_ratio/mean": 1.0022342205047607, "sampling/importance_sampling_ratio/min": 0.8752981424331665, "sampling/sampling_logp_difference/max": 0.16196465492248535, "sampling/sampling_logp_difference/mean": 0.007681888062506914, "step": 1041, "step_time": 9.48010564100332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.358860794454813, "epoch": 0.01042, "grad_norm": 0.09876610338687897, "kl": 0.3496484365314245, "learning_rate": 9.35174431438772e-06, "loss": -0.0132, "step": 1042, "step_time": 5.808951565002644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 519.71875, "completions/mean_terminated_length": 519.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3566959798336029, "epoch": 0.01043, "frac_reward_zero_std": 0.0, "grad_norm": 0.19070908427238464, "kl": 0.44952985458076, "learning_rate": 9.327049272509752e-06, "loss": 0.0212, "num_tokens": 22387866.0, "reward": 0.7814509868621826, "reward_std": 0.5344738960266113, "rewards/rollout_reward_func/mean": 0.7814509868621826, "rewards/rollout_reward_func/std": 0.5428634285926819, "sampling/importance_sampling_ratio/max": 1.1365793943405151, "sampling/importance_sampling_ratio/mean": 1.0039300918579102, "sampling/importance_sampling_ratio/min": 0.7990379333496094, "sampling/sampling_logp_difference/max": 0.24695253372192383, "sampling/sampling_logp_difference/mean": 0.008487469516694546, "step": 1043, "step_time": 10.277818821999972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.35432618483901024, "epoch": 0.01044, "grad_norm": 0.16943548619747162, "kl": 0.4605720732361078, "learning_rate": 9.302499912907318e-06, "loss": 0.021, "step": 1044, "step_time": 5.272959163998166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1569.0, "completions/max_terminated_length": 1569.0, "completions/mean_length": 468.46875, "completions/mean_terminated_length": 468.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3250255174934864, "epoch": 0.01045, "frac_reward_zero_std": 0.25, "grad_norm": 0.09106972813606262, "kl": 0.5096468888223171, "learning_rate": 9.27809642533084e-06, "loss": -0.0012, "num_tokens": 22430864.0, "reward": 0.6840323209762573, "reward_std": 0.5070920586585999, "rewards/rollout_reward_func/mean": 0.6840323209762573, "rewards/rollout_reward_func/std": 0.6409109234809875, "sampling/importance_sampling_ratio/max": 1.042015790939331, "sampling/importance_sampling_ratio/mean": 0.9929934740066528, "sampling/importance_sampling_ratio/min": 0.8964815139770508, "sampling/sampling_logp_difference/max": 0.10422897338867188, "sampling/sampling_logp_difference/mean": 0.004037704784423113, "step": 1045, "step_time": 9.337467122997623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.31826578825712204, "epoch": 0.01046, "grad_norm": 0.09085864573717117, "kl": 0.5107631348073483, "learning_rate": 9.253838998403249e-06, "loss": -0.0012, "step": 1046, "step_time": 5.197608460999618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1589.0, "completions/max_terminated_length": 1589.0, "completions/mean_length": 484.46875, "completions/mean_terminated_length": 484.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.40476319659501314, "epoch": 0.01047, "frac_reward_zero_std": 0.0, "grad_norm": 0.2772216200828552, "kl": 0.44030156917870045, "learning_rate": 9.229727819618507e-06, "loss": 0.0302, "num_tokens": 22474617.0, "reward": 0.6515136957168579, "reward_std": 0.6636727452278137, "rewards/rollout_reward_func/mean": 0.6515136957168579, "rewards/rollout_reward_func/std": 0.6494937539100647, "sampling/importance_sampling_ratio/max": 1.1442294120788574, "sampling/importance_sampling_ratio/mean": 1.004911184310913, "sampling/importance_sampling_ratio/min": 0.9087486267089844, "sampling/sampling_logp_difference/max": 0.2552759647369385, "sampling/sampling_logp_difference/mean": 0.013152332976460457, "step": 1047, "step_time": 9.499109058000613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0029761905316263437, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029761905316263437, "entropy": 0.398785138502717, "epoch": 0.01048, "grad_norm": 0.21805502474308014, "kl": 0.4399401694536209, "learning_rate": 9.205763075340194e-06, "loss": 0.0287, "step": 1048, "step_time": 5.213938408998729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1472.0, "completions/max_terminated_length": 1472.0, "completions/mean_length": 478.5, "completions/mean_terminated_length": 478.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31820257753133774, "epoch": 0.01049, "frac_reward_zero_std": 0.5, "grad_norm": 0.08648572117090225, "kl": 1.0667355079203844, "learning_rate": 9.181944950800031e-06, "loss": 0.0005, "num_tokens": 22518027.0, "reward": 0.7804548740386963, "reward_std": 0.4164862036705017, "rewards/rollout_reward_func/mean": 0.7804548740386963, "rewards/rollout_reward_func/std": 0.6059563755989075, "sampling/importance_sampling_ratio/max": 1.1332162618637085, "sampling/importance_sampling_ratio/mean": 1.0015883445739746, "sampling/importance_sampling_ratio/min": 0.8941875100135803, "sampling/sampling_logp_difference/max": 0.12207984924316406, "sampling/sampling_logp_difference/mean": 0.00920182466506958, "step": 1049, "step_time": 9.662050014998385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3097911365330219, "epoch": 0.0105, "grad_norm": 0.08176665008068085, "kl": 1.0916020572185516, "learning_rate": 9.158273630096472e-06, "loss": 0.0005, "step": 1050, "step_time": 5.5203424470000755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2254.0, "completions/max_terminated_length": 2254.0, "completions/mean_length": 567.75, "completions/mean_terminated_length": 560.9031982421875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.399919705465436, "epoch": 0.01051, "frac_reward_zero_std": 0.25, "grad_norm": 0.27997663617134094, "kl": 0.47516701370477676, "learning_rate": 9.134749296193276e-06, "loss": 0.0328, "num_tokens": 22564490.0, "reward": 0.6840512156486511, "reward_std": 0.5103729367256165, "rewards/rollout_reward_func/mean": 0.6840512156486511, "rewards/rollout_reward_func/std": 0.6416630148887634, "sampling/importance_sampling_ratio/max": 1.1061992645263672, "sampling/importance_sampling_ratio/mean": 0.9918243885040283, "sampling/importance_sampling_ratio/min": 0.6713361740112305, "sampling/sampling_logp_difference/max": 0.187591552734375, "sampling/sampling_logp_difference/mean": 0.012560725212097168, "step": 1051, "step_time": 11.507306503996006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016447368543595076, "entropy": 0.39444511756300926, "epoch": 0.01052, "grad_norm": 0.28029972314834595, "kl": 0.4877605140209198, "learning_rate": 9.111372130918094e-06, "loss": 0.0317, "step": 1052, "step_time": 6.444802413996513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1524.0, "completions/max_terminated_length": 1524.0, "completions/mean_length": 439.6875, "completions/mean_terminated_length": 439.6875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.33689522789791226, "epoch": 0.01053, "frac_reward_zero_std": 0.5, "grad_norm": 0.11772708594799042, "kl": 0.4707602858543396, "learning_rate": 9.088142314961045e-06, "loss": 0.0042, "num_tokens": 22606750.0, "reward": 0.6536016464233398, "reward_std": 0.4347113072872162, "rewards/rollout_reward_func/mean": 0.6536016464233398, "rewards/rollout_reward_func/std": 0.6989619135856628, "sampling/importance_sampling_ratio/max": 1.1163842678070068, "sampling/importance_sampling_ratio/mean": 0.9910528063774109, "sampling/importance_sampling_ratio/min": 0.8853083252906799, "sampling/sampling_logp_difference/max": 0.12583589553833008, "sampling/sampling_logp_difference/mean": 0.007041080854833126, "step": 1053, "step_time": 9.019177201997081 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.33831473789177835, "epoch": 0.01054, "grad_norm": 0.11959227919578552, "kl": 0.4698013961315155, "learning_rate": 9.065060027873342e-06, "loss": 0.0043, "step": 1054, "step_time": 4.980191805998402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1644.0, "completions/max_terminated_length": 1420.0, "completions/mean_length": 433.8125, "completions/mean_terminated_length": 394.774169921875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.47562482580542564, "epoch": 0.01055, "frac_reward_zero_std": 0.25, "grad_norm": 0.338712602853775, "kl": 0.627407930791378, "learning_rate": 9.042125448065904e-06, "loss": 0.0299, "num_tokens": 22648873.0, "reward": 0.7226930856704712, "reward_std": 0.5844000577926636, "rewards/rollout_reward_func/mean": 0.7226930856704712, "rewards/rollout_reward_func/std": 0.6698856949806213, "sampling/importance_sampling_ratio/max": 1.104061484336853, "sampling/importance_sampling_ratio/mean": 0.9927852153778076, "sampling/importance_sampling_ratio/min": 0.7576152682304382, "sampling/sampling_logp_difference/max": 0.1819171905517578, "sampling/sampling_logp_difference/mean": 0.01330641470849514, "step": 1055, "step_time": 9.819867881997197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.47403199411928654, "epoch": 0.01056, "grad_norm": 0.31540802121162415, "kl": 0.6284837014973164, "learning_rate": 9.019338752807965e-06, "loss": 0.0282, "step": 1056, "step_time": 5.7978819370036945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1641.0, "completions/max_terminated_length": 1641.0, "completions/mean_length": 510.1875, "completions/mean_terminated_length": 510.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24295570887625217, "epoch": 0.01057, "frac_reward_zero_std": 0.25, "grad_norm": 0.08302038162946701, "kl": 0.7105354256927967, "learning_rate": 8.996700118225708e-06, "loss": 0.0081, "num_tokens": 22693882.0, "reward": 0.7481119632720947, "reward_std": 0.40356141328811646, "rewards/rollout_reward_func/mean": 0.7481119632720947, "rewards/rollout_reward_func/std": 0.5641643404960632, "sampling/importance_sampling_ratio/max": 1.1325584650039673, "sampling/importance_sampling_ratio/mean": 0.9932246804237366, "sampling/importance_sampling_ratio/min": 0.793649435043335, "sampling/sampling_logp_difference/max": 0.22584152221679688, "sampling/sampling_logp_difference/mean": 0.006504569202661514, "step": 1057, "step_time": 9.54248716800248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.23860608600080013, "epoch": 0.01058, "grad_norm": 0.08191939443349838, "kl": 0.7297929152846336, "learning_rate": 8.974209719300898e-06, "loss": 0.0083, "step": 1058, "step_time": 5.321625315998972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1382.0, "completions/max_terminated_length": 1382.0, "completions/mean_length": 324.46875, "completions/mean_terminated_length": 324.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22833205852657557, "epoch": 0.01059, "frac_reward_zero_std": 0.25, "grad_norm": 0.04367811232805252, "kl": 0.5060882568359375, "learning_rate": 8.951867729869552e-06, "loss": -0.0003, "num_tokens": 22731983.0, "reward": 0.8295266628265381, "reward_std": 0.34961676597595215, "rewards/rollout_reward_func/mean": 0.8295266628265381, "rewards/rollout_reward_func/std": 0.4025214910507202, "sampling/importance_sampling_ratio/max": 1.0285474061965942, "sampling/importance_sampling_ratio/mean": 0.9953867793083191, "sampling/importance_sampling_ratio/min": 0.9104041457176208, "sampling/sampling_logp_difference/max": 0.10524821281433105, "sampling/sampling_logp_difference/mean": 0.00395559286698699, "step": 1059, "step_time": 8.541143583999656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.22881071083247662, "epoch": 0.0106, "grad_norm": 0.0434047132730484, "kl": 0.5041688829660416, "learning_rate": 8.929674322620558e-06, "loss": -0.0004, "step": 1060, "step_time": 4.6590959949971875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2245.0, "completions/max_terminated_length": 2245.0, "completions/mean_length": 564.625, "completions/mean_terminated_length": 564.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26000055857002735, "epoch": 0.01061, "frac_reward_zero_std": 0.0, "grad_norm": 0.1111983209848404, "kl": 0.42957746237516403, "learning_rate": 8.907629669094378e-06, "loss": 0.0084, "num_tokens": 22778448.0, "reward": 0.6300641894340515, "reward_std": 0.6916847229003906, "rewards/rollout_reward_func/mean": 0.6300641894340515, "rewards/rollout_reward_func/std": 0.6902191042900085, "sampling/importance_sampling_ratio/max": 1.0422213077545166, "sampling/importance_sampling_ratio/mean": 0.9913418889045715, "sampling/importance_sampling_ratio/min": 0.8549982905387878, "sampling/sampling_logp_difference/max": 0.11703062057495117, "sampling/sampling_logp_difference/mean": 0.005756302736699581, "step": 1061, "step_time": 11.287751201998617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.2610400803387165, "epoch": 0.01062, "grad_norm": 0.10190980136394501, "kl": 0.42432059720158577, "learning_rate": 8.885733939681703e-06, "loss": 0.0082, "step": 1062, "step_time": 6.84169811799984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3116.0, "completions/max_terminated_length": 3116.0, "completions/mean_length": 398.8125, "completions/mean_terminated_length": 398.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18088906351476908, "epoch": 0.01063, "frac_reward_zero_std": 0.5, "grad_norm": 0.02722090296447277, "kl": 0.4090450815856457, "learning_rate": 8.863987303622129e-06, "loss": -0.0027, "num_tokens": 22818184.0, "reward": 0.8632384538650513, "reward_std": 0.23838075995445251, "rewards/rollout_reward_func/mean": 0.8632384538650513, "rewards/rollout_reward_func/std": 0.3676624596118927, "sampling/importance_sampling_ratio/max": 1.01045560836792, "sampling/importance_sampling_ratio/mean": 0.9927517771720886, "sampling/importance_sampling_ratio/min": 0.8974665403366089, "sampling/sampling_logp_difference/max": 0.11286401748657227, "sampling/sampling_logp_difference/mean": 0.0050683580338954926, "step": 1063, "step_time": 13.218191980002302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.18082571122795343, "epoch": 0.01064, "grad_norm": 0.03034386783838272, "kl": 0.4178295135498047, "learning_rate": 8.842389929002872e-06, "loss": -0.0026, "step": 1064, "step_time": 7.787939551000818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 467.34375, "completions/mean_terminated_length": 467.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3614361835643649, "epoch": 0.01065, "frac_reward_zero_std": 0.25, "grad_norm": 0.3777448534965515, "kl": 0.5758232772350311, "learning_rate": 8.820941982757435e-06, "loss": 0.0404, "num_tokens": 22862090.0, "reward": 0.777869701385498, "reward_std": 0.4799022078514099, "rewards/rollout_reward_func/mean": 0.777869701385498, "rewards/rollout_reward_func/std": 0.5517655611038208, "sampling/importance_sampling_ratio/max": 1.1092290878295898, "sampling/importance_sampling_ratio/mean": 0.9919670224189758, "sampling/importance_sampling_ratio/min": 0.8510570526123047, "sampling/sampling_logp_difference/max": 0.12828922271728516, "sampling/sampling_logp_difference/mean": 0.011965589597821236, "step": 1065, "step_time": 9.376106736999645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.36270343605428934, "epoch": 0.01066, "grad_norm": 0.3562244176864624, "kl": 0.5697130523622036, "learning_rate": 8.799643630664363e-06, "loss": 0.039, "step": 1066, "step_time": 5.215511880998747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1614.0, "completions/max_terminated_length": 1614.0, "completions/mean_length": 450.21875, "completions/mean_terminated_length": 450.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.278018930926919, "epoch": 0.01067, "frac_reward_zero_std": 0.0, "grad_norm": 0.08959126472473145, "kl": 0.46130549162626266, "learning_rate": 8.778495037345924e-06, "loss": -0.0092, "num_tokens": 22904840.0, "reward": 0.808812141418457, "reward_std": 0.5407611131668091, "rewards/rollout_reward_func/mean": 0.808812141418457, "rewards/rollout_reward_func/std": 0.5374311208724976, "sampling/importance_sampling_ratio/max": 1.1148617267608643, "sampling/importance_sampling_ratio/mean": 0.999653697013855, "sampling/importance_sampling_ratio/min": 0.8835790753364563, "sampling/sampling_logp_difference/max": 0.1261129379272461, "sampling/sampling_logp_difference/mean": 0.008206892758607864, "step": 1067, "step_time": 9.649406300999544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.2786626033484936, "epoch": 0.01068, "grad_norm": 0.08594606816768646, "kl": 0.46240610256791115, "learning_rate": 8.75749636626684e-06, "loss": -0.009, "step": 1068, "step_time": 5.288499778000187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1587.0, "completions/max_terminated_length": 1587.0, "completions/mean_length": 591.78125, "completions/mean_terminated_length": 591.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3096534162759781, "epoch": 0.01069, "frac_reward_zero_std": 0.5, "grad_norm": 0.1256857067346573, "kl": 0.45966344326734543, "learning_rate": 8.736647779733048e-06, "loss": -0.0034, "num_tokens": 22953620.0, "reward": 0.8038309812545776, "reward_std": 0.31478413939476013, "rewards/rollout_reward_func/mean": 0.8038309812545776, "rewards/rollout_reward_func/std": 0.48238247632980347, "sampling/importance_sampling_ratio/max": 1.1182689666748047, "sampling/importance_sampling_ratio/mean": 0.9990621209144592, "sampling/importance_sampling_ratio/min": 0.8288938999176025, "sampling/sampling_logp_difference/max": 0.11159276962280273, "sampling/sampling_logp_difference/mean": 0.008118413388729095, "step": 1069, "step_time": 10.324559380997016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.31185940466821194, "epoch": 0.0107, "grad_norm": 0.09674348682165146, "kl": 0.45782935805618763, "learning_rate": 8.715949438890425e-06, "loss": -0.0038, "step": 1070, "step_time": 5.781137250998654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1578.0, "completions/max_terminated_length": 1578.0, "completions/mean_length": 416.84375, "completions/mean_terminated_length": 416.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23072329349815845, "epoch": 0.01071, "frac_reward_zero_std": 0.0, "grad_norm": 0.05716804414987564, "kl": 0.4020780920982361, "learning_rate": 8.695401503723542e-06, "loss": -0.0025, "num_tokens": 22994785.0, "reward": 0.701674222946167, "reward_std": 0.5468101501464844, "rewards/rollout_reward_func/mean": 0.701674222946167, "rewards/rollout_reward_func/std": 0.5458328127861023, "sampling/importance_sampling_ratio/max": 1.024645447731018, "sampling/importance_sampling_ratio/mean": 0.9977872967720032, "sampling/importance_sampling_ratio/min": 0.8964433073997498, "sampling/sampling_logp_difference/max": 0.10968232154846191, "sampling/sampling_logp_difference/mean": 0.003134356811642647, "step": 1071, "step_time": 9.109685821998937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.23045906610786915, "epoch": 0.01072, "grad_norm": 0.05738214775919914, "kl": 0.3997420482337475, "learning_rate": 8.675004133054441e-06, "loss": -0.0024, "step": 1072, "step_time": 5.120844875002149 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1490.0, "completions/max_terminated_length": 1490.0, "completions/mean_length": 404.15625, "completions/mean_terminated_length": 404.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2264578640460968, "epoch": 0.01073, "frac_reward_zero_std": 0.75, "grad_norm": 0.011162705719470978, "kl": 0.5180751644074917, "learning_rate": 8.65475748454139e-06, "loss": -0.0033, "num_tokens": 23036410.0, "reward": 0.9650489091873169, "reward_std": 0.09885671734809875, "rewards/rollout_reward_func/mean": 0.9650489091873169, "rewards/rollout_reward_func/std": 0.1977134346961975, "sampling/importance_sampling_ratio/max": 1.1220507621765137, "sampling/importance_sampling_ratio/mean": 1.004334568977356, "sampling/importance_sampling_ratio/min": 0.9007783532142639, "sampling/sampling_logp_difference/max": 0.11511564254760742, "sampling/sampling_logp_difference/mean": 0.0044988482259213924, "step": 1073, "step_time": 9.661780449001526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.22786229476332664, "epoch": 0.01074, "grad_norm": 0.011285630986094475, "kl": 0.51821269094944, "learning_rate": 8.63466171467769e-06, "loss": -0.0033, "step": 1074, "step_time": 5.017808053997214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 446.9375, "completions/mean_terminated_length": 446.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24156934581696987, "epoch": 0.01075, "frac_reward_zero_std": 0.5, "grad_norm": 0.1905220001935959, "kl": 0.46116207353770733, "learning_rate": 8.614716978790426e-06, "loss": 0.0055, "num_tokens": 23078510.0, "reward": 0.8450131416320801, "reward_std": 0.35905200242996216, "rewards/rollout_reward_func/mean": 0.8450131416320801, "rewards/rollout_reward_func/std": 0.5100948810577393, "sampling/importance_sampling_ratio/max": 1.126880168914795, "sampling/importance_sampling_ratio/mean": 1.010589838027954, "sampling/importance_sampling_ratio/min": 0.972494900226593, "sampling/sampling_logp_difference/max": 0.11513710021972656, "sampling/sampling_logp_difference/mean": 0.005116766318678856, "step": 1075, "step_time": 9.493649536001612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.23798217810690403, "epoch": 0.01076, "grad_norm": 0.16145706176757812, "kl": 0.4602934494614601, "learning_rate": 8.594923431039315e-06, "loss": 0.0048, "step": 1076, "step_time": 5.479807029998483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 461.84375, "completions/mean_terminated_length": 461.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.26416694186627865, "epoch": 0.01077, "frac_reward_zero_std": 0.0, "grad_norm": 0.13819226622581482, "kl": 0.49167245626449585, "learning_rate": 8.575281224415466e-06, "loss": -0.0042, "num_tokens": 23122903.0, "reward": 0.6532799601554871, "reward_std": 0.7119948863983154, "rewards/rollout_reward_func/mean": 0.6532799601554871, "rewards/rollout_reward_func/std": 0.697978138923645, "sampling/importance_sampling_ratio/max": 1.1184771060943604, "sampling/importance_sampling_ratio/mean": 1.0069851875305176, "sampling/importance_sampling_ratio/min": 0.968165397644043, "sampling/sampling_logp_difference/max": 0.11166071891784668, "sampling/sampling_logp_difference/mean": 0.004868670832365751, "step": 1077, "step_time": 9.646733186000347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.2592574991285801, "epoch": 0.01078, "grad_norm": 0.12455639243125916, "kl": 0.49327579513192177, "learning_rate": 8.555790510740235e-06, "loss": -0.0049, "step": 1078, "step_time": 5.056474726003216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 412.75, "completions/mean_terminated_length": 412.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.22668366692960262, "epoch": 0.01079, "frac_reward_zero_std": 0.0, "grad_norm": 0.05896841362118721, "kl": 0.669188842177391, "learning_rate": 8.536451440664032e-06, "loss": -0.0039, "num_tokens": 23164805.0, "reward": 0.6742790937423706, "reward_std": 0.6255390644073486, "rewards/rollout_reward_func/mean": 0.6742790937423706, "rewards/rollout_reward_func/std": 0.6067628860473633, "sampling/importance_sampling_ratio/max": 1.0931106805801392, "sampling/importance_sampling_ratio/mean": 1.0014973878860474, "sampling/importance_sampling_ratio/min": 0.9223719239234924, "sampling/sampling_logp_difference/max": 0.08913946151733398, "sampling/sampling_logp_difference/mean": 0.0038312338292598724, "step": 1079, "step_time": 9.036371832000441 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.22686211112886667, "epoch": 0.0108, "grad_norm": 0.05858428031206131, "kl": 0.6492383908480406, "learning_rate": 8.51726416366516e-06, "loss": -0.0037, "step": 1080, "step_time": 5.004615352998371 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1569.0, "completions/max_terminated_length": 1569.0, "completions/mean_length": 476.0, "completions/mean_terminated_length": 476.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2684989017434418, "epoch": 0.01081, "frac_reward_zero_std": 0.0, "grad_norm": 0.18554386496543884, "kl": 0.6292683780193329, "learning_rate": 8.498228828048673e-06, "loss": 0.0211, "num_tokens": 23208304.0, "reward": 0.7818640470504761, "reward_std": 0.537085771560669, "rewards/rollout_reward_func/mean": 0.7818640470504761, "rewards/rollout_reward_func/std": 0.5482771992683411, "sampling/importance_sampling_ratio/max": 1.1211334466934204, "sampling/importance_sampling_ratio/mean": 0.9966644048690796, "sampling/importance_sampling_ratio/min": 0.9015609622001648, "sampling/sampling_logp_difference/max": 0.11439132690429688, "sampling/sampling_logp_difference/mean": 0.009088730439543724, "step": 1081, "step_time": 9.420143258999815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.26569790323264897, "epoch": 0.01082, "grad_norm": 0.16729754209518433, "kl": 0.6342995837330818, "learning_rate": 8.479345580945194e-06, "loss": 0.0202, "step": 1082, "step_time": 5.195667152000169 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 445.59375, "completions/mean_terminated_length": 445.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36809172108769417, "epoch": 0.01083, "frac_reward_zero_std": 0.25, "grad_norm": 0.2560299038887024, "kl": 0.5050256848335266, "learning_rate": 8.460614568309829e-06, "loss": -0.0015, "num_tokens": 23250902.0, "reward": 0.8708070516586304, "reward_std": 0.3654128909111023, "rewards/rollout_reward_func/mean": 0.8708070516586304, "rewards/rollout_reward_func/std": 0.4304378032684326, "sampling/importance_sampling_ratio/max": 1.0489152669906616, "sampling/importance_sampling_ratio/mean": 0.9960245490074158, "sampling/importance_sampling_ratio/min": 0.9674866795539856, "sampling/sampling_logp_difference/max": 0.2177443504333496, "sampling/sampling_logp_difference/mean": 0.009431262500584126, "step": 1083, "step_time": 10.37212524199822 }, { "clip_ratio/high_max": 0.008928571827709675, "clip_ratio/high_mean": 0.004464285913854837, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "entropy": 0.3675379268825054, "epoch": 0.01084, "grad_norm": 0.23602715134620667, "kl": 0.5052991211414337, "learning_rate": 8.442035934920992e-06, "loss": -0.0021, "step": 1084, "step_time": 4.986682132999704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2504.0, "completions/max_terminated_length": 2504.0, "completions/mean_length": 580.59375, "completions/mean_terminated_length": 580.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2363385632634163, "epoch": 0.01085, "frac_reward_zero_std": 0.25, "grad_norm": 0.13626311719417572, "kl": 0.9349316880106926, "learning_rate": 8.42360982437932e-06, "loss": 0.0189, "num_tokens": 23297992.0, "reward": 0.804507851600647, "reward_std": 0.3456522822380066, "rewards/rollout_reward_func/mean": 0.804507851600647, "rewards/rollout_reward_func/std": 0.41395804286003113, "sampling/importance_sampling_ratio/max": 1.1084718704223633, "sampling/importance_sampling_ratio/mean": 0.9927401542663574, "sampling/importance_sampling_ratio/min": 0.9024538397789001, "sampling/sampling_logp_difference/max": 0.10457849502563477, "sampling/sampling_logp_difference/mean": 0.006550848484039307, "step": 1085, "step_time": 11.988951609999276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.23919539153575897, "epoch": 0.01086, "grad_norm": 0.12154050916433334, "kl": 0.9287264049053192, "learning_rate": 8.405336379106532e-06, "loss": 0.0185, "step": 1086, "step_time": 6.771460990998094 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3323.0, "completions/max_terminated_length": 3323.0, "completions/mean_length": 405.03125, "completions/mean_terminated_length": 405.03125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2201549676246941, "epoch": 0.01087, "frac_reward_zero_std": 0.5, "grad_norm": 0.06999171525239944, "kl": 0.39836302399635315, "learning_rate": 8.387215740344354e-06, "loss": 0.0069, "num_tokens": 23338474.0, "reward": 0.9052610397338867, "reward_std": 0.26796215772628784, "rewards/rollout_reward_func/mean": 0.9052610397338867, "rewards/rollout_reward_func/std": 0.38899531960487366, "sampling/importance_sampling_ratio/max": 1.0277780294418335, "sampling/importance_sampling_ratio/mean": 0.9931108951568604, "sampling/importance_sampling_ratio/min": 0.9108919501304626, "sampling/sampling_logp_difference/max": 0.09333133697509766, "sampling/sampling_logp_difference/mean": 0.0034523787908256054, "step": 1087, "step_time": 13.718529108000439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.22644706070423126, "epoch": 0.01088, "grad_norm": 0.07008935511112213, "kl": 0.3900897167623043, "learning_rate": 8.369248048153426e-06, "loss": 0.0066, "step": 1088, "step_time": 8.151349538995419 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1560.0, "completions/max_terminated_length": 1560.0, "completions/mean_length": 482.125, "completions/mean_terminated_length": 482.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3153818864375353, "epoch": 0.01089, "frac_reward_zero_std": 0.0, "grad_norm": 0.07200273126363754, "kl": 0.5010207183659077, "learning_rate": 8.351433441412197e-06, "loss": 0.0043, "num_tokens": 23382290.0, "reward": 0.6839402914047241, "reward_std": 0.6007345914840698, "rewards/rollout_reward_func/mean": 0.6839402914047241, "rewards/rollout_reward_func/std": 0.6421211957931519, "sampling/importance_sampling_ratio/max": 1.0423372983932495, "sampling/importance_sampling_ratio/mean": 0.9959174394607544, "sampling/importance_sampling_ratio/min": 0.8986464142799377, "sampling/sampling_logp_difference/max": 0.10663270950317383, "sampling/sampling_logp_difference/mean": 0.00604808796197176, "step": 1089, "step_time": 9.37182093699812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3173520416021347, "epoch": 0.0109, "grad_norm": 0.07304983586072922, "kl": 0.49798733554780483, "learning_rate": 8.333772057815877e-06, "loss": 0.0041, "step": 1090, "step_time": 6.265428561999215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1506.0, "completions/max_terminated_length": 1506.0, "completions/mean_length": 494.875, "completions/mean_terminated_length": 494.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2717474903911352, "epoch": 0.01091, "frac_reward_zero_std": 0.75, "grad_norm": 0.136806920170784, "kl": 0.4586724117398262, "learning_rate": 8.316264033875352e-06, "loss": 0.003, "num_tokens": 23425782.0, "reward": 0.879570484161377, "reward_std": 0.2229921668767929, "rewards/rollout_reward_func/mean": 0.879570484161377, "rewards/rollout_reward_func/std": 0.4738847613334656, "sampling/importance_sampling_ratio/max": 1.115903377532959, "sampling/importance_sampling_ratio/mean": 0.9982657432556152, "sampling/importance_sampling_ratio/min": 0.9026070833206177, "sampling/sampling_logp_difference/max": 0.11674952507019043, "sampling/sampling_logp_difference/mean": 0.0073256418108940125, "step": 1091, "step_time": 9.166672588000438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.27783893421292305, "epoch": 0.01092, "grad_norm": 0.14987191557884216, "kl": 0.4475875794887543, "learning_rate": 8.298909504916144e-06, "loss": 0.0029, "step": 1092, "step_time": 5.024624932000734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 390.0625, "completions/mean_terminated_length": 390.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2462918609380722, "epoch": 0.01093, "frac_reward_zero_std": 0.0, "grad_norm": 0.06261121481657028, "kl": 0.5219688527286053, "learning_rate": 8.281708605077367e-06, "loss": 0.0017, "num_tokens": 23465684.0, "reward": 0.6547831296920776, "reward_std": 0.6817915439605713, "rewards/rollout_reward_func/mean": 0.6547831296920776, "rewards/rollout_reward_func/std": 0.6948884725570679, "sampling/importance_sampling_ratio/max": 1.1254329681396484, "sampling/importance_sampling_ratio/mean": 1.0046669244766235, "sampling/importance_sampling_ratio/min": 0.9665688276290894, "sampling/sampling_logp_difference/max": 0.11497855186462402, "sampling/sampling_logp_difference/mean": 0.005114741623401642, "step": 1093, "step_time": 9.483865294998395 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.25209171883761883, "epoch": 0.01094, "grad_norm": 0.06494791060686111, "kl": 0.5192476138472557, "learning_rate": 8.264661467310666e-06, "loss": 0.0014, "step": 1094, "step_time": 5.240588126000148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1355.0, "completions/max_terminated_length": 1355.0, "completions/mean_length": 529.5, "completions/mean_terminated_length": 529.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.35025839030276984, "epoch": 0.01095, "frac_reward_zero_std": 0.25, "grad_norm": 0.12177559733390808, "kl": 0.43242916092276573, "learning_rate": 8.247768223379224e-06, "loss": 0.0031, "num_tokens": 23512237.0, "reward": 0.8170170783996582, "reward_std": 0.5175538063049316, "rewards/rollout_reward_func/mean": 0.8170170783996582, "rewards/rollout_reward_func/std": 0.5780307650566101, "sampling/importance_sampling_ratio/max": 1.1099787950515747, "sampling/importance_sampling_ratio/mean": 1.000449776649475, "sampling/importance_sampling_ratio/min": 0.8988704085350037, "sampling/sampling_logp_difference/max": 0.13258624076843262, "sampling/sampling_logp_difference/mean": 0.010403139516711235, "step": 1095, "step_time": 9.006445519999033 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.352808577939868, "epoch": 0.01096, "grad_norm": 0.12289508432149887, "kl": 0.43280818313360214, "learning_rate": 8.23102900385671e-06, "loss": 0.0031, "step": 1096, "step_time": 4.747766304999459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1569.0, "completions/max_terminated_length": 1569.0, "completions/mean_length": 561.5, "completions/mean_terminated_length": 561.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5779245402663946, "epoch": 0.01097, "frac_reward_zero_std": 0.25, "grad_norm": 0.17731623351573944, "kl": 0.4174916371703148, "learning_rate": 8.214443938126299e-06, "loss": -0.0386, "num_tokens": 23559107.0, "reward": 0.775587260723114, "reward_std": 0.4485362768173218, "rewards/rollout_reward_func/mean": 0.775587260723114, "rewards/rollout_reward_func/std": 0.5556575655937195, "sampling/importance_sampling_ratio/max": 1.2065891027450562, "sampling/importance_sampling_ratio/mean": 1.0153777599334717, "sampling/importance_sampling_ratio/min": 0.9628409147262573, "sampling/sampling_logp_difference/max": 0.11366844177246094, "sampling/sampling_logp_difference/mean": 0.013132947497069836, "step": 1097, "step_time": 10.570258331001241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5808900725096464, "epoch": 0.01098, "grad_norm": 0.18010278046131134, "kl": 0.4126411061733961, "learning_rate": 8.198013154379645e-06, "loss": -0.0394, "step": 1098, "step_time": 5.1035589480015915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2281.0, "completions/max_terminated_length": 2281.0, "completions/mean_length": 561.65625, "completions/mean_terminated_length": 561.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2975123021751642, "epoch": 0.01099, "frac_reward_zero_std": 0.0, "grad_norm": 0.10066211223602295, "kl": 0.3709176555275917, "learning_rate": 8.181736779615915e-06, "loss": -0.0017, "num_tokens": 23605284.0, "reward": 0.7715686559677124, "reward_std": 0.5165228843688965, "rewards/rollout_reward_func/mean": 0.7715686559677124, "rewards/rollout_reward_func/std": 0.5053003430366516, "sampling/importance_sampling_ratio/max": 1.0411337614059448, "sampling/importance_sampling_ratio/mean": 0.9930475950241089, "sampling/importance_sampling_ratio/min": 0.8860348463058472, "sampling/sampling_logp_difference/max": 0.12134075164794922, "sampling/sampling_logp_difference/mean": 0.00702091958373785, "step": 1099, "step_time": 11.566034497000146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.301618330180645, "epoch": 0.011, "grad_norm": 0.09991636127233505, "kl": 0.370284765958786, "learning_rate": 8.165614939640796e-06, "loss": -0.0019, "step": 1100, "step_time": 6.556738848999885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1659.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 284.78125, "completions/mean_terminated_length": 284.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.19494295632466674, "epoch": 0.01101, "frac_reward_zero_std": 0.25, "grad_norm": 0.10356244444847107, "kl": 0.3762635514140129, "learning_rate": 8.149647759065512e-06, "loss": 0.0035, "num_tokens": 23640952.0, "reward": 0.8011044263839722, "reward_std": 0.38486140966415405, "rewards/rollout_reward_func/mean": 0.8011044263839722, "rewards/rollout_reward_func/std": 0.4908045828342438, "sampling/importance_sampling_ratio/max": 1.0852385759353638, "sampling/importance_sampling_ratio/mean": 0.9949979782104492, "sampling/importance_sampling_ratio/min": 0.9021697044372559, "sampling/sampling_logp_difference/max": 0.1045234203338623, "sampling/sampling_logp_difference/mean": 0.008119367063045502, "step": 1101, "step_time": 9.21375082600025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.196523267775774, "epoch": 0.01102, "grad_norm": 0.11020547896623611, "kl": 0.3754427917301655, "learning_rate": 8.133835361305882e-06, "loss": 0.0038, "step": 1102, "step_time": 5.117249583001467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1569.0, "completions/max_terminated_length": 1569.0, "completions/mean_length": 557.5, "completions/mean_terminated_length": 557.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3466855548322201, "epoch": 0.01103, "frac_reward_zero_std": 0.25, "grad_norm": 0.06890179961919785, "kl": 0.3582201935350895, "learning_rate": 8.118177868581349e-06, "loss": -0.0017, "num_tokens": 23688515.0, "reward": 0.778113603591919, "reward_std": 0.46285542845726013, "rewards/rollout_reward_func/mean": 0.778113603591919, "rewards/rollout_reward_func/std": 0.5516242384910583, "sampling/importance_sampling_ratio/max": 1.0656832456588745, "sampling/importance_sampling_ratio/mean": 0.9943398833274841, "sampling/importance_sampling_ratio/min": 0.8522886633872986, "sampling/sampling_logp_difference/max": 0.15984535217285156, "sampling/sampling_logp_difference/mean": 0.007735740393400192, "step": 1103, "step_time": 9.525176748997183 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.348053976893425, "epoch": 0.01104, "grad_norm": 0.0737963542342186, "kl": 0.35923378355801105, "learning_rate": 8.102675401914043e-06, "loss": -0.0014, "step": 1104, "step_time": 6.303723675000583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1256.0, "completions/max_terminated_length": 1256.0, "completions/mean_length": 241.65625, "completions/mean_terminated_length": 241.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.23586331214755774, "epoch": 0.01105, "frac_reward_zero_std": 0.0, "grad_norm": 0.10389025509357452, "kl": 0.34961854107677937, "learning_rate": 8.087328081127851e-06, "loss": -0.0014, "num_tokens": 23723019.0, "reward": 0.705235481262207, "reward_std": 0.5470653772354126, "rewards/rollout_reward_func/mean": 0.705235481262207, "rewards/rollout_reward_func/std": 0.59682297706604, "sampling/importance_sampling_ratio/max": 1.119697093963623, "sampling/importance_sampling_ratio/mean": 1.0028603076934814, "sampling/importance_sampling_ratio/min": 0.8870294094085693, "sampling/sampling_logp_difference/max": 0.11307334899902344, "sampling/sampling_logp_difference/mean": 0.00494034867733717, "step": 1105, "step_time": 8.098071529000663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.23551632091403008, "epoch": 0.01106, "grad_norm": 0.10729881376028061, "kl": 0.35357876494526863, "learning_rate": 8.072136024847476e-06, "loss": -0.0013, "step": 1106, "step_time": 4.4638285969995195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1535.0, "completions/max_terminated_length": 1535.0, "completions/mean_length": 465.21875, "completions/mean_terminated_length": 465.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3455664925277233, "epoch": 0.01107, "frac_reward_zero_std": 0.0, "grad_norm": 0.10632126778364182, "kl": 1.4897930808365345, "learning_rate": 8.057099350497533e-06, "loss": -0.0014, "num_tokens": 23766449.0, "reward": 0.6949639320373535, "reward_std": 0.7425845265388489, "rewards/rollout_reward_func/mean": 0.6949639320373535, "rewards/rollout_reward_func/std": 0.7203238606452942, "sampling/importance_sampling_ratio/max": 1.1150777339935303, "sampling/importance_sampling_ratio/mean": 1.0036214590072632, "sampling/importance_sampling_ratio/min": 0.9007224440574646, "sampling/sampling_logp_difference/max": 0.10547375679016113, "sampling/sampling_logp_difference/mean": 0.003631938248872757, "step": 1107, "step_time": 9.158674785996482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3489113450050354, "epoch": 0.01108, "grad_norm": 0.09329847246408463, "kl": 1.4343798905611038, "learning_rate": 8.04221817430163e-06, "loss": -0.0016, "step": 1108, "step_time": 5.059047361000921 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1632.0, "completions/max_terminated_length": 1632.0, "completions/mean_length": 504.125, "completions/mean_terminated_length": 504.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28911625780165195, "epoch": 0.01109, "frac_reward_zero_std": 0.0, "grad_norm": 0.036176394671201706, "kl": 0.4100327081978321, "learning_rate": 8.02749261128149e-06, "loss": -0.0032, "num_tokens": 23810934.0, "reward": 0.7706704139709473, "reward_std": 0.5026500821113586, "rewards/rollout_reward_func/mean": 0.7706704139709473, "rewards/rollout_reward_func/std": 0.5070146322250366, "sampling/importance_sampling_ratio/max": 1.0833557844161987, "sampling/importance_sampling_ratio/mean": 1.000317096710205, "sampling/importance_sampling_ratio/min": 0.857185959815979, "sampling/sampling_logp_difference/max": 0.08746051788330078, "sampling/sampling_logp_difference/mean": 0.005845062434673309, "step": 1109, "step_time": 9.61668101399664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.2886505797505379, "epoch": 0.0111, "grad_norm": 0.03486807644367218, "kl": 0.4092823136597872, "learning_rate": 8.012922775256035e-06, "loss": -0.0032, "step": 1110, "step_time": 5.817275495999638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1562.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 463.625, "completions/mean_terminated_length": 463.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3956603370606899, "epoch": 0.01111, "frac_reward_zero_std": 0.0, "grad_norm": 0.08827869594097137, "kl": 0.43754699639976025, "learning_rate": 7.998508778840524e-06, "loss": -0.0029, "num_tokens": 23854942.0, "reward": 0.6905722618103027, "reward_std": 0.6713078022003174, "rewards/rollout_reward_func/mean": 0.6905722618103027, "rewards/rollout_reward_func/std": 0.6778855323791504, "sampling/importance_sampling_ratio/max": 1.037294626235962, "sampling/importance_sampling_ratio/mean": 0.99266117811203, "sampling/importance_sampling_ratio/min": 0.9080671668052673, "sampling/sampling_logp_difference/max": 0.11524128913879395, "sampling/sampling_logp_difference/mean": 0.006039340980350971, "step": 1111, "step_time": 10.01416260300175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3962757894769311, "epoch": 0.01112, "grad_norm": 0.08864501118659973, "kl": 0.43735779635608196, "learning_rate": 7.98425073344568e-06, "loss": -0.0029, "step": 1112, "step_time": 5.171194235002986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2378.0, "completions/max_terminated_length": 2378.0, "completions/mean_length": 572.0625, "completions/mean_terminated_length": 572.0625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31462214584462345, "epoch": 0.01113, "frac_reward_zero_std": 0.0, "grad_norm": 0.14527414739131927, "kl": 0.5243429746478796, "learning_rate": 7.970148749276827e-06, "loss": 0.0157, "num_tokens": 23902198.0, "reward": 0.6179721355438232, "reward_std": 0.6350964903831482, "rewards/rollout_reward_func/mean": 0.6179721355438232, "rewards/rollout_reward_func/std": 0.6104446053504944, "sampling/importance_sampling_ratio/max": 1.1121851205825806, "sampling/importance_sampling_ratio/mean": 0.9983702898025513, "sampling/importance_sampling_ratio/min": 0.9264326095581055, "sampling/sampling_logp_difference/max": 0.11580324172973633, "sampling/sampling_logp_difference/mean": 0.006950095295906067, "step": 1113, "step_time": 11.536465381001108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3172421511262655, "epoch": 0.01114, "grad_norm": 0.13731050491333008, "kl": 0.5262057539075613, "learning_rate": 7.956202935333038e-06, "loss": 0.0153, "step": 1114, "step_time": 6.4944587879999744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1578.0, "completions/max_terminated_length": 1578.0, "completions/mean_length": 462.40625, "completions/mean_terminated_length": 462.40625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3497121296823025, "epoch": 0.01115, "frac_reward_zero_std": 0.25, "grad_norm": 0.1501230150461197, "kl": 1.0113289654254913, "learning_rate": 7.94241339940629e-06, "loss": 0.0025, "num_tokens": 23945888.0, "reward": 0.7439275979995728, "reward_std": 0.47184860706329346, "rewards/rollout_reward_func/mean": 0.7439275979995728, "rewards/rollout_reward_func/std": 0.5760960578918457, "sampling/importance_sampling_ratio/max": 1.1279487609863281, "sampling/importance_sampling_ratio/mean": 0.9829685688018799, "sampling/importance_sampling_ratio/min": 0.7017344236373901, "sampling/sampling_logp_difference/max": 0.21410846710205078, "sampling/sampling_logp_difference/mean": 0.012486808933317661, "step": 1115, "step_time": 9.238046224003483 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.3538783248513937, "epoch": 0.01116, "grad_norm": 0.13367481529712677, "kl": 0.9355197139084339, "learning_rate": 7.928780248080636e-06, "loss": 0.0018, "step": 1116, "step_time": 5.065771087998655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1229.0, "completions/max_terminated_length": 1229.0, "completions/mean_length": 275.09375, "completions/mean_terminated_length": 275.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.259127545170486, "epoch": 0.01117, "frac_reward_zero_std": 0.25, "grad_norm": 0.04063013941049576, "kl": 0.35565802082419395, "learning_rate": 7.915303586731381e-06, "loss": -0.0061, "num_tokens": 23981626.0, "reward": 0.8357138633728027, "reward_std": 0.3985442519187927, "rewards/rollout_reward_func/mean": 0.8357138633728027, "rewards/rollout_reward_func/std": 0.46219372749328613, "sampling/importance_sampling_ratio/max": 1.044189453125, "sampling/importance_sampling_ratio/mean": 0.9973543882369995, "sampling/importance_sampling_ratio/min": 0.8749157786369324, "sampling/sampling_logp_difference/max": 0.1337132453918457, "sampling/sampling_logp_difference/mean": 0.0052352119237184525, "step": 1117, "step_time": 8.907419493001726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.2601444497704506, "epoch": 0.01118, "grad_norm": 0.04179171845316887, "kl": 0.3516540266573429, "learning_rate": 7.901983519524258e-06, "loss": -0.0062, "step": 1118, "step_time": 4.325815238002178 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2378.0, "completions/max_terminated_length": 2378.0, "completions/mean_length": 515.78125, "completions/mean_terminated_length": 515.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.30329800955951214, "epoch": 0.01119, "frac_reward_zero_std": 0.25, "grad_norm": 0.09697119146585464, "kl": 0.4624577686190605, "learning_rate": 7.888820149414636e-06, "loss": 0.0107, "num_tokens": 24026552.0, "reward": 0.7743051052093506, "reward_std": 0.3684181869029999, "rewards/rollout_reward_func/mean": 0.7743051052093506, "rewards/rollout_reward_func/std": 0.4983366131782532, "sampling/importance_sampling_ratio/max": 1.081939935684204, "sampling/importance_sampling_ratio/mean": 0.994789183139801, "sampling/importance_sampling_ratio/min": 0.8549216985702515, "sampling/sampling_logp_difference/max": 0.11810874938964844, "sampling/sampling_logp_difference/mean": 0.007184871472418308, "step": 1119, "step_time": 11.656872747998932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3072546888142824, "epoch": 0.0112, "grad_norm": 0.09972427040338516, "kl": 0.4557904042303562, "learning_rate": 7.87581357814672e-06, "loss": 0.0102, "step": 1120, "step_time": 6.590524548002577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 779.0, "completions/max_terminated_length": 779.0, "completions/mean_length": 337.84375, "completions/mean_terminated_length": 337.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3466086685657501, "epoch": 0.01121, "frac_reward_zero_std": 0.25, "grad_norm": 0.16465190052986145, "kl": 0.4577211104333401, "learning_rate": 7.862963906252762e-06, "loss": 0.0054, "num_tokens": 24065638.0, "reward": 0.7124221920967102, "reward_std": 0.5101326107978821, "rewards/rollout_reward_func/mean": 0.7124221920967102, "rewards/rollout_reward_func/std": 0.6393809914588928, "sampling/importance_sampling_ratio/max": 1.1130194664001465, "sampling/importance_sampling_ratio/mean": 0.9950488209724426, "sampling/importance_sampling_ratio/min": 0.8807254433631897, "sampling/sampling_logp_difference/max": 0.10705041885375977, "sampling/sampling_logp_difference/mean": 0.006513469852507114, "step": 1121, "step_time": 7.035162706997653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3480172511190176, "epoch": 0.01122, "grad_norm": 0.1651795357465744, "kl": 0.4531768411397934, "learning_rate": 7.850271233052278e-06, "loss": 0.0054, "step": 1122, "step_time": 3.6536715990023367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1587.0, "completions/max_terminated_length": 1587.0, "completions/mean_length": 350.25, "completions/mean_terminated_length": 336.3870849609375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4513383600860834, "epoch": 0.01123, "frac_reward_zero_std": 0.0, "grad_norm": 0.24597999453544617, "kl": 0.41012105345726013, "learning_rate": 7.837735656651293e-06, "loss": 0.0169, "num_tokens": 24104419.0, "reward": 0.7695952653884888, "reward_std": 0.5051674842834473, "rewards/rollout_reward_func/mean": 0.7695952653884888, "rewards/rollout_reward_func/std": 0.5088249444961548, "sampling/importance_sampling_ratio/max": 1.1259227991104126, "sampling/importance_sampling_ratio/mean": 0.9894092082977295, "sampling/importance_sampling_ratio/min": 0.5854288935661316, "sampling/sampling_logp_difference/max": 0.17869257926940918, "sampling/sampling_logp_difference/mean": 0.011707672849297523, "step": 1123, "step_time": 9.274540114000047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0034722222480922937, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034722222480922937, "entropy": 0.4513647351413965, "epoch": 0.01124, "grad_norm": 0.18913695216178894, "kl": 0.40838638320565224, "learning_rate": 7.825357273941582e-06, "loss": 0.016, "step": 1124, "step_time": 6.20524154099985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2146.0, "completions/max_terminated_length": 2146.0, "completions/mean_length": 461.8125, "completions/mean_terminated_length": 461.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.29316053725779057, "epoch": 0.01125, "frac_reward_zero_std": 0.25, "grad_norm": 0.04714103415608406, "kl": 1.6985696405172348, "learning_rate": 7.813136180599901e-06, "loss": 0.0081, "num_tokens": 24146905.0, "reward": 0.7113009691238403, "reward_std": 0.5092836618423462, "rewards/rollout_reward_func/mean": 0.7113009691238403, "rewards/rollout_reward_func/std": 0.5885919332504272, "sampling/importance_sampling_ratio/max": 1.0495432615280151, "sampling/importance_sampling_ratio/mean": 0.9995229244232178, "sampling/importance_sampling_ratio/min": 0.9416017532348633, "sampling/sampling_logp_difference/max": 0.0610349178314209, "sampling/sampling_logp_difference/mean": 0.003261422738432884, "step": 1125, "step_time": 10.84023680700011 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.2938273958861828, "epoch": 0.01126, "grad_norm": 0.047428715974092484, "kl": 1.6990451775491238, "learning_rate": 7.801072471087283e-06, "loss": 0.0083, "step": 1126, "step_time": 6.070282684000631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1220.0, "completions/max_terminated_length": 1220.0, "completions/mean_length": 341.125, "completions/mean_terminated_length": 341.125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.30266017094254494, "epoch": 0.01127, "frac_reward_zero_std": 0.25, "grad_norm": 0.05050059035420418, "kl": 0.2983066625893116, "learning_rate": 7.789166238648267e-06, "loss": -0.0053, "num_tokens": 24185601.0, "reward": 0.7401952743530273, "reward_std": 0.47761601209640503, "rewards/rollout_reward_func/mean": 0.7401952743530273, "rewards/rollout_reward_func/std": 0.5802157521247864, "sampling/importance_sampling_ratio/max": 1.1134463548660278, "sampling/importance_sampling_ratio/mean": 0.9970656037330627, "sampling/importance_sampling_ratio/min": 0.921802282333374, "sampling/sampling_logp_difference/max": 0.1074986457824707, "sampling/sampling_logp_difference/mean": 0.005572512745857239, "step": 1127, "step_time": 7.945490248997885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3034000378102064, "epoch": 0.01128, "grad_norm": 0.04881564527750015, "kl": 0.2985796220600605, "learning_rate": 7.77741757531021e-06, "loss": -0.0053, "step": 1128, "step_time": 4.375140315001772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2434.0, "completions/max_terminated_length": 2434.0, "completions/mean_length": 503.21875, "completions/mean_terminated_length": 499.7419128417969, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4617800358682871, "epoch": 0.01129, "frac_reward_zero_std": 0.5, "grad_norm": 0.03503274545073509, "kl": 0.3828548453748226, "learning_rate": 7.765826571882564e-06, "loss": 0.0076, "num_tokens": 24229617.0, "reward": 0.9067748785018921, "reward_std": 0.26368048787117004, "rewards/rollout_reward_func/mean": 0.9067748785018921, "rewards/rollout_reward_func/std": 0.38648107647895813, "sampling/importance_sampling_ratio/max": 1.0373637676239014, "sampling/importance_sampling_ratio/mean": 0.9789332151412964, "sampling/importance_sampling_ratio/min": 0.6851309537887573, "sampling/sampling_logp_difference/max": 0.31172895431518555, "sampling/sampling_logp_difference/mean": 0.014222885482013226, "step": 1129, "step_time": 11.66083893400355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.46312381885945797, "epoch": 0.0113, "grad_norm": 0.03588174283504486, "kl": 0.3816947042942047, "learning_rate": 7.754393317956164e-06, "loss": 0.0077, "step": 1130, "step_time": 7.000631122995401 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1659.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 537.90625, "completions/mean_terminated_length": 537.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3754165153950453, "epoch": 0.01131, "frac_reward_zero_std": 0.25, "grad_norm": 0.04776426777243614, "kl": 0.47674431279301643, "learning_rate": 7.743117901902556e-06, "loss": 0.0084, "num_tokens": 24275147.0, "reward": 0.8080776929855347, "reward_std": 0.39968734979629517, "rewards/rollout_reward_func/mean": 0.8080776929855347, "rewards/rollout_reward_func/std": 0.47266000509262085, "sampling/importance_sampling_ratio/max": 1.1152911186218262, "sampling/importance_sampling_ratio/mean": 0.9695532917976379, "sampling/importance_sampling_ratio/min": 2.8471420100012468e-12, "sampling/sampling_logp_difference/max": 25.983741760253906, "sampling/sampling_logp_difference/mean": 0.17121270298957825, "step": 1131, "step_time": 10.381145446999653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.37567635998129845, "epoch": 0.01132, "grad_norm": 0.04515437036752701, "kl": 0.48260902613401413, "learning_rate": 7.732000410873295e-06, "loss": 0.0082, "step": 1132, "step_time": 5.285319206999702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2486.0, "completions/max_terminated_length": 2486.0, "completions/mean_length": 573.59375, "completions/mean_terminated_length": 573.59375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36257128044962883, "epoch": 0.01133, "frac_reward_zero_std": 0.0, "grad_norm": 0.09790684282779694, "kl": 0.4099868107587099, "learning_rate": 7.721040930799285e-06, "loss": 0.0032, "num_tokens": 24322180.0, "reward": 0.7147722840309143, "reward_std": 0.5582202672958374, "rewards/rollout_reward_func/mean": 0.7147722840309143, "rewards/rollout_reward_func/std": 0.5798343420028687, "sampling/importance_sampling_ratio/max": 1.0951781272888184, "sampling/importance_sampling_ratio/mean": 0.9965794086456299, "sampling/importance_sampling_ratio/min": 0.9041699767112732, "sampling/sampling_logp_difference/max": 0.10117721557617188, "sampling/sampling_logp_difference/mean": 0.007367363199591637, "step": 1133, "step_time": 11.713055291002092 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3665776140987873, "epoch": 0.01134, "grad_norm": 0.10543898493051529, "kl": 0.40488263219594955, "learning_rate": 7.710239546390101e-06, "loss": 0.0027, "step": 1134, "step_time": 6.603560420997383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1549.0, "completions/max_terminated_length": 1549.0, "completions/mean_length": 647.46875, "completions/mean_terminated_length": 647.46875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.41333395428955555, "epoch": 0.01135, "frac_reward_zero_std": 0.5, "grad_norm": 0.1439160406589508, "kl": 0.3305730130523443, "learning_rate": 7.699596341133354e-06, "loss": 0.0137, "num_tokens": 24372183.0, "reward": 0.8458515405654907, "reward_std": 0.3561742603778839, "rewards/rollout_reward_func/mean": 0.8458515405654907, "rewards/rollout_reward_func/std": 0.5064374208450317, "sampling/importance_sampling_ratio/max": 1.1258875131607056, "sampling/importance_sampling_ratio/mean": 0.9736686944961548, "sampling/importance_sampling_ratio/min": 5.9990408772137016e-05, "sampling/sampling_logp_difference/max": 9.496773719787598, "sampling/sampling_logp_difference/mean": 0.05419035255908966, "step": 1135, "step_time": 9.959118083997964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0024038462433964014, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024038462433964014, "entropy": 0.4152236469089985, "epoch": 0.01136, "grad_norm": 0.11348040401935577, "kl": 0.3289421256631613, "learning_rate": 7.689111397294023e-06, "loss": 0.0134, "step": 1136, "step_time": 5.144184610997399 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2164.0, "completions/max_terminated_length": 2164.0, "completions/mean_length": 524.3125, "completions/mean_terminated_length": 524.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.32301198691129684, "epoch": 0.01137, "frac_reward_zero_std": 0.25, "grad_norm": 0.07827978581190109, "kl": 0.48324188962578773, "learning_rate": 7.678784795913838e-06, "loss": 0.0016, "num_tokens": 24417358.0, "reward": 0.8345206379890442, "reward_std": 0.33856379985809326, "rewards/rollout_reward_func/mean": 0.8345206379890442, "rewards/rollout_reward_func/std": 0.39114731550216675, "sampling/importance_sampling_ratio/max": 1.1052523851394653, "sampling/importance_sampling_ratio/mean": 1.0108765363693237, "sampling/importance_sampling_ratio/min": 0.9897467494010925, "sampling/sampling_logp_difference/max": 0.11550116539001465, "sampling/sampling_logp_difference/mean": 0.00583013566210866, "step": 1137, "step_time": 11.452536554001199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3250530809164047, "epoch": 0.01138, "grad_norm": 0.08922392129898071, "kl": 0.4832633100450039, "learning_rate": 7.66861661681064e-06, "loss": 0.0018, "step": 1138, "step_time": 6.595624616002169 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1533.0, "completions/max_terminated_length": 1533.0, "completions/mean_length": 467.625, "completions/mean_terminated_length": 467.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3210211433470249, "epoch": 0.01139, "frac_reward_zero_std": 0.25, "grad_norm": 0.1755581945180893, "kl": 0.41731391474604607, "learning_rate": 7.658606938577774e-06, "loss": -0.003, "num_tokens": 24461063.0, "reward": 0.7709963917732239, "reward_std": 0.41983598470687866, "rewards/rollout_reward_func/mean": 0.7709963917732239, "rewards/rollout_reward_func/std": 0.5039772391319275, "sampling/importance_sampling_ratio/max": 1.075091004371643, "sampling/importance_sampling_ratio/mean": 0.9999426007270813, "sampling/importance_sampling_ratio/min": 0.891261100769043, "sampling/sampling_logp_difference/max": 0.11864995956420898, "sampling/sampling_logp_difference/mean": 0.0070459842681884766, "step": 1139, "step_time": 9.186605396997038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3199873799458146, "epoch": 0.0114, "grad_norm": 0.15770502388477325, "kl": 0.43145208433270454, "learning_rate": 7.648755838583477e-06, "loss": -0.0037, "step": 1140, "step_time": 5.043244180000329 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1632.0, "completions/max_terminated_length": 1632.0, "completions/mean_length": 391.84375, "completions/mean_terminated_length": 391.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3270619120448828, "epoch": 0.01141, "frac_reward_zero_std": 0.0, "grad_norm": 0.0648997575044632, "kl": 0.4270226117223501, "learning_rate": 7.63906339297028e-06, "loss": -0.0021, "num_tokens": 24501681.0, "reward": 0.7353314161300659, "reward_std": 0.5328948497772217, "rewards/rollout_reward_func/mean": 0.7353314161300659, "rewards/rollout_reward_func/std": 0.5306755900382996, "sampling/importance_sampling_ratio/max": 1.1223039627075195, "sampling/importance_sampling_ratio/mean": 1.0108457803726196, "sampling/importance_sampling_ratio/min": 0.9657288789749146, "sampling/sampling_logp_difference/max": 0.11494207382202148, "sampling/sampling_logp_difference/mean": 0.006738423369824886, "step": 1141, "step_time": 9.399306013998284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3260287158191204, "epoch": 0.01142, "grad_norm": 0.06429152190685272, "kl": 0.4273950904607773, "learning_rate": 7.62952967665442e-06, "loss": -0.0023, "step": 1142, "step_time": 5.192483634999007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1542.0, "completions/max_terminated_length": 1542.0, "completions/mean_length": 400.53125, "completions/mean_terminated_length": 400.53125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3204123489558697, "epoch": 0.01143, "frac_reward_zero_std": 0.25, "grad_norm": 0.03844049945473671, "kl": 0.3415314666926861, "learning_rate": 7.620154763325258e-06, "loss": -0.0016, "num_tokens": 24542702.0, "reward": 0.8716233968734741, "reward_std": 0.36310380697250366, "rewards/rollout_reward_func/mean": 0.8716233968734741, "rewards/rollout_reward_func/std": 0.4283415675163269, "sampling/importance_sampling_ratio/max": 1.0426074266433716, "sampling/importance_sampling_ratio/mean": 0.9980788230895996, "sampling/importance_sampling_ratio/min": 0.9119091629981995, "sampling/sampling_logp_difference/max": 0.08605575561523438, "sampling/sampling_logp_difference/mean": 0.0035251190420240164, "step": 1143, "step_time": 9.068360756999027 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3184078074991703, "epoch": 0.01144, "grad_norm": 0.038407642394304276, "kl": 0.341945543885231, "learning_rate": 7.6109387254447135e-06, "loss": -0.0018, "step": 1144, "step_time": 5.46695606600224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 348.9375, "completions/mean_terminated_length": 348.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.343499225564301, "epoch": 0.01145, "frac_reward_zero_std": 0.25, "grad_norm": 0.094535693526268, "kl": 0.6286270543932915, "learning_rate": 7.601881634246706e-06, "loss": 0.0131, "num_tokens": 24580369.0, "reward": 0.7728464007377625, "reward_std": 0.4299491047859192, "rewards/rollout_reward_func/mean": 0.7728464007377625, "rewards/rollout_reward_func/std": 0.5007884502410889, "sampling/importance_sampling_ratio/max": 1.0490936040878296, "sampling/importance_sampling_ratio/mean": 0.9901045560836792, "sampling/importance_sampling_ratio/min": 0.8003456592559814, "sampling/sampling_logp_difference/max": 0.12222695350646973, "sampling/sampling_logp_difference/mean": 0.006989526096731424, "step": 1145, "step_time": 9.649456780998662 }, { "clip_ratio/high_max": 0.02083333395421505, "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010416666977107525, "entropy": 0.34221680276095867, "epoch": 0.01146, "grad_norm": 0.0873958095908165, "kl": 0.6277336403727531, "learning_rate": 7.592983559736597e-06, "loss": 0.0128, "step": 1146, "step_time": 5.100399388000369 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2450.0, "completions/max_terminated_length": 2450.0, "completions/mean_length": 537.71875, "completions/mean_terminated_length": 537.71875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.5054403878748417, "epoch": 0.01147, "frac_reward_zero_std": 0.0, "grad_norm": 0.49885430932044983, "kl": 0.4563268609344959, "learning_rate": 7.584244570690663e-06, "loss": 0.07, "num_tokens": 24626412.0, "reward": 0.6600159406661987, "reward_std": 0.7214610576629639, "rewards/rollout_reward_func/mean": 0.6600159406661987, "rewards/rollout_reward_func/std": 0.6883727312088013, "sampling/importance_sampling_ratio/max": 1.1204512119293213, "sampling/importance_sampling_ratio/mean": 1.0002918243408203, "sampling/importance_sampling_ratio/min": 0.8670933246612549, "sampling/sampling_logp_difference/max": 0.12488031387329102, "sampling/sampling_logp_difference/mean": 0.011698611080646515, "step": 1147, "step_time": 11.703723772001467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.5080006681382656, "epoch": 0.01148, "grad_norm": 0.4709063172340393, "kl": 0.45466709323227406, "learning_rate": 7.57566473465554e-06, "loss": 0.0674, "step": 1148, "step_time": 6.468425949999073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1722.0, "completions/max_terminated_length": 1722.0, "completions/mean_length": 574.65625, "completions/mean_terminated_length": 574.65625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.40308702923357487, "epoch": 0.01149, "frac_reward_zero_std": 0.0, "grad_norm": 1.3191295862197876, "kl": 4.881042633205652, "learning_rate": 7.567244117947729e-06, "loss": 0.0363, "num_tokens": 24673667.0, "reward": 0.6555825471878052, "reward_std": 0.6234310865402222, "rewards/rollout_reward_func/mean": 0.6555825471878052, "rewards/rollout_reward_func/std": 0.6468570232391357, "sampling/importance_sampling_ratio/max": 1.1268597841262817, "sampling/importance_sampling_ratio/mean": 1.000711441040039, "sampling/importance_sampling_ratio/min": 0.883019208908081, "sampling/sampling_logp_difference/max": 0.1248941421508789, "sampling/sampling_logp_difference/mean": 0.009815078228712082, "step": 1149, "step_time": 9.738100881002538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4019896648824215, "epoch": 0.0115, "grad_norm": 0.6335217356681824, "kl": 3.162489350885153, "learning_rate": 7.558982785653061e-06, "loss": 0.0295, "step": 1150, "step_time": 5.385132248997252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 433.8125, "completions/mean_terminated_length": 433.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3272371534258127, "epoch": 0.01151, "frac_reward_zero_std": 0.0, "grad_norm": 0.06255776435136795, "kl": 0.36241549998521805, "learning_rate": 7.550880801626209e-06, "loss": -0.0038, "num_tokens": 24715933.0, "reward": 0.8380983471870422, "reward_std": 0.45792707800865173, "rewards/rollout_reward_func/mean": 0.8380983471870422, "rewards/rollout_reward_func/std": 0.45756009221076965, "sampling/importance_sampling_ratio/max": 1.1843621730804443, "sampling/importance_sampling_ratio/mean": 1.0049350261688232, "sampling/importance_sampling_ratio/min": 0.9227225184440613, "sampling/sampling_logp_difference/max": 0.1133120059967041, "sampling/sampling_logp_difference/mean": 0.006813920568674803, "step": 1151, "step_time": 9.514002325000547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.33039759285748005, "epoch": 0.01152, "grad_norm": 0.06270521134138107, "kl": 0.3584509398788214, "learning_rate": 7.542938228490185e-06, "loss": -0.004, "step": 1152, "step_time": 5.583505146003517 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 651.25, "completions/mean_terminated_length": 651.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37172550708055496, "epoch": 0.01153, "frac_reward_zero_std": 0.0, "grad_norm": 0.11717294156551361, "kl": 0.5514911748468876, "learning_rate": 7.535155127635861e-06, "loss": -0.0082, "num_tokens": 24765604.0, "reward": 0.7810662984848022, "reward_std": 0.619238018989563, "rewards/rollout_reward_func/mean": 0.7810662984848022, "rewards/rollout_reward_func/std": 0.6033167243003845, "sampling/importance_sampling_ratio/max": 1.1171214580535889, "sampling/importance_sampling_ratio/mean": 0.9816321134567261, "sampling/importance_sampling_ratio/min": 0.8833318948745728, "sampling/sampling_logp_difference/max": 0.10483384132385254, "sampling/sampling_logp_difference/mean": 0.007960072718560696, "step": 1153, "step_time": 9.803957105001246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3702952153980732, "epoch": 0.01154, "grad_norm": 0.1146160140633583, "kl": 0.5525416620075703, "learning_rate": 7.5275315592214925e-06, "loss": -0.0084, "step": 1154, "step_time": 5.266063650000433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 497.34375, "completions/mean_terminated_length": 497.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.33066368848085403, "epoch": 0.01155, "frac_reward_zero_std": 0.25, "grad_norm": 0.08582253754138947, "kl": 0.334420969709754, "learning_rate": 7.520067582172255e-06, "loss": 0.0032, "num_tokens": 24810152.0, "reward": 0.8030634522438049, "reward_std": 0.40909188985824585, "rewards/rollout_reward_func/mean": 0.8030634522438049, "rewards/rollout_reward_func/std": 0.48676785826683044, "sampling/importance_sampling_ratio/max": 1.0198776721954346, "sampling/importance_sampling_ratio/mean": 0.9837026596069336, "sampling/importance_sampling_ratio/min": 0.8457732796669006, "sampling/sampling_logp_difference/max": 0.11307287216186523, "sampling/sampling_logp_difference/mean": 0.005949478596448898, "step": 1155, "step_time": 9.167853013997956 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3327655605971813, "epoch": 0.01156, "grad_norm": 0.08359839767217636, "kl": 0.33349416218698025, "learning_rate": 7.512763254179786e-06, "loss": 0.003, "step": 1156, "step_time": 4.97653483999602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 580.34375, "completions/mean_terminated_length": 580.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.36042869836091995, "epoch": 0.01157, "frac_reward_zero_std": 0.5, "grad_norm": 0.08610866963863373, "kl": 0.39452768862247467, "learning_rate": 7.505618631701744e-06, "loss": 0.0043, "num_tokens": 24857613.0, "reward": 0.8139506578445435, "reward_std": 0.3155354857444763, "rewards/rollout_reward_func/mean": 0.8139506578445435, "rewards/rollout_reward_func/std": 0.5306194424629211, "sampling/importance_sampling_ratio/max": 1.1637020111083984, "sampling/importance_sampling_ratio/mean": 1.0047640800476074, "sampling/importance_sampling_ratio/min": 0.8812686800956726, "sampling/sampling_logp_difference/max": 0.14660859107971191, "sampling/sampling_logp_difference/mean": 0.010805709287524223, "step": 1157, "step_time": 10.182341727002495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3605843707919121, "epoch": 0.01158, "grad_norm": 0.08638034760951996, "kl": 0.3902804497629404, "learning_rate": 7.498633769961366e-06, "loss": 0.0041, "step": 1158, "step_time": 5.281476000998737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1605.0, "completions/max_terminated_length": 1605.0, "completions/mean_length": 355.28125, "completions/mean_terminated_length": 355.28125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3174554519355297, "epoch": 0.01159, "frac_reward_zero_std": 0.0, "grad_norm": 0.07428514212369919, "kl": 0.45518265292048454, "learning_rate": 7.491808722947044e-06, "loss": 0.007, "num_tokens": 24896073.0, "reward": 0.7795381546020508, "reward_std": 0.5406589508056641, "rewards/rollout_reward_func/mean": 0.7795381546020508, "rewards/rollout_reward_func/std": 0.5501977205276489, "sampling/importance_sampling_ratio/max": 1.119655966758728, "sampling/importance_sampling_ratio/mean": 1.0135414600372314, "sampling/importance_sampling_ratio/min": 0.966796875, "sampling/sampling_logp_difference/max": 0.11279702186584473, "sampling/sampling_logp_difference/mean": 0.006927589885890484, "step": 1159, "step_time": 9.666310963999422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3179806601256132, "epoch": 0.0116, "grad_norm": 0.07478157430887222, "kl": 0.4553162641823292, "learning_rate": 7.4851435434119034e-06, "loss": 0.0071, "step": 1160, "step_time": 5.186712382997939 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2308.0, "completions/max_terminated_length": 2308.0, "completions/mean_length": 565.875, "completions/mean_terminated_length": 565.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.38078197091817856, "epoch": 0.01161, "frac_reward_zero_std": 0.0, "grad_norm": 0.08723516017198563, "kl": 0.8313598483800888, "learning_rate": 7.478638282873411e-06, "loss": 0.0039, "num_tokens": 24943958.0, "reward": 0.6845481991767883, "reward_std": 0.6037174463272095, "rewards/rollout_reward_func/mean": 0.6845481991767883, "rewards/rollout_reward_func/std": 0.6393876671791077, "sampling/importance_sampling_ratio/max": 1.1046189069747925, "sampling/importance_sampling_ratio/mean": 1.0077420473098755, "sampling/importance_sampling_ratio/min": 0.9257749319076538, "sampling/sampling_logp_difference/max": 0.09014558792114258, "sampling/sampling_logp_difference/mean": 0.005831493064761162, "step": 1161, "step_time": 11.665759122002783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3822104763239622, "epoch": 0.01162, "grad_norm": 0.08698359876871109, "kl": 0.8314863257110119, "learning_rate": 7.472292991612947e-06, "loss": 0.0036, "step": 1162, "step_time": 6.4486145279988705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1560.0, "completions/max_terminated_length": 1560.0, "completions/mean_length": 674.1875, "completions/mean_terminated_length": 674.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3444922901690006, "epoch": 0.01163, "frac_reward_zero_std": 0.25, "grad_norm": 0.16683101654052734, "kl": 0.5297840945422649, "learning_rate": 7.466107718675448e-06, "loss": 0.0014, "num_tokens": 24995472.0, "reward": 0.7835754156112671, "reward_std": 0.5314303636550903, "rewards/rollout_reward_func/mean": 0.7835754156112671, "rewards/rollout_reward_func/std": 0.5995468497276306, "sampling/importance_sampling_ratio/max": 1.0973947048187256, "sampling/importance_sampling_ratio/mean": 0.9930298328399658, "sampling/importance_sampling_ratio/min": 0.789984941482544, "sampling/sampling_logp_difference/max": 0.23992252349853516, "sampling/sampling_logp_difference/mean": 0.00816485658288002, "step": 1163, "step_time": 9.666728397998668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.34138905443251133, "epoch": 0.01164, "grad_norm": 0.1564626693725586, "kl": 0.5490126498043537, "learning_rate": 7.460082511869015e-06, "loss": 0.0011, "step": 1164, "step_time": 5.723990533000688 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1706.0, "completions/max_terminated_length": 1706.0, "completions/mean_length": 407.375, "completions/mean_terminated_length": 407.375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3658481054008007, "epoch": 0.01165, "frac_reward_zero_std": 0.0, "grad_norm": 0.07881036400794983, "kl": 0.3682403080165386, "learning_rate": 7.45421741776453e-06, "loss": -0.0037, "num_tokens": 25036720.0, "reward": 0.6522878408432007, "reward_std": 0.6832966208457947, "rewards/rollout_reward_func/mean": 0.6522878408432007, "rewards/rollout_reward_func/std": 0.6962034702301025, "sampling/importance_sampling_ratio/max": 1.0833419561386108, "sampling/importance_sampling_ratio/mean": 0.997648298740387, "sampling/importance_sampling_ratio/min": 0.9080222249031067, "sampling/sampling_logp_difference/max": 0.11632895469665527, "sampling/sampling_logp_difference/mean": 0.005803881213068962, "step": 1165, "step_time": 9.875886279001861 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.36582047306001186, "epoch": 0.01166, "grad_norm": 0.07941430062055588, "kl": 0.3683222811669111, "learning_rate": 7.448512481695321e-06, "loss": -0.0039, "step": 1166, "step_time": 5.242720070997166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1553.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 457.4375, "completions/mean_terminated_length": 457.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.319552356377244, "epoch": 0.01167, "frac_reward_zero_std": 0.25, "grad_norm": 0.10259630531072617, "kl": 0.3306410647928715, "learning_rate": 7.442967747756794e-06, "loss": -0.0051, "num_tokens": 25080503.0, "reward": 0.8688787817955017, "reward_std": 0.3708668053150177, "rewards/rollout_reward_func/mean": 0.8688787817955017, "rewards/rollout_reward_func/std": 0.43244868516921997, "sampling/importance_sampling_ratio/max": 1.0749949216842651, "sampling/importance_sampling_ratio/mean": 0.9990459084510803, "sampling/importance_sampling_ratio/min": 0.8978480100631714, "sampling/sampling_logp_difference/max": 0.11754512786865234, "sampling/sampling_logp_difference/mean": 0.006572311744093895, "step": 1167, "step_time": 9.468378043002303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 0.3173594754189253, "epoch": 0.01168, "grad_norm": 0.021241329610347748, "kl": 0.3337254226207733, "learning_rate": 7.4375832588061e-06, "loss": -0.0055, "step": 1168, "step_time": 5.192354917004195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 431.1875, "completions/mean_terminated_length": 431.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3876950219273567, "epoch": 0.01169, "frac_reward_zero_std": 0.0, "grad_norm": 0.10672401636838913, "kl": 0.7614333368837833, "learning_rate": 7.432359056461802e-06, "loss": 0.0019, "num_tokens": 25123668.0, "reward": 0.5646090507507324, "reward_std": 0.8135692477226257, "rewards/rollout_reward_func/mean": 0.5646090507507324, "rewards/rollout_reward_func/std": 0.7901650071144104, "sampling/importance_sampling_ratio/max": 1.0680615901947021, "sampling/importance_sampling_ratio/mean": 1.0017054080963135, "sampling/importance_sampling_ratio/min": 0.9201233386993408, "sampling/sampling_logp_difference/max": 0.0836954116821289, "sampling/sampling_logp_difference/mean": 0.006164140999317169, "step": 1169, "step_time": 9.099394136999763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3870028145611286, "epoch": 0.0117, "grad_norm": 0.09597314894199371, "kl": 0.7716177776455879, "learning_rate": 7.427295181103551e-06, "loss": 0.0014, "step": 1170, "step_time": 5.002216003000285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 2409.0, "completions/max_terminated_length": 2409.0, "completions/mean_length": 568.3125, "completions/mean_terminated_length": 562.6451416015625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.786732854321599, "epoch": 0.01171, "frac_reward_zero_std": 0.0, "grad_norm": 0.3056056499481201, "kl": 0.5049581434577703, "learning_rate": 7.422391671871774e-06, "loss": 0.0198, "num_tokens": 25170774.0, "reward": 0.47668373584747314, "reward_std": 0.8700827360153198, "rewards/rollout_reward_func/mean": 0.47668373584747314, "rewards/rollout_reward_func/std": 0.863054633140564, "sampling/importance_sampling_ratio/max": 1.1052134037017822, "sampling/importance_sampling_ratio/mean": 0.9848783016204834, "sampling/importance_sampling_ratio/min": 0.7812561392784119, "sampling/sampling_logp_difference/max": 0.24645519256591797, "sampling/sampling_logp_difference/mean": 0.01377458032220602, "step": 1171, "step_time": 12.371219926999402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.7802801094949245, "epoch": 0.01172, "grad_norm": 0.29505160450935364, "kl": 0.5090091414749622, "learning_rate": 7.41764856666738e-06, "loss": 0.0185, "step": 1172, "step_time": 7.318462279999949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1375.0, "completions/max_terminated_length": 1375.0, "completions/mean_length": 436.3125, "completions/mean_terminated_length": 436.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37769515067338943, "epoch": 0.01173, "frac_reward_zero_std": 0.25, "grad_norm": 0.4327647089958191, "kl": 1.811415035277605, "learning_rate": 7.413065902151452e-06, "loss": 0.0311, "num_tokens": 25213066.0, "reward": 0.8010920286178589, "reward_std": 0.4293597340583801, "rewards/rollout_reward_func/mean": 0.8010920286178589, "rewards/rollout_reward_func/std": 0.4918088912963867, "sampling/importance_sampling_ratio/max": 1.1932493448257446, "sampling/importance_sampling_ratio/mean": 0.9829898476600647, "sampling/importance_sampling_ratio/min": 0.7998616695404053, "sampling/sampling_logp_difference/max": 0.22347235679626465, "sampling/sampling_logp_difference/mean": 0.017263852059841156, "step": 1173, "step_time": 8.982080201996723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004310344811528921, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004310344811528921, "entropy": 0.3793895374983549, "epoch": 0.01174, "grad_norm": 0.37126821279525757, "kl": 1.584910063073039, "learning_rate": 7.408643713744977e-06, "loss": 0.0296, "step": 1174, "step_time": 4.808087663001061 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1497.0, "completions/max_terminated_length": 1497.0, "completions/mean_length": 535.90625, "completions/mean_terminated_length": 535.90625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37378663197159767, "epoch": 0.01175, "frac_reward_zero_std": 0.25, "grad_norm": 0.15154115855693817, "kl": 0.653054516762495, "learning_rate": 7.404382035628565e-06, "loss": 0.0032, "num_tokens": 25259294.0, "reward": 0.7197188138961792, "reward_std": 0.5016232132911682, "rewards/rollout_reward_func/mean": 0.7197188138961792, "rewards/rollout_reward_func/std": 0.6283928155899048, "sampling/importance_sampling_ratio/max": 1.1161943674087524, "sampling/importance_sampling_ratio/mean": 0.9933279156684875, "sampling/importance_sampling_ratio/min": 0.882108211517334, "sampling/sampling_logp_difference/max": 0.10848379135131836, "sampling/sampling_logp_difference/mean": 0.00885913334786892, "step": 1175, "step_time": 9.334431347000645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3755065929144621, "epoch": 0.01176, "grad_norm": 0.14541003108024597, "kl": 0.6545734293758869, "learning_rate": 7.400280900742191e-06, "loss": 0.0029, "step": 1176, "step_time": 5.025940236000679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1337.0, "completions/max_terminated_length": 1337.0, "completions/mean_length": 409.09375, "completions/mean_terminated_length": 409.09375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2971785068511963, "epoch": 0.01177, "frac_reward_zero_std": 0.25, "grad_norm": 0.04340878874063492, "kl": 0.43565150536596775, "learning_rate": 7.39634034078493e-06, "loss": -0.0033, "num_tokens": 25301080.0, "reward": 0.8716025352478027, "reward_std": 0.36316290497779846, "rewards/rollout_reward_func/mean": 0.8716025352478027, "rewards/rollout_reward_func/std": 0.4246032238006592, "sampling/importance_sampling_ratio/max": 1.085216999053955, "sampling/importance_sampling_ratio/mean": 0.9905527830123901, "sampling/importance_sampling_ratio/min": 0.8931243419647217, "sampling/sampling_logp_difference/max": 0.11306285858154297, "sampling/sampling_logp_difference/mean": 0.007253129966557026, "step": 1177, "step_time": 8.493914972999846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.29791107401251793, "epoch": 0.01178, "grad_norm": 0.04333825409412384, "kl": 0.4297677092254162, "learning_rate": 7.392560386214726e-06, "loss": -0.0035, "step": 1178, "step_time": 5.188774753996768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1650.0, "completions/max_terminated_length": 1650.0, "completions/mean_length": 458.21875, "completions/mean_terminated_length": 458.21875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.42224961519241333, "epoch": 0.01179, "frac_reward_zero_std": 0.0, "grad_norm": 0.11872178316116333, "kl": 0.49928259663283825, "learning_rate": 7.388941066248135e-06, "loss": -0.0077, "num_tokens": 25345344.0, "reward": 0.7141381502151489, "reward_std": 0.635505199432373, "rewards/rollout_reward_func/mean": 0.7141381502151489, "rewards/rollout_reward_func/std": 0.63787442445755, "sampling/importance_sampling_ratio/max": 1.0988030433654785, "sampling/importance_sampling_ratio/mean": 0.9937781095504761, "sampling/importance_sampling_ratio/min": 0.8924301266670227, "sampling/sampling_logp_difference/max": 0.1138460636138916, "sampling/sampling_logp_difference/mean": 0.010987157002091408, "step": 1179, "step_time": 9.952101780998419 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4237825348973274, "epoch": 0.0118, "grad_norm": 0.11966004222631454, "kl": 0.49998271092772484, "learning_rate": 7.385482408860134e-06, "loss": -0.0077, "step": 1180, "step_time": 5.197970785000507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1488.0, "completions/max_terminated_length": 1488.0, "completions/mean_length": 413.75, "completions/mean_terminated_length": 413.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3318000789731741, "epoch": 0.01181, "frac_reward_zero_std": 0.25, "grad_norm": 0.1182820126414299, "kl": 0.4073416702449322, "learning_rate": 7.3821844407838635e-06, "loss": 0.0036, "num_tokens": 25386072.0, "reward": 0.6807413101196289, "reward_std": 0.5527632236480713, "rewards/rollout_reward_func/mean": 0.6807413101196289, "rewards/rollout_reward_func/std": 0.6459510326385498, "sampling/importance_sampling_ratio/max": 1.0848298072814941, "sampling/importance_sampling_ratio/mean": 1.002352237701416, "sampling/importance_sampling_ratio/min": 0.9324914216995239, "sampling/sampling_logp_difference/max": 0.08665847778320312, "sampling/sampling_logp_difference/mean": 0.006509007420390844, "step": 1181, "step_time": 9.026157656997384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3331645857542753, "epoch": 0.01182, "grad_norm": 0.11527332663536072, "kl": 0.4065372059121728, "learning_rate": 7.379047187510451e-06, "loss": 0.0034, "step": 1182, "step_time": 4.945097843999974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2423.0, "completions/max_terminated_length": 2423.0, "completions/mean_length": 434.4375, "completions/mean_terminated_length": 434.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.37486014887690544, "epoch": 0.01183, "frac_reward_zero_std": 0.25, "grad_norm": 0.07456258684396744, "kl": 0.573048859834671, "learning_rate": 7.376070673288807e-06, "loss": 0.0, "num_tokens": 25427544.0, "reward": 0.6853972673416138, "reward_std": 0.4966137707233429, "rewards/rollout_reward_func/mean": 0.6853972673416138, "rewards/rollout_reward_func/std": 0.6400430202484131, "sampling/importance_sampling_ratio/max": 1.2586876153945923, "sampling/importance_sampling_ratio/mean": 1.0182963609695435, "sampling/importance_sampling_ratio/min": 0.9643715620040894, "sampling/sampling_logp_difference/max": 0.10622596740722656, "sampling/sampling_logp_difference/mean": 0.008939458057284355, "step": 1183, "step_time": 11.73160191399802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.378239369019866, "epoch": 0.01184, "grad_norm": 0.07560837268829346, "kl": 0.5722689386457205, "learning_rate": 7.373254921125427e-06, "loss": -0.0002, "step": 1184, "step_time": 7.070207234000918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1596.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 364.4375, "completions/mean_terminated_length": 364.4375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.407701276242733, "epoch": 0.01185, "frac_reward_zero_std": 0.0, "grad_norm": 0.13146010041236877, "kl": 0.739354956895113, "learning_rate": 7.3705999527842306e-06, "loss": -0.0023, "num_tokens": 25467139.0, "reward": 0.5631816387176514, "reward_std": 0.8148397207260132, "rewards/rollout_reward_func/mean": 0.5631816387176514, "rewards/rollout_reward_func/std": 0.7924367189407349, "sampling/importance_sampling_ratio/max": 1.1124696731567383, "sampling/importance_sampling_ratio/mean": 1.0071258544921875, "sampling/importance_sampling_ratio/min": 0.9353018999099731, "sampling/sampling_logp_difference/max": 0.11489534378051758, "sampling/sampling_logp_difference/mean": 0.00790510606020689, "step": 1185, "step_time": 9.256674476999251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.4096546098589897, "epoch": 0.01186, "grad_norm": 0.12028548866510391, "kl": 0.772100605070591, "learning_rate": 7.368105788786372e-06, "loss": -0.0026, "step": 1186, "step_time": 5.703054781999526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 567.15625, "completions/mean_terminated_length": 567.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4621211625635624, "epoch": 0.01187, "frac_reward_zero_std": 0.0, "grad_norm": 0.17077255249023438, "kl": 0.5571670606732368, "learning_rate": 7.365772448410103e-06, "loss": -0.0086, "num_tokens": 25515436.0, "reward": 0.5272465944290161, "reward_std": 0.7898602485656738, "rewards/rollout_reward_func/mean": 0.5272465944290161, "rewards/rollout_reward_func/std": 0.7560599446296692, "sampling/importance_sampling_ratio/max": 1.0957906246185303, "sampling/importance_sampling_ratio/mean": 1.0026443004608154, "sampling/importance_sampling_ratio/min": 0.9262242317199707, "sampling/sampling_logp_difference/max": 0.1165778636932373, "sampling/sampling_logp_difference/mean": 0.011055698618292809, "step": 1187, "step_time": 9.702849600000263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.46424475125968456, "epoch": 0.01188, "grad_norm": 0.16208600997924805, "kl": 0.5619240403175354, "learning_rate": 7.363599949690613e-06, "loss": -0.0095, "step": 1188, "step_time": 5.101259330000175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2308.0, "completions/max_terminated_length": 2308.0, "completions/mean_length": 455.875, "completions/mean_terminated_length": 455.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3317800424993038, "epoch": 0.01189, "frac_reward_zero_std": 1.0, "grad_norm": 0.018697699531912804, "kl": 0.4489769656211138, "learning_rate": 7.361588309419886e-06, "loss": 0.0019, "num_tokens": 25557521.0, "reward": 1.0, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.0, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0866703987121582, "sampling/importance_sampling_ratio/mean": 0.9974712133407593, "sampling/importance_sampling_ratio/min": 0.8548961877822876, "sampling/sampling_logp_difference/max": 0.12370538711547852, "sampling/sampling_logp_difference/mean": 0.009155730716884136, "step": 1189, "step_time": 11.287869110001338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3322701957076788, "epoch": 0.0119, "grad_norm": 0.015788713470101357, "kl": 0.43887810967862606, "learning_rate": 7.359737543146584e-06, "loss": 0.0019, "step": 1190, "step_time": 6.398366085999442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2254.0, "completions/max_terminated_length": 2254.0, "completions/mean_length": 492.25, "completions/mean_terminated_length": 492.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4497838504612446, "epoch": 0.01191, "frac_reward_zero_std": 0.0, "grad_norm": 0.2120337188243866, "kl": 1.3441686313599348, "learning_rate": 7.35804766517591e-06, "loss": -0.0008, "num_tokens": 25601199.0, "reward": 0.8082038164138794, "reward_std": 0.542481541633606, "rewards/rollout_reward_func/mean": 0.8082038164138794, "rewards/rollout_reward_func/std": 0.5395398139953613, "sampling/importance_sampling_ratio/max": 1.2041583061218262, "sampling/importance_sampling_ratio/mean": 0.9993816614151001, "sampling/importance_sampling_ratio/min": 0.9040138721466064, "sampling/sampling_logp_difference/max": 0.3056936264038086, "sampling/sampling_logp_difference/mean": 0.01614988222718239, "step": 1191, "step_time": 11.748097432002396 }, { "clip_ratio/high_max": 0.0062500000931322575, "clip_ratio/high_mean": 0.0031250000465661287, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0031250000465661287, "entropy": 0.450321925804019, "epoch": 0.01192, "grad_norm": 0.17566081881523132, "kl": 1.3066012859344482, "learning_rate": 7.356518688569514e-06, "loss": -0.001, "step": 1192, "step_time": 6.324721178998516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2263.0, "completions/max_terminated_length": 2263.0, "completions/mean_length": 651.875, "completions/mean_terminated_length": 651.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.39918383583426476, "epoch": 0.01193, "frac_reward_zero_std": 0.0, "grad_norm": 0.14778709411621094, "kl": 0.5479717031121254, "learning_rate": 7.3551506251453825e-06, "loss": -0.0047, "num_tokens": 25651259.0, "reward": 0.771920382976532, "reward_std": 0.4993020296096802, "rewards/rollout_reward_func/mean": 0.771920382976532, "rewards/rollout_reward_func/std": 0.5030689239501953, "sampling/importance_sampling_ratio/max": 1.0846041440963745, "sampling/importance_sampling_ratio/mean": 0.9898972511291504, "sampling/importance_sampling_ratio/min": 0.8392700552940369, "sampling/sampling_logp_difference/max": 0.15286535024642944, "sampling/sampling_logp_difference/mean": 0.008460406213998795, "step": 1193, "step_time": 12.496336354995947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.39897575974464417, "epoch": 0.01194, "grad_norm": 0.14409348368644714, "kl": 0.5477961078286171, "learning_rate": 7.35394348547774e-06, "loss": -0.0049, "step": 1194, "step_time": 6.555918449001183 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1463.0, "completions/max_terminated_length": 1463.0, "completions/mean_length": 485.15625, "completions/mean_terminated_length": 485.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.38551088236272335, "epoch": 0.01195, "frac_reward_zero_std": 0.0, "grad_norm": 0.377543568611145, "kl": 0.4886476546525955, "learning_rate": 7.352897278896986e-06, "loss": -0.0079, "num_tokens": 25694385.0, "reward": 0.7127167582511902, "reward_std": 0.5830162763595581, "rewards/rollout_reward_func/mean": 0.7127167582511902, "rewards/rollout_reward_func/std": 0.5805530548095703, "sampling/importance_sampling_ratio/max": 1.8064838647842407, "sampling/importance_sampling_ratio/mean": 1.0137369632720947, "sampling/importance_sampling_ratio/min": 0.911780834197998, "sampling/sampling_logp_difference/max": 0.7383561134338379, "sampling/sampling_logp_difference/mean": 0.013489820994436741, "step": 1195, "step_time": 8.967309980998834 }, { "clip_ratio/high_max": 0.009615384973585606, "clip_ratio/high_mean": 0.004807692486792803, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004807692486792803, "entropy": 0.3846013080328703, "epoch": 0.01196, "grad_norm": 0.12900981307029724, "kl": 0.4986220970749855, "learning_rate": 7.352012013489613e-06, "loss": -0.0085, "step": 1196, "step_time": 4.88395886500075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1632.0, "completions/max_terminated_length": 1632.0, "completions/mean_length": 419.8125, "completions/mean_terminated_length": 419.8125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.31127203814685345, "epoch": 0.01197, "frac_reward_zero_std": 0.25, "grad_norm": 0.13149842619895935, "kl": 0.530716298148036, "learning_rate": 7.351287696098139e-06, "loss": 0.0036, "num_tokens": 25735142.0, "reward": 0.8126294612884521, "reward_std": 0.4509439766407013, "rewards/rollout_reward_func/mean": 0.8126294612884521, "rewards/rollout_reward_func/std": 0.5302748680114746, "sampling/importance_sampling_ratio/max": 1.0321441888809204, "sampling/importance_sampling_ratio/mean": 0.9850649833679199, "sampling/importance_sampling_ratio/min": 0.8774042129516602, "sampling/sampling_logp_difference/max": 0.12473392486572266, "sampling/sampling_logp_difference/mean": 0.006448517553508282, "step": 1197, "step_time": 9.471551407001243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.30812165327370167, "epoch": 0.01198, "grad_norm": 0.11637952923774719, "kl": 0.5598549619317055, "learning_rate": 7.35072433232106e-06, "loss": 0.0033, "step": 1198, "step_time": 5.597067335002066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 759.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 262.15625, "completions/mean_terminated_length": 262.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3871047291904688, "epoch": 0.01199, "frac_reward_zero_std": 0.25, "grad_norm": 0.08151981234550476, "kl": 0.5274644233286381, "learning_rate": 7.350321926512808e-06, "loss": 0.0059, "num_tokens": 25769901.0, "reward": 0.8379889726638794, "reward_std": 0.37469130754470825, "rewards/rollout_reward_func/mean": 0.8379889726638794, "rewards/rollout_reward_func/std": 0.45892128348350525, "sampling/importance_sampling_ratio/max": 1.1329388618469238, "sampling/importance_sampling_ratio/mean": 0.9984176754951477, "sampling/importance_sampling_ratio/min": 0.8753233551979065, "sampling/sampling_logp_difference/max": 0.12289798259735107, "sampling/sampling_logp_difference/mean": 0.00588146410882473, "step": 1199, "step_time": 7.291192705997673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.3851148821413517, "epoch": 0.012, "grad_norm": 0.07738614827394485, "kl": 0.5312281586229801, "learning_rate": 7.350080481783718e-06, "loss": 0.0056, "step": 1200, "step_time": 3.5757568369954242 } ], "logging_steps": 1.0, "max_steps": 1200, "num_input_tokens_seen": 25769901, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }