{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.01526,
  "eval_steps": 500,
  "global_step": 763,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 709.0,
      "completions/max_terminated_length": 664.0,
      "completions/mean_length": 165.65625,
      "completions/mean_terminated_length": 144.25926208496094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.311180017888546,
      "epoch": 2e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8896363973617554,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": -0.2595,
      "num_tokens": 38128.0,
      "reward": -0.5365065336227417,
      "reward_std": 0.8142693042755127,
      "rewards/rollout_reward_func/mean": -0.5365065336227417,
      "rewards/rollout_reward_func/std": 0.8154047131538391,
      "sampling/importance_sampling_ratio/max": 1.4019912481307983,
      "sampling/importance_sampling_ratio/mean": 0.9388847351074219,
      "sampling/importance_sampling_ratio/min": 3.81146809236718e-19,
      "sampling/sampling_logp_difference/max": 19.187313079833984,
      "sampling/sampling_logp_difference/mean": 0.14528149366378784,
      "step": 1,
      "step_time": 12.833032263006316
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 617.0,
      "completions/max_terminated_length": 617.0,
      "completions/mean_length": 187.84375,
      "completions/mean_terminated_length": 188.29629516601562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.319596093147993,
      "epoch": 4e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.0004594326019287,
      "kl": 0.0,
      "learning_rate": 2.8571428571428575e-07,
      "loss": -0.2038,
      "num_tokens": 77868.0,
      "reward": -0.4847926199436188,
      "reward_std": 0.8636826872825623,
      "rewards/rollout_reward_func/mean": -0.4847926199436188,
      "rewards/rollout_reward_func/std": 0.8997911214828491,
      "sampling/importance_sampling_ratio/max": 1.5944570302963257,
      "sampling/importance_sampling_ratio/mean": 0.9668053388595581,
      "sampling/importance_sampling_ratio/min": 5.900435439681366e-19,
      "sampling/sampling_logp_difference/max": 22.0562744140625,
      "sampling/sampling_logp_difference/mean": 0.23154181241989136,
      "step": 2,
      "step_time": 12.070783853007015
    },
    {
      "clip_ratio/high_max": 0.004166666883975267,
      "clip_ratio/high_mean": 0.0020833334419876337,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0020833334419876337,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 611.0,
      "completions/max_terminated_length": 517.0,
      "completions/mean_length": 126.03125,
      "completions/mean_terminated_length": 90.03846740722656,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4450889378786087,
      "epoch": 6e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4505630731582642,
      "kl": 0.001412635664564732,
      "learning_rate": 5.714285714285715e-07,
      "loss": -0.0775,
      "num_tokens": 113612.0,
      "reward": -0.7025327682495117,
      "reward_std": 0.6840498447418213,
      "rewards/rollout_reward_func/mean": -0.7025327682495117,
      "rewards/rollout_reward_func/std": 0.7812584042549133,
      "sampling/importance_sampling_ratio/max": 1.1692482233047485,
      "sampling/importance_sampling_ratio/mean": 0.9305992126464844,
      "sampling/importance_sampling_ratio/min": 0.7146958112716675,
      "sampling/sampling_logp_difference/max": 0.38480159640312195,
      "sampling/sampling_logp_difference/mean": 0.03510897979140282,
      "step": 3,
      "step_time": 11.669138184966869
    },
    {
      "clip_ratio/high_max": 0.016741071827709675,
      "clip_ratio/high_mean": 0.008370535913854837,
      "clip_ratio/low_mean": 0.006417410913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014787946827709675,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 673.0,
      "completions/max_terminated_length": 673.0,
      "completions/mean_length": 110.15625,
      "completions/mean_terminated_length": 115.85714721679688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.267135325819254,
      "epoch": 8e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2133665084838867,
      "kl": 0.002277123923704494,
      "learning_rate": 8.571428571428572e-07,
      "loss": -0.0813,
      "num_tokens": 149947.0,
      "reward": -0.796975314617157,
      "reward_std": 0.5659746527671814,
      "rewards/rollout_reward_func/mean": -0.796975314617157,
      "rewards/rollout_reward_func/std": 0.7059975266456604,
      "sampling/importance_sampling_ratio/max": 1.7776038646697998,
      "sampling/importance_sampling_ratio/mean": 0.9955545663833618,
      "sampling/importance_sampling_ratio/min": 0.7075220942497253,
      "sampling/sampling_logp_difference/max": 0.4674229621887207,
      "sampling/sampling_logp_difference/mean": 0.03857395797967911,
      "step": 4,
      "step_time": 11.291042466007639
    },
    {
      "clip_ratio/high_max": 0.005078125046566129,
      "clip_ratio/high_mean": 0.003959516994655132,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019584516994655132,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 213.5,
      "completions/mean_terminated_length": 160.1923065185547,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3899846971035004,
      "epoch": 0.0001,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.8213181495666504,
      "kl": 0.002625905031891307,
      "learning_rate": 1.142857142857143e-06,
      "loss": -0.3893,
      "num_tokens": 189743.0,
      "reward": -0.5066970586776733,
      "reward_std": 0.7797619104385376,
      "rewards/rollout_reward_func/mean": -0.5066970586776733,
      "rewards/rollout_reward_func/std": 0.767596960067749,
      "sampling/importance_sampling_ratio/max": 1.6355137825012207,
      "sampling/importance_sampling_ratio/mean": 0.9999442100524902,
      "sampling/importance_sampling_ratio/min": 0.42015060782432556,
      "sampling/sampling_logp_difference/max": 0.7738604545593262,
      "sampling/sampling_logp_difference/mean": 0.04114900901913643,
      "step": 5,
      "step_time": 12.965868017010507
    },
    {
      "clip_ratio/high_max": 0.009598849574103951,
      "clip_ratio/high_mean": 0.004799424787051976,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004799424787051976,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 586.0,
      "completions/max_terminated_length": 586.0,
      "completions/mean_length": 128.40625,
      "completions/mean_terminated_length": 115.53572082519531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2549964375793934,
      "epoch": 0.00012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.163228988647461,
      "kl": 0.002137913768819999,
      "learning_rate": 1.4285714285714286e-06,
      "loss": -0.3945,
      "num_tokens": 226841.0,
      "reward": -0.7640929222106934,
      "reward_std": 0.6146565079689026,
      "rewards/rollout_reward_func/mean": -0.7640929222106934,
      "rewards/rollout_reward_func/std": 0.6997751593589783,
      "sampling/importance_sampling_ratio/max": 1.192243218421936,
      "sampling/importance_sampling_ratio/mean": 1.0114014148712158,
      "sampling/importance_sampling_ratio/min": 0.7369391322135925,
      "sampling/sampling_logp_difference/max": 0.3427910804748535,
      "sampling/sampling_logp_difference/mean": 0.03766397386789322,
      "step": 6,
      "step_time": 10.965820800993242
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 609.0,
      "completions/mean_length": 166.96875,
      "completions/mean_terminated_length": 145.27586364746094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2526164837181568,
      "epoch": 0.00014,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.1136302947998047,
      "kl": 0.002126004663296044,
      "learning_rate": 1.7142857142857145e-06,
      "loss": -0.2098,
      "num_tokens": 264690.0,
      "reward": -0.5239564180374146,
      "reward_std": 0.7401394844055176,
      "rewards/rollout_reward_func/mean": -0.5239564180374146,
      "rewards/rollout_reward_func/std": 0.8376838564872742,
      "sampling/importance_sampling_ratio/max": 1.706546425819397,
      "sampling/importance_sampling_ratio/mean": 0.929302990436554,
      "sampling/importance_sampling_ratio/min": 2.700833109049505e-16,
      "sampling/sampling_logp_difference/max": 23.282142639160156,
      "sampling/sampling_logp_difference/mean": 0.29891493916511536,
      "step": 7,
      "step_time": 11.320659755976521
    },
    {
      "clip_ratio/high_max": 0.0020833334419876337,
      "clip_ratio/high_mean": 0.002018229104578495,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002018229104578495,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 627.0,
      "completions/max_terminated_length": 627.0,
      "completions/mean_length": 222.5,
      "completions/mean_terminated_length": 188.11111450195312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2242905274033546,
      "epoch": 0.00016,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6946978569030762,
      "kl": 0.0022606493075727485,
      "learning_rate": 2.0000000000000003e-06,
      "loss": -0.0745,
      "num_tokens": 305137.0,
      "reward": -0.7431124448776245,
      "reward_std": 0.6849480867385864,
      "rewards/rollout_reward_func/mean": -0.7431124448776245,
      "rewards/rollout_reward_func/std": 0.6799914240837097,
      "sampling/importance_sampling_ratio/max": 1.4932160377502441,
      "sampling/importance_sampling_ratio/mean": 0.9482932090759277,
      "sampling/importance_sampling_ratio/min": 8.78426158801472e-16,
      "sampling/sampling_logp_difference/max": 19.693910598754883,
      "sampling/sampling_logp_difference/mean": 0.156938374042511,
      "step": 8,
      "step_time": 12.489437919022748
    },
    {
      "clip_ratio/high_max": 0.03515625,
      "clip_ratio/high_mean": 0.017578125,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0185546875,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 510.0,
      "completions/max_terminated_length": 510.0,
      "completions/mean_length": 123.46875,
      "completions/mean_terminated_length": 86.60714721679688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1092922799289227,
      "epoch": 0.00018,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.591063141822815,
      "kl": 0.0013983989556436427,
      "learning_rate": 2.285714285714286e-06,
      "loss": -0.1152,
      "num_tokens": 340324.0,
      "reward": -0.38865357637405396,
      "reward_std": 0.9937188029289246,
      "rewards/rollout_reward_func/mean": -0.38865357637405396,
      "rewards/rollout_reward_func/std": 0.9949004054069519,
      "sampling/importance_sampling_ratio/max": 1.402244210243225,
      "sampling/importance_sampling_ratio/mean": 0.9395061135292053,
      "sampling/importance_sampling_ratio/min": 0.5746212005615234,
      "sampling/sampling_logp_difference/max": 0.6010019779205322,
      "sampling/sampling_logp_difference/mean": 0.030809305608272552,
      "step": 9,
      "step_time": 10.788100366975414
    },
    {
      "clip_ratio/high_max": 0.02864583395421505,
      "clip_ratio/high_mean": 0.014322916977107525,
      "clip_ratio/low_mean": 0.006417410913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020740327890962362,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 560.0,
      "completions/mean_length": 111.6875,
      "completions/mean_terminated_length": 82.91999816894531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3130881078541279,
      "epoch": 0.0002,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8821022510528564,
      "kl": 0.003388037090189755,
      "learning_rate": 2.571428571428571e-06,
      "loss": -0.1755,
      "num_tokens": 376032.0,
      "reward": -0.32175493240356445,
      "reward_std": 0.8624504804611206,
      "rewards/rollout_reward_func/mean": -0.32175493240356445,
      "rewards/rollout_reward_func/std": 0.964446485042572,
      "sampling/importance_sampling_ratio/max": 1.520476222038269,
      "sampling/importance_sampling_ratio/mean": 0.9889786243438721,
      "sampling/importance_sampling_ratio/min": 0.5052693486213684,
      "sampling/sampling_logp_difference/max": 0.3002147674560547,
      "sampling/sampling_logp_difference/mean": 0.03807542100548744,
      "step": 10,
      "step_time": 11.664558338976349
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 627.0,
      "completions/mean_length": 164.125,
      "completions/mean_terminated_length": 132.42308044433594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3130264319479465,
      "epoch": 0.00022,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6921491622924805,
      "kl": 0.002810076988680521,
      "learning_rate": 2.8571428571428573e-06,
      "loss": -0.0849,
      "num_tokens": 414422.0,
      "reward": -0.8717756271362305,
      "reward_std": 0.5068723559379578,
      "rewards/rollout_reward_func/mean": -0.8717756271362305,
      "rewards/rollout_reward_func/std": 0.5629597306251526,
      "sampling/importance_sampling_ratio/max": 1.586397647857666,
      "sampling/importance_sampling_ratio/mean": 0.9789683818817139,
      "sampling/importance_sampling_ratio/min": 0.3991892337799072,
      "sampling/sampling_logp_difference/max": 0.3938761353492737,
      "sampling/sampling_logp_difference/mean": 0.043996065855026245,
      "step": 11,
      "step_time": 12.4948006549821
    },
    {
      "clip_ratio/high_max": 0.0026041667442768812,
      "clip_ratio/high_mean": 0.0013020833721384406,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022786458721384406,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 611.0,
      "completions/max_terminated_length": 609.0,
      "completions/mean_length": 193.5625,
      "completions/mean_terminated_length": 182.85714721679688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3143102452158928,
      "epoch": 0.00024,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.7590057849884033,
      "kl": 0.0025648477458162233,
      "learning_rate": 3.142857142857143e-06,
      "loss": -0.3398,
      "num_tokens": 452371.0,
      "reward": -0.5853269696235657,
      "reward_std": 0.8320633172988892,
      "rewards/rollout_reward_func/mean": -0.5853269696235657,
      "rewards/rollout_reward_func/std": 0.8285741209983826,
      "sampling/importance_sampling_ratio/max": 1.891076683998108,
      "sampling/importance_sampling_ratio/mean": 0.966175377368927,
      "sampling/importance_sampling_ratio/min": 0.5888997912406921,
      "sampling/sampling_logp_difference/max": 0.4839949607849121,
      "sampling/sampling_logp_difference/mean": 0.03853028267621994,
      "step": 12,
      "step_time": 11.412322128991946
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 682.0,
      "completions/max_terminated_length": 679.0,
      "completions/mean_length": 200.5625,
      "completions/mean_terminated_length": 177.58334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3637108765542507,
      "epoch": 0.00026,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4081811904907227,
      "kl": 0.004853192054724786,
      "learning_rate": 3.428571428571429e-06,
      "loss": -0.3982,
      "num_tokens": 492153.0,
      "reward": -0.21151718497276306,
      "reward_std": 0.9287054538726807,
      "rewards/rollout_reward_func/mean": -0.21151718497276306,
      "rewards/rollout_reward_func/std": 0.9370281100273132,
      "sampling/importance_sampling_ratio/max": 1.4452542066574097,
      "sampling/importance_sampling_ratio/mean": 0.9603732824325562,
      "sampling/importance_sampling_ratio/min": 1.2158485625433598e-15,
      "sampling/sampling_logp_difference/max": 21.371957778930664,
      "sampling/sampling_logp_difference/mean": 0.1360340565443039,
      "step": 13,
      "step_time": 11.984762279986171
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006417410913854837,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 184.375,
      "completions/mean_terminated_length": 103.13636779785156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.408212948590517,
      "epoch": 0.00028,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.5414741039276123,
      "kl": 0.003073074833082501,
      "learning_rate": 3.7142857142857146e-06,
      "loss": -0.1449,
      "num_tokens": 529467.0,
      "reward": -0.49260032176971436,
      "reward_std": 0.8508284091949463,
      "rewards/rollout_reward_func/mean": -0.49260032176971436,
      "rewards/rollout_reward_func/std": 0.8683142066001892,
      "sampling/importance_sampling_ratio/max": 1.8936798572540283,
      "sampling/importance_sampling_ratio/mean": 1.049086570739746,
      "sampling/importance_sampling_ratio/min": 2.0033956987380957e-22,
      "sampling/sampling_logp_difference/max": 20.88869285583496,
      "sampling/sampling_logp_difference/mean": 0.15057729184627533,
      "step": 14,
      "step_time": 13.066578525016666
    },
    {
      "clip_ratio/high_max": 0.0064062499441206455,
      "clip_ratio/high_mean": 0.004319196566939354,
      "clip_ratio/low_mean": 0.018637613160535693,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022956809727475047,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 654.0,
      "completions/mean_length": 135.1875,
      "completions/mean_terminated_length": 101.14814758300781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.171111274510622,
      "epoch": 0.0003,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2220170497894287,
      "kl": 0.003935700347938109,
      "learning_rate": 4.000000000000001e-06,
      "loss": -0.2664,
      "num_tokens": 565747.0,
      "reward": -0.6203838586807251,
      "reward_std": 0.6837176084518433,
      "rewards/rollout_reward_func/mean": -0.6203838586807251,
      "rewards/rollout_reward_func/std": 0.8200032711029053,
      "sampling/importance_sampling_ratio/max": 1.8238638639450073,
      "sampling/importance_sampling_ratio/mean": 1.1093041896820068,
      "sampling/importance_sampling_ratio/min": 0.6131741404533386,
      "sampling/sampling_logp_difference/max": 0.5632424354553223,
      "sampling/sampling_logp_difference/mean": 0.037908367812633514,
      "step": 15,
      "step_time": 11.77081656397786
    },
    {
      "clip_ratio/high_max": 0.06770833348855376,
      "clip_ratio/high_mean": 0.03385416674427688,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04947916674427688,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 662.0,
      "completions/max_terminated_length": 662.0,
      "completions/mean_length": 187.15625,
      "completions/mean_terminated_length": 168.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2740980796515942,
      "epoch": 0.00032,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9211934804916382,
      "kl": 0.007412969440338202,
      "learning_rate": 4.2857142857142855e-06,
      "loss": -0.212,
      "num_tokens": 605068.0,
      "reward": -0.35430702567100525,
      "reward_std": 0.8963297605514526,
      "rewards/rollout_reward_func/mean": -0.35430702567100525,
      "rewards/rollout_reward_func/std": 0.8739709258079529,
      "sampling/importance_sampling_ratio/max": 1.3636423349380493,
      "sampling/importance_sampling_ratio/mean": 0.932923436164856,
      "sampling/importance_sampling_ratio/min": 0.24788931012153625,
      "sampling/sampling_logp_difference/max": 0.5082712173461914,
      "sampling/sampling_logp_difference/mean": 0.03663131594657898,
      "step": 16,
      "step_time": 11.773229568003444
    },
    {
      "clip_ratio/high_max": 0.011753015452995896,
      "clip_ratio/high_mean": 0.005876507726497948,
      "clip_ratio/low_mean": 0.020089285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.025965793640352786,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 611.0,
      "completions/max_terminated_length": 567.0,
      "completions/mean_length": 129.4375,
      "completions/mean_terminated_length": 88.56521606445312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.35793412104249,
      "epoch": 0.00034,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.743879556655884,
      "kl": 0.00883992858507554,
      "learning_rate": 4.571428571428572e-06,
      "loss": -0.2842,
      "num_tokens": 641948.0,
      "reward": -0.427093505859375,
      "reward_std": 0.8510888814926147,
      "rewards/rollout_reward_func/mean": -0.427093505859375,
      "rewards/rollout_reward_func/std": 0.9136654138565063,
      "sampling/importance_sampling_ratio/max": 2.5361130237579346,
      "sampling/importance_sampling_ratio/mean": 1.043760895729065,
      "sampling/importance_sampling_ratio/min": 4.185795950336377e-17,
      "sampling/sampling_logp_difference/max": 19.90718650817871,
      "sampling/sampling_logp_difference/mean": 0.27757465839385986,
      "step": 17,
      "step_time": 13.368114275988773
    },
    {
      "clip_ratio/high_max": 0.005681818351149559,
      "clip_ratio/high_mean": 0.0028409091755747795,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0028409091755747795,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 123.40625,
      "completions/mean_terminated_length": 96.08695983886719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4243219494819641,
      "epoch": 0.00036,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.489845037460327,
      "kl": 0.015196269989246503,
      "learning_rate": 4.857142857142858e-06,
      "loss": -0.1851,
      "num_tokens": 679383.0,
      "reward": -0.5160565972328186,
      "reward_std": 0.8708747625350952,
      "rewards/rollout_reward_func/mean": -0.5160565972328186,
      "rewards/rollout_reward_func/std": 0.8823010921478271,
      "sampling/importance_sampling_ratio/max": 1.5874239206314087,
      "sampling/importance_sampling_ratio/mean": 0.9520589113235474,
      "sampling/importance_sampling_ratio/min": 1.7547460880174004e-12,
      "sampling/sampling_logp_difference/max": 26.967424392700195,
      "sampling/sampling_logp_difference/mean": 0.11855955421924591,
      "step": 18,
      "step_time": 11.921432229020866
    },
    {
      "clip_ratio/high_max": 0.007244130363687873,
      "clip_ratio/high_mean": 0.004779472714290023,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005756035214290023,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 570.0,
      "completions/mean_length": 188.0625,
      "completions/mean_terminated_length": 140.6999969482422,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1907794810831547,
      "epoch": 0.00038,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4401724338531494,
      "kl": 0.017040928709320724,
      "learning_rate": 5.142857142857142e-06,
      "loss": -0.1956,
      "num_tokens": 718391.0,
      "reward": -0.3141319155693054,
      "reward_std": 0.896587610244751,
      "rewards/rollout_reward_func/mean": -0.3141319155693054,
      "rewards/rollout_reward_func/std": 0.9096906781196594,
      "sampling/importance_sampling_ratio/max": 2.34269380569458,
      "sampling/importance_sampling_ratio/mean": 1.0541638135910034,
      "sampling/importance_sampling_ratio/min": 0.4154016673564911,
      "sampling/sampling_logp_difference/max": 0.40622854232788086,
      "sampling/sampling_logp_difference/mean": 0.04237822815775871,
      "step": 19,
      "step_time": 12.751815825002268
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 573.0,
      "completions/max_terminated_length": 573.0,
      "completions/mean_length": 113.71875,
      "completions/mean_terminated_length": 103.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2980789467692375,
      "epoch": 0.0004,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.13360595703125,
      "kl": 0.03493207323481329,
      "learning_rate": 5.428571428571429e-06,
      "loss": -0.1133,
      "num_tokens": 754645.0,
      "reward": -0.31959348917007446,
      "reward_std": 0.9202141761779785,
      "rewards/rollout_reward_func/mean": -0.31959348917007446,
      "rewards/rollout_reward_func/std": 1.0103421211242676,
      "sampling/importance_sampling_ratio/max": 1.5369470119476318,
      "sampling/importance_sampling_ratio/mean": 0.8885481357574463,
      "sampling/importance_sampling_ratio/min": 1.1214207324252226e-13,
      "sampling/sampling_logp_difference/max": 8.88003158569336,
      "sampling/sampling_logp_difference/mean": 0.14728209376335144,
      "step": 20,
      "step_time": 12.034123226010706
    },
    {
      "clip_ratio/high_max": 0.007634943351149559,
      "clip_ratio/high_mean": 0.0038174716755747795,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01944247167557478,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 602.0,
      "completions/max_terminated_length": 591.0,
      "completions/mean_length": 157.6875,
      "completions/mean_terminated_length": 108.73077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3747540935873985,
      "epoch": 0.00042,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4938822984695435,
      "kl": 0.06438212288776413,
      "learning_rate": 5.7142857142857145e-06,
      "loss": -0.1,
      "num_tokens": 792243.0,
      "reward": -0.3738252520561218,
      "reward_std": 0.9256372451782227,
      "rewards/rollout_reward_func/mean": -0.3738252520561218,
      "rewards/rollout_reward_func/std": 0.9255111813545227,
      "sampling/importance_sampling_ratio/max": 1.6604886054992676,
      "sampling/importance_sampling_ratio/mean": 0.8727307319641113,
      "sampling/importance_sampling_ratio/min": 0.3606182932853699,
      "sampling/sampling_logp_difference/max": 0.918931245803833,
      "sampling/sampling_logp_difference/mean": 0.06452113389968872,
      "step": 21,
      "step_time": 11.438725079016876
    },
    {
      "clip_ratio/high_max": 0.00983796315267682,
      "clip_ratio/high_mean": 0.00491898157633841,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009383267490193248,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 580.0,
      "completions/max_terminated_length": 580.0,
      "completions/mean_length": 227.65625,
      "completions/mean_terminated_length": 254.57693481445312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3308311328291893,
      "epoch": 0.00044,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.7486307621002197,
      "kl": 0.068185698997695,
      "learning_rate": 6e-06,
      "loss": -0.3333,
      "num_tokens": 832666.0,
      "reward": -0.20104746520519257,
      "reward_std": 1.0254323482513428,
      "rewards/rollout_reward_func/mean": -0.20104746520519257,
      "rewards/rollout_reward_func/std": 0.9808058142662048,
      "sampling/importance_sampling_ratio/max": 1.768102765083313,
      "sampling/importance_sampling_ratio/mean": 0.8560611009597778,
      "sampling/importance_sampling_ratio/min": 2.5722225380141682e-16,
      "sampling/sampling_logp_difference/max": 24.601207733154297,
      "sampling/sampling_logp_difference/mean": 0.3779761493206024,
      "step": 22,
      "step_time": 12.780610617017373
    },
    {
      "clip_ratio/high_max": 0.03049479192122817,
      "clip_ratio/high_mean": 0.015247395960614085,
      "clip_ratio/low_mean": 0.03766741091385484,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.05291480687446892,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 692.0,
      "completions/mean_length": 184.34375,
      "completions/mean_terminated_length": 191.4545440673828,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.336107350885868,
      "epoch": 0.00046,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.919745445251465,
      "kl": 0.12588163682084996,
      "learning_rate": 6.285714285714286e-06,
      "loss": -0.245,
      "num_tokens": 871667.0,
      "reward": -0.14790122210979462,
      "reward_std": 0.9092764854431152,
      "rewards/rollout_reward_func/mean": -0.14790122210979462,
      "rewards/rollout_reward_func/std": 0.8701663613319397,
      "sampling/importance_sampling_ratio/max": 2.4878475666046143,
      "sampling/importance_sampling_ratio/mean": 1.0415394306182861,
      "sampling/importance_sampling_ratio/min": 1.1708223366518888e-13,
      "sampling/sampling_logp_difference/max": 18.44968032836914,
      "sampling/sampling_logp_difference/mean": 0.14526084065437317,
      "step": 23,
      "step_time": 11.960473290993832
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017578125,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 198.03125,
      "completions/mean_terminated_length": 146.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.29803366959095,
      "epoch": 0.00048,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.52923321723938,
      "kl": 0.22498597187222913,
      "learning_rate": 6.571428571428572e-06,
      "loss": -0.5774,
      "num_tokens": 910885.0,
      "reward": -0.047677457332611084,
      "reward_std": 1.0225427150726318,
      "rewards/rollout_reward_func/mean": -0.047677457332611084,
      "rewards/rollout_reward_func/std": 1.0035980939865112,
      "sampling/importance_sampling_ratio/max": 2.115689277648926,
      "sampling/importance_sampling_ratio/mean": 1.0643537044525146,
      "sampling/importance_sampling_ratio/min": 0.31220144033432007,
      "sampling/sampling_logp_difference/max": 1.0372114181518555,
      "sampling/sampling_logp_difference/mean": 0.06783659011125565,
      "step": 24,
      "step_time": 12.168675857989001
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 593.0,
      "completions/max_terminated_length": 564.0,
      "completions/mean_length": 144.625,
      "completions/mean_terminated_length": 70.09091186523438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3542137630283833,
      "epoch": 0.0005,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.8559019565582275,
      "kl": 0.4584066985989921,
      "learning_rate": 6.857142857142858e-06,
      "loss": -0.6288,
      "num_tokens": 947124.0,
      "reward": -0.3790113925933838,
      "reward_std": 0.9996820688247681,
      "rewards/rollout_reward_func/mean": -0.3790113925933838,
      "rewards/rollout_reward_func/std": 0.9527615904808044,
      "sampling/importance_sampling_ratio/max": 2.2629711627960205,
      "sampling/importance_sampling_ratio/mean": 0.8424215316772461,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.623473882675171,
      "sampling/sampling_logp_difference/mean": 0.0840912014245987,
      "step": 25,
      "step_time": 12.46073328802595
    },
    {
      "clip_ratio/high_max": 0.009765625,
      "clip_ratio/high_mean": 0.0048828125,
      "clip_ratio/low_mean": 0.017708333441987634,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022591145941987634,
      "completions/clipped_ratio": 0.53125,
      "completions/max_length": 602.0,
      "completions/max_terminated_length": 544.0,
      "completions/mean_length": 273.625,
      "completions/mean_terminated_length": 163.73333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.223763294517994,
      "epoch": 0.00052,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.011223554611206,
      "kl": 0.8628979871282354,
      "learning_rate": 7.1428571428571436e-06,
      "loss": -0.5437,
      "num_tokens": 989582.0,
      "reward": 0.011283770203590393,
      "reward_std": 0.9776384234428406,
      "rewards/rollout_reward_func/mean": 0.011283770203590393,
      "rewards/rollout_reward_func/std": 0.9817814230918884,
      "sampling/importance_sampling_ratio/max": 2.8080434799194336,
      "sampling/importance_sampling_ratio/mean": 0.9708855152130127,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.209257125854492,
      "sampling/sampling_logp_difference/mean": 0.07115766406059265,
      "step": 26,
      "step_time": 12.830889011034742
    },
    {
      "clip_ratio/high_max": 0.027890624944120646,
      "clip_ratio/high_mean": 0.013945312472060323,
      "clip_ratio/low_mean": 0.012276785913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02622209838591516,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 684.0,
      "completions/max_terminated_length": 684.0,
      "completions/mean_length": 215.65625,
      "completions/mean_terminated_length": 201.69566345214844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2758811637759209,
      "epoch": 0.00054,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.5620768070220947,
      "kl": 2.00925292854663,
      "learning_rate": 7.428571428571429e-06,
      "loss": -0.526,
      "num_tokens": 1028391.0,
      "reward": -0.2764366567134857,
      "reward_std": 0.9408585429191589,
      "rewards/rollout_reward_func/mean": -0.2764366567134857,
      "rewards/rollout_reward_func/std": 0.9261830449104309,
      "sampling/importance_sampling_ratio/max": 2.4296875,
      "sampling/importance_sampling_ratio/mean": 0.6688175201416016,
      "sampling/importance_sampling_ratio/min": 7.006625279970535e-10,
      "sampling/sampling_logp_difference/max": 5.829579830169678,
      "sampling/sampling_logp_difference/mean": 0.16394366323947906,
      "step": 27,
      "step_time": 12.109036321009626
    },
    {
      "clip_ratio/high_max": 0.011979166883975267,
      "clip_ratio/high_mean": 0.005989583441987634,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005989583441987634,
      "completions/clipped_ratio": 0.53125,
      "completions/max_length": 681.0,
      "completions/max_terminated_length": 438.0,
      "completions/mean_length": 218.1875,
      "completions/mean_terminated_length": 110.00000762939453,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3303226046264172,
      "epoch": 0.00056,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7852479219436646,
      "kl": 0.5620120603125542,
      "learning_rate": 7.714285714285716e-06,
      "loss": -0.4069,
      "num_tokens": 1066791.0,
      "reward": 0.4433053433895111,
      "reward_std": 0.7434237003326416,
      "rewards/rollout_reward_func/mean": 0.4433053433895111,
      "rewards/rollout_reward_func/std": 0.7727333903312683,
      "sampling/importance_sampling_ratio/max": 2.6013448238372803,
      "sampling/importance_sampling_ratio/mean": 0.8827865123748779,
      "sampling/importance_sampling_ratio/min": 4.088573629884187e-21,
      "sampling/sampling_logp_difference/max": 19.298248291015625,
      "sampling/sampling_logp_difference/mean": 0.16345565021038055,
      "step": 28,
      "step_time": 13.72175006303587
    },
    {
      "clip_ratio/high_max": 0.012508680578321218,
      "clip_ratio/high_mean": 0.006254340289160609,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010718626086600125,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 627.0,
      "completions/mean_length": 282.75,
      "completions/mean_terminated_length": 208.8095245361328,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2037345804274082,
      "epoch": 0.00058,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 5.508101463317871,
      "kl": 11.269055972166825,
      "learning_rate": 8.000000000000001e-06,
      "loss": -0.1698,
      "num_tokens": 1109203.0,
      "reward": 0.07700280100107193,
      "reward_std": 0.8241150379180908,
      "rewards/rollout_reward_func/mean": 0.07700280100107193,
      "rewards/rollout_reward_func/std": 0.8405054807662964,
      "sampling/importance_sampling_ratio/max": 2.86274790763855,
      "sampling/importance_sampling_ratio/mean": 0.9066811800003052,
      "sampling/importance_sampling_ratio/min": 0.01681259460747242,
      "sampling/sampling_logp_difference/max": 4.051913261413574,
      "sampling/sampling_logp_difference/mean": 0.11221412569284439,
      "step": 29,
      "step_time": 12.718567863019416
    },
    {
      "clip_ratio/high_max": 0.03510795393958688,
      "clip_ratio/high_mean": 0.018562041455879807,
      "clip_ratio/low_mean": 0.022046638652682304,
      "clip_ratio/low_min": 0.0052083334885537624,
      "clip_ratio/region_mean": 0.040608680224977434,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 618.0,
      "completions/mean_length": 272.53125,
      "completions/mean_terminated_length": 287.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2472670152783394,
      "epoch": 0.0006,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.559933662414551,
      "kl": 2.212796541978605,
      "learning_rate": 8.285714285714287e-06,
      "loss": -0.2593,
      "num_tokens": 1151053.0,
      "reward": 0.03223041072487831,
      "reward_std": 0.9080570936203003,
      "rewards/rollout_reward_func/mean": 0.03223041072487831,
      "rewards/rollout_reward_func/std": 0.8770092725753784,
      "sampling/importance_sampling_ratio/max": 2.369997262954712,
      "sampling/importance_sampling_ratio/mean": 0.9031524658203125,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.734834909439087,
      "sampling/sampling_logp_difference/mean": 0.0942210778594017,
      "step": 30,
      "step_time": 12.399974271989777
    },
    {
      "clip_ratio/high_max": 0.0020833334419876337,
      "clip_ratio/high_mean": 0.0010416667209938169,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002994791720993817,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 670.0,
      "completions/max_terminated_length": 670.0,
      "completions/mean_length": 285.65625,
      "completions/mean_terminated_length": 227.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.213874440640211,
      "epoch": 0.00062,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.132397174835205,
      "kl": 1.742662149714306,
      "learning_rate": 8.571428571428571e-06,
      "loss": -0.0097,
      "num_tokens": 1193112.0,
      "reward": 0.09106145799160004,
      "reward_std": 0.780171811580658,
      "rewards/rollout_reward_func/mean": 0.09106145799160004,
      "rewards/rollout_reward_func/std": 0.9324904680252075,
      "sampling/importance_sampling_ratio/max": 2.422135591506958,
      "sampling/importance_sampling_ratio/mean": 0.8946410417556763,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 7.716803073883057,
      "sampling/sampling_logp_difference/mean": 0.14541180431842804,
      "step": 31,
      "step_time": 13.688056645987672
    },
    {
      "clip_ratio/high_max": 0.010312499944120646,
      "clip_ratio/high_mean": 0.006132812472060323,
      "clip_ratio/low_mean": 0.0048828125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011015624972060323,
      "completions/clipped_ratio": 0.625,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 258.4375,
      "completions/mean_terminated_length": 181.1666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0814621523022652,
      "epoch": 0.00064,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.968383550643921,
      "kl": 1.8962593984324485,
      "learning_rate": 8.857142857142858e-06,
      "loss": -0.369,
      "num_tokens": 1235228.0,
      "reward": 0.00343988835811615,
      "reward_std": 0.8214678764343262,
      "rewards/rollout_reward_func/mean": 0.00343988835811615,
      "rewards/rollout_reward_func/std": 0.900937557220459,
      "sampling/importance_sampling_ratio/max": 2.8939833641052246,
      "sampling/importance_sampling_ratio/mean": 1.0162222385406494,
      "sampling/importance_sampling_ratio/min": 7.23888915102306e-18,
      "sampling/sampling_logp_difference/max": 24.544570922851562,
      "sampling/sampling_logp_difference/mean": 0.12960600852966309,
      "step": 32,
      "step_time": 13.092168052971829
    },
    {
      "clip_ratio/high_max": 0.008713942486792803,
      "clip_ratio/high_mean": 0.004356971243396401,
      "clip_ratio/low_mean": 0.0107421875,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015099158743396401,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 235.6875,
      "completions/mean_terminated_length": 243.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1770143881440163,
      "epoch": 0.00066,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.100741147994995,
      "kl": 2.1499057188630104,
      "learning_rate": 9.142857142857144e-06,
      "loss": -0.2792,
      "num_tokens": 1274975.0,
      "reward": 0.2791275978088379,
      "reward_std": 0.780487060546875,
      "rewards/rollout_reward_func/mean": 0.2791275978088379,
      "rewards/rollout_reward_func/std": 0.8439303040504456,
      "sampling/importance_sampling_ratio/max": 2.5422511100769043,
      "sampling/importance_sampling_ratio/mean": 0.9451858997344971,
      "sampling/importance_sampling_ratio/min": 4.847359319493094e-11,
      "sampling/sampling_logp_difference/max": 19.437362670898438,
      "sampling/sampling_logp_difference/mean": 0.13978037238121033,
      "step": 33,
      "step_time": 12.334184916995582
    },
    {
      "clip_ratio/high_max": 0.010312499944120646,
      "clip_ratio/high_mean": 0.005156249972060323,
      "clip_ratio/low_mean": 0.0166015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021757812472060323,
      "completions/clipped_ratio": 0.53125,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 693.0,
      "completions/mean_length": 260.5,
      "completions/mean_terminated_length": 168.53334045410156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2332624532282352,
      "epoch": 0.00068,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.299915313720703,
      "kl": 7.750924182822928,
      "learning_rate": 9.42857142857143e-06,
      "loss": -0.503,
      "num_tokens": 1316431.0,
      "reward": 0.04624081403017044,
      "reward_std": 0.956036388874054,
      "rewards/rollout_reward_func/mean": 0.04624081403017044,
      "rewards/rollout_reward_func/std": 0.9308807849884033,
      "sampling/importance_sampling_ratio/max": 2.9622371196746826,
      "sampling/importance_sampling_ratio/mean": 0.8872480392456055,
      "sampling/importance_sampling_ratio/min": 1.2781168562601526e-23,
      "sampling/sampling_logp_difference/max": 19.268573760986328,
      "sampling/sampling_logp_difference/mean": 0.18415531516075134,
      "step": 34,
      "step_time": 13.430990070992266
    },
    {
      "clip_ratio/high_max": 0.013671875,
      "clip_ratio/high_mean": 0.0068359375,
      "clip_ratio/low_mean": 0.0029296875,
      "clip_ratio/low_min": 0.001953125,
      "clip_ratio/region_mean": 0.009765625,
      "completions/clipped_ratio": 0.5625,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 592.0,
      "completions/mean_length": 240.03125,
      "completions/mean_terminated_length": 202.00001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.049317855387926,
      "epoch": 0.0007,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 8.343193054199219,
      "kl": 15.15319203818217,
      "learning_rate": 9.714285714285715e-06,
      "loss": -0.4965,
      "num_tokens": 1357835.0,
      "reward": 0.35522377490997314,
      "reward_std": 0.8883234262466431,
      "rewards/rollout_reward_func/mean": 0.35522377490997314,
      "rewards/rollout_reward_func/std": 0.8605958819389343,
      "sampling/importance_sampling_ratio/max": 2.466466188430786,
      "sampling/importance_sampling_ratio/mean": 0.868963360786438,
      "sampling/importance_sampling_ratio/min": 8.786470357714793e-13,
      "sampling/sampling_logp_difference/max": 11.585200309753418,
      "sampling/sampling_logp_difference/mean": 0.13173317909240723,
      "step": 35,
      "step_time": 13.321685539034661
    },
    {
      "clip_ratio/high_max": 0.03326612897217274,
      "clip_ratio/high_mean": 0.01663306448608637,
      "clip_ratio/low_mean": 0.01171875,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02835181448608637,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 222.3125,
      "completions/mean_terminated_length": 221.1999969482422,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1060705073177814,
      "epoch": 0.00072,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.184360980987549,
      "kl": 1.0409424516838044,
      "learning_rate": 1e-05,
      "loss": -0.263,
      "num_tokens": 1397311.0,
      "reward": 0.1552642583847046,
      "reward_std": 0.9061756134033203,
      "rewards/rollout_reward_func/mean": 0.1552642583847046,
      "rewards/rollout_reward_func/std": 0.8826220631599426,
      "sampling/importance_sampling_ratio/max": 2.3881959915161133,
      "sampling/importance_sampling_ratio/mean": 0.8509060144424438,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.1544408798217773,
      "sampling/sampling_logp_difference/mean": 0.09212174266576767,
      "step": 36,
      "step_time": 12.994348758002161
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.006510416744276881,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007486979244276881,
      "completions/clipped_ratio": 0.5625,
      "completions/max_length": 668.0,
      "completions/max_terminated_length": 668.0,
      "completions/mean_length": 237.03125,
      "completions/mean_terminated_length": 215.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0860639698803425,
      "epoch": 0.00074,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.5980350971221924,
      "kl": 1.2122145568719134,
      "learning_rate": 9.999975267482496e-06,
      "loss": -0.3827,
      "num_tokens": 1438076.0,
      "reward": 0.2174517661333084,
      "reward_std": 0.8759725093841553,
      "rewards/rollout_reward_func/mean": 0.2174517661333084,
      "rewards/rollout_reward_func/std": 0.8674449920654297,
      "sampling/importance_sampling_ratio/max": 2.8948092460632324,
      "sampling/importance_sampling_ratio/mean": 0.9303849339485168,
      "sampling/importance_sampling_ratio/min": 1.8520833467133152e-13,
      "sampling/sampling_logp_difference/max": 10.263285636901855,
      "sampling/sampling_logp_difference/mean": 0.1275598406791687,
      "step": 37,
      "step_time": 13.203734765003901
    },
    {
      "clip_ratio/high_max": 0.01455965917557478,
      "clip_ratio/high_mean": 0.00727982958778739,
      "clip_ratio/low_mean": 0.033203125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04048295458778739,
      "completions/clipped_ratio": 0.40625,
      "completions/max_length": 589.0,
      "completions/max_terminated_length": 589.0,
      "completions/mean_length": 225.125,
      "completions/mean_terminated_length": 202.36842346191406,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.155680451542139,
      "epoch": 0.00076,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4092390537261963,
      "kl": 4.133358426624909,
      "learning_rate": 9.99990107025622e-06,
      "loss": -0.5875,
      "num_tokens": 1478871.0,
      "reward": 0.21051502227783203,
      "reward_std": 0.935226559638977,
      "rewards/rollout_reward_func/mean": 0.21051502227783203,
      "rewards/rollout_reward_func/std": 0.9536476135253906,
      "sampling/importance_sampling_ratio/max": 2.811084270477295,
      "sampling/importance_sampling_ratio/mean": 0.7887800931930542,
      "sampling/importance_sampling_ratio/min": 4.156989706036705e-19,
      "sampling/sampling_logp_difference/max": 22.87867546081543,
      "sampling/sampling_logp_difference/mean": 0.19020409882068634,
      "step": 38,
      "step_time": 12.744203613023274
    },
    {
      "clip_ratio/high_max": 0.012620192486792803,
      "clip_ratio/high_mean": 0.007286658743396401,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011750944657251239,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 274.03125,
      "completions/mean_terminated_length": 208.9091033935547,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.031787384301424,
      "epoch": 0.00078,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.511378765106201,
      "kl": 0.2938755719223991,
      "learning_rate": 9.99977740929988e-06,
      "loss": 0.0119,
      "num_tokens": 1519872.0,
      "reward": 0.22999998927116394,
      "reward_std": 0.85052889585495,
      "rewards/rollout_reward_func/mean": 0.22999998927116394,
      "rewards/rollout_reward_func/std": 0.8179991245269775,
      "sampling/importance_sampling_ratio/max": 2.8975095748901367,
      "sampling/importance_sampling_ratio/mean": 0.9791565537452698,
      "sampling/importance_sampling_ratio/min": 0.12304184585809708,
      "sampling/sampling_logp_difference/max": 1.7892673015594482,
      "sampling/sampling_logp_difference/mean": 0.07746706157922745,
      "step": 39,
      "step_time": 12.708630217035534
    },
    {
      "clip_ratio/high_max": 0.014669471187517047,
      "clip_ratio/high_mean": 0.0073347355937585235,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011799021507613361,
      "completions/clipped_ratio": 0.53125,
      "completions/max_length": 710.0,
      "completions/max_terminated_length": 585.0,
      "completions/mean_length": 260.59375,
      "completions/mean_terminated_length": 199.00001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1196067593991756,
      "epoch": 0.0008,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.846652030944824,
      "kl": 0.8095282192807645,
      "learning_rate": 9.999604286244655e-06,
      "loss": -0.2668,
      "num_tokens": 1561101.0,
      "reward": 0.32859569787979126,
      "reward_std": 0.810870885848999,
      "rewards/rollout_reward_func/mean": 0.32859569787979126,
      "rewards/rollout_reward_func/std": 0.8544008135795593,
      "sampling/importance_sampling_ratio/max": 2.480098247528076,
      "sampling/importance_sampling_ratio/mean": 0.9871659874916077,
      "sampling/importance_sampling_ratio/min": 0.1635282188653946,
      "sampling/sampling_logp_difference/max": 1.901123046875,
      "sampling/sampling_logp_difference/mean": 0.07796694338321686,
      "step": 40,
      "step_time": 12.927784984989557
    },
    {
      "clip_ratio/high_max": 0.02846827683970332,
      "clip_ratio/high_mean": 0.01423413841985166,
      "clip_ratio/low_mean": 0.0020833334419876337,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016317471861839294,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 693.0,
      "completions/max_terminated_length": 693.0,
      "completions/mean_length": 224.28125,
      "completions/mean_terminated_length": 149.72222900390625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0890238247811794,
      "epoch": 0.00082,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.696411371231079,
      "kl": 1.716181415016763,
      "learning_rate": 9.999381703374151e-06,
      "loss": -0.1842,
      "num_tokens": 1600903.0,
      "reward": 0.17870177328586578,
      "reward_std": 0.8818931579589844,
      "rewards/rollout_reward_func/mean": 0.17870177328586578,
      "rewards/rollout_reward_func/std": 0.8923808336257935,
      "sampling/importance_sampling_ratio/max": 2.372279644012451,
      "sampling/importance_sampling_ratio/mean": 0.9427394866943359,
      "sampling/importance_sampling_ratio/min": 0.08107419312000275,
      "sampling/sampling_logp_difference/max": 2.500169277191162,
      "sampling/sampling_logp_difference/mean": 0.08845722675323486,
      "step": 41,
      "step_time": 12.208342930010986
    },
    {
      "clip_ratio/high_max": 0.011104130418971181,
      "clip_ratio/high_mean": 0.0055520652094855905,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0055520652094855905,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 703.0,
      "completions/max_terminated_length": 576.0,
      "completions/mean_length": 144.3125,
      "completions/mean_terminated_length": 108.7727279663086,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.038298036903143,
      "epoch": 0.00084,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.733799695968628,
      "kl": 1.8870345742907375,
      "learning_rate": 9.999109663624387e-06,
      "loss": -0.118,
      "num_tokens": 1635316.0,
      "reward": -0.16055701673030853,
      "reward_std": 1.0385457277297974,
      "rewards/rollout_reward_func/mean": -0.16055701673030853,
      "rewards/rollout_reward_func/std": 1.005665898323059,
      "sampling/importance_sampling_ratio/max": 1.7625305652618408,
      "sampling/importance_sampling_ratio/mean": 0.6761658191680908,
      "sampling/importance_sampling_ratio/min": 3.963593267887564e-16,
      "sampling/sampling_logp_difference/max": 22.187162399291992,
      "sampling/sampling_logp_difference/mean": 0.22613836824893951,
      "step": 42,
      "step_time": 12.541349949984578
    },
    {
      "clip_ratio/high_max": 0.014822583412751555,
      "clip_ratio/high_mean": 0.007411291706375778,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009364416706375778,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 228.125,
      "completions/mean_terminated_length": 203.20834350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1823361925780773,
      "epoch": 0.00086,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9977695941925049,
      "kl": 0.5937003343133256,
      "learning_rate": 9.99878817058375e-06,
      "loss": -0.1402,
      "num_tokens": 1675992.0,
      "reward": 0.28343746066093445,
      "reward_std": 0.8324289321899414,
      "rewards/rollout_reward_func/mean": 0.28343746066093445,
      "rewards/rollout_reward_func/std": 0.8495069146156311,
      "sampling/importance_sampling_ratio/max": 2.2500710487365723,
      "sampling/importance_sampling_ratio/mean": 0.8333957195281982,
      "sampling/importance_sampling_ratio/min": 0.14577610790729523,
      "sampling/sampling_logp_difference/max": 1.9014002084732056,
      "sampling/sampling_logp_difference/mean": 0.08698943257331848,
      "step": 43,
      "step_time": 12.529271317995153
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0029296875,
      "completions/clipped_ratio": 0.40625,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 603.0,
      "completions/mean_length": 278.59375,
      "completions/mean_terminated_length": 211.94737243652344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0612012669444084,
      "epoch": 0.00088,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.604757070541382,
      "kl": 0.23596757056657225,
      "learning_rate": 9.998417228492952e-06,
      "loss": -0.3464,
      "num_tokens": 1718830.0,
      "reward": -0.1312331259250641,
      "reward_std": 1.003420114517212,
      "rewards/rollout_reward_func/mean": -0.1312331259250641,
      "rewards/rollout_reward_func/std": 0.9654220938682556,
      "sampling/importance_sampling_ratio/max": 2.4708800315856934,
      "sampling/importance_sampling_ratio/mean": 0.8872724771499634,
      "sampling/importance_sampling_ratio/min": 0.22671109437942505,
      "sampling/sampling_logp_difference/max": 1.5190391540527344,
      "sampling/sampling_logp_difference/mean": 0.08081002533435822,
      "step": 44,
      "step_time": 12.713807573993108
    },
    {
      "clip_ratio/high_max": 0.01341145858168602,
      "clip_ratio/high_mean": 0.007682291907258332,
      "clip_ratio/low_mean": 0.0068359375,
      "clip_ratio/low_min": 0.00390625,
      "clip_ratio/region_mean": 0.014518229407258332,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 633.0,
      "completions/max_terminated_length": 633.0,
      "completions/mean_length": 189.25,
      "completions/mean_terminated_length": 104.3888931274414,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0731625594198704,
      "epoch": 0.0009,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4884719848632812,
      "kl": 1.5841654413379729,
      "learning_rate": 9.99799684224497e-06,
      "loss": -0.4672,
      "num_tokens": 1757751.0,
      "reward": 0.1198418140411377,
      "reward_std": 0.9273171424865723,
      "rewards/rollout_reward_func/mean": 0.1198418140411377,
      "rewards/rollout_reward_func/std": 0.9397498965263367,
      "sampling/importance_sampling_ratio/max": 2.385958194732666,
      "sampling/importance_sampling_ratio/mean": 0.8179432153701782,
      "sampling/importance_sampling_ratio/min": 0.08850578218698502,
      "sampling/sampling_logp_difference/max": 2.0960440635681152,
      "sampling/sampling_logp_difference/mean": 0.09452584385871887,
      "step": 45,
      "step_time": 13.00205962502514
    },
    {
      "clip_ratio/high_max": 0.009765625,
      "clip_ratio/high_mean": 0.005859375,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010323660913854837,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 705.0,
      "completions/max_terminated_length": 555.0,
      "completions/mean_length": 222.96875,
      "completions/mean_terminated_length": 205.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2059733271598816,
      "epoch": 0.00092,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.994724154472351,
      "kl": 0.6992635009810328,
      "learning_rate": 9.997527017384983e-06,
      "loss": -0.3209,
      "num_tokens": 1797424.0,
      "reward": 0.2451658844947815,
      "reward_std": 0.8973381519317627,
      "rewards/rollout_reward_func/mean": 0.2451658844947815,
      "rewards/rollout_reward_func/std": 0.8691799640655518,
      "sampling/importance_sampling_ratio/max": 2.933239698410034,
      "sampling/importance_sampling_ratio/mean": 0.8591638803482056,
      "sampling/importance_sampling_ratio/min": 0.12129704654216766,
      "sampling/sampling_logp_difference/max": 2.0878586769104004,
      "sampling/sampling_logp_difference/mean": 0.10459914803504944,
      "step": 46,
      "step_time": 11.963761335005984
    },
    {
      "clip_ratio/high_max": 0.02168642240576446,
      "clip_ratio/high_mean": 0.01084321120288223,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02646821120288223,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 141.8125,
      "completions/mean_terminated_length": 134.65000915527344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.056411225348711,
      "epoch": 0.00094,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.1516730785369873,
      "kl": 2.11896897200495,
      "learning_rate": 9.997007760110298e-06,
      "loss": -0.5285,
      "num_tokens": 1833496.0,
      "reward": 0.16420233249664307,
      "reward_std": 0.9279652833938599,
      "rewards/rollout_reward_func/mean": 0.16420233249664307,
      "rewards/rollout_reward_func/std": 0.9548859000205994,
      "sampling/importance_sampling_ratio/max": 2.1473188400268555,
      "sampling/importance_sampling_ratio/mean": 1.0528087615966797,
      "sampling/importance_sampling_ratio/min": 0.12547586858272552,
      "sampling/sampling_logp_difference/max": 1.9607396125793457,
      "sampling/sampling_logp_difference/mean": 0.09657371044158936,
      "step": 47,
      "step_time": 12.144370723035536
    },
    {
      "clip_ratio/high_max": 0.015159521019086242,
      "clip_ratio/high_mean": 0.007579760509543121,
      "clip_ratio/low_mean": 0.0029296875,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01050944800954312,
      "completions/clipped_ratio": 0.46875,
      "completions/max_length": 659.0,
      "completions/max_terminated_length": 546.0,
      "completions/mean_length": 218.4375,
      "completions/mean_terminated_length": 148.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.011305171996355,
      "epoch": 0.00096,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.437297821044922,
      "kl": 1.2609674967825413,
      "learning_rate": 9.996439077270269e-06,
      "loss": -0.1812,
      "num_tokens": 1873097.0,
      "reward": 0.2603124976158142,
      "reward_std": 0.8108988404273987,
      "rewards/rollout_reward_func/mean": 0.2603124976158142,
      "rewards/rollout_reward_func/std": 0.8314987421035767,
      "sampling/importance_sampling_ratio/max": 2.387929677963257,
      "sampling/importance_sampling_ratio/mean": 1.0243021249771118,
      "sampling/importance_sampling_ratio/min": 0.18960832059383392,
      "sampling/sampling_logp_difference/max": 1.627047061920166,
      "sampling/sampling_logp_difference/mean": 0.08398944139480591,
      "step": 48,
      "step_time": 13.225894877992687
    },
    {
      "clip_ratio/high_max": 0.010416666744276881,
      "clip_ratio/high_mean": 0.005208333372138441,
      "clip_ratio/low_mean": 0.013392857741564512,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018601191113702953,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 617.0,
      "completions/max_terminated_length": 617.0,
      "completions/mean_length": 200.21875,
      "completions/mean_terminated_length": 188.70834350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1909760311245918,
      "epoch": 0.00098,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2060630321502686,
      "kl": 1.0443120293784887,
      "learning_rate": 9.995820976366208e-06,
      "loss": -0.5695,
      "num_tokens": 1913451.0,
      "reward": -0.10987994074821472,
      "reward_std": 1.0004258155822754,
      "rewards/rollout_reward_func/mean": -0.10987994074821472,
      "rewards/rollout_reward_func/std": 0.9845090508460999,
      "sampling/importance_sampling_ratio/max": 1.8609850406646729,
      "sampling/importance_sampling_ratio/mean": 0.7646557092666626,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9108220338821411,
      "sampling/sampling_logp_difference/mean": 0.0973869264125824,
      "step": 49,
      "step_time": 12.05092378500558
    },
    {
      "clip_ratio/high_max": 0.009895833441987634,
      "clip_ratio/high_mean": 0.005924479220993817,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010388765134848654,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 664.0,
      "completions/max_terminated_length": 664.0,
      "completions/mean_length": 236.65625,
      "completions/mean_terminated_length": 200.60000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.035363145172596,
      "epoch": 0.001,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2321982383728027,
      "kl": 0.40543810930103064,
      "learning_rate": 9.995153465551283e-06,
      "loss": -0.1918,
      "num_tokens": 1954132.0,
      "reward": 0.3022550046443939,
      "reward_std": 0.8429325819015503,
      "rewards/rollout_reward_func/mean": 0.3022550046443939,
      "rewards/rollout_reward_func/std": 0.840745210647583,
      "sampling/importance_sampling_ratio/max": 2.0344431400299072,
      "sampling/importance_sampling_ratio/mean": 0.9563087821006775,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 22.298357009887695,
      "sampling/sampling_logp_difference/mean": 0.1731124371290207,
      "step": 50,
      "step_time": 12.619984562013997
    },
    {
      "clip_ratio/high_max": 0.0020833334419876337,
      "clip_ratio/high_mean": 0.0010416667209938169,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002018229220993817,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 620.0,
      "completions/max_terminated_length": 582.0,
      "completions/mean_length": 202.5,
      "completions/mean_terminated_length": 143.8260955810547,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1524967774748802,
      "epoch": 0.00102,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.267749071121216,
      "kl": 2.382586153689772,
      "learning_rate": 9.994436553630411e-06,
      "loss": -0.1767,
      "num_tokens": 1992989.0,
      "reward": 0.12416569143533707,
      "reward_std": 0.9674448370933533,
      "rewards/rollout_reward_func/mean": 0.12416569143533707,
      "rewards/rollout_reward_func/std": 0.9420199394226074,
      "sampling/importance_sampling_ratio/max": 2.2858333587646484,
      "sampling/importance_sampling_ratio/mean": 0.9116540551185608,
      "sampling/importance_sampling_ratio/min": 3.1440880628166994e-21,
      "sampling/sampling_logp_difference/max": 20.789052963256836,
      "sampling/sampling_logp_difference/mean": 0.18523873388767242,
      "step": 51,
      "step_time": 12.818164225012879
    },
    {
      "clip_ratio/high_max": 0.059133184840902686,
      "clip_ratio/high_mean": 0.029566592420451343,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.030543154920451343,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 588.0,
      "completions/max_terminated_length": 588.0,
      "completions/mean_length": 175.84375,
      "completions/mean_terminated_length": 150.68182373046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.039426688104868,
      "epoch": 0.00104,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.307471752166748,
      "kl": 0.4292449204949662,
      "learning_rate": 9.993670250060152e-06,
      "loss": -0.375,
      "num_tokens": 2031083.0,
      "reward": 0.3758564591407776,
      "reward_std": 0.7202433943748474,
      "rewards/rollout_reward_func/mean": 0.3758564591407776,
      "rewards/rollout_reward_func/std": 0.9132484793663025,
      "sampling/importance_sampling_ratio/max": 2.4312193393707275,
      "sampling/importance_sampling_ratio/mean": 0.9176362156867981,
      "sampling/importance_sampling_ratio/min": 2.832834070432e-06,
      "sampling/sampling_logp_difference/max": 3.5390875339508057,
      "sampling/sampling_logp_difference/mean": 0.1186237633228302,
      "step": 52,
      "step_time": 12.418085069017252
    },
    {
      "clip_ratio/high_max": 0.009070513071492314,
      "clip_ratio/high_mean": 0.004535256535746157,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008999542449600995,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 515.0,
      "completions/max_terminated_length": 515.0,
      "completions/mean_length": 188.46875,
      "completions/mean_terminated_length": 194.629638671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9173995554447174,
      "epoch": 0.00106,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.623557686805725,
      "kl": 1.4017960086930543,
      "learning_rate": 9.99285456494856e-06,
      "loss": -0.3493,
      "num_tokens": 2070874.0,
      "reward": 0.006554782390594482,
      "reward_std": 0.9577794075012207,
      "rewards/rollout_reward_func/mean": 0.006554782390594482,
      "rewards/rollout_reward_func/std": 0.9720827341079712,
      "sampling/importance_sampling_ratio/max": 2.751818895339966,
      "sampling/importance_sampling_ratio/mean": 0.7605128884315491,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.359661817550659,
      "sampling/sampling_logp_difference/mean": 0.10347742587327957,
      "step": 53,
      "step_time": 11.920965786965098
    },
    {
      "clip_ratio/high_max": 0.022265625186264515,
      "clip_ratio/high_mean": 0.011132812593132257,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011132812593132257,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 706.0,
      "completions/max_terminated_length": 608.0,
      "completions/mean_length": 196.90625,
      "completions/mean_terminated_length": 67.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0331540182232857,
      "epoch": 0.00108,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.1346065998077393,
      "kl": 0.14310680143535137,
      "learning_rate": 9.991989509055075e-06,
      "loss": -0.2322,
      "num_tokens": 2107650.0,
      "reward": 0.36687496304512024,
      "reward_std": 0.6656548976898193,
      "rewards/rollout_reward_func/mean": 0.36687496304512024,
      "rewards/rollout_reward_func/std": 0.7554380893707275,
      "sampling/importance_sampling_ratio/max": 2.575469970703125,
      "sampling/importance_sampling_ratio/mean": 1.1061111688613892,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 20.34263801574707,
      "sampling/sampling_logp_difference/mean": 0.1733194887638092,
      "step": 54,
      "step_time": 13.705937975959387
    },
    {
      "clip_ratio/high_max": 0.009949758183211088,
      "clip_ratio/high_mean": 0.004974879091605544,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005951441591605544,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 671.0,
      "completions/max_terminated_length": 671.0,
      "completions/mean_length": 260.65625,
      "completions/mean_terminated_length": 203.22222900390625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.94914985075593,
      "epoch": 0.0011,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0821571350097656,
      "kl": 1.1788070937618613,
      "learning_rate": 9.991075093790372e-06,
      "loss": -0.1681,
      "num_tokens": 2149418.0,
      "reward": 0.375,
      "reward_std": 0.8361747860908508,
      "rewards/rollout_reward_func/mean": 0.375,
      "rewards/rollout_reward_func/std": 0.8235994577407837,
      "sampling/importance_sampling_ratio/max": 2.2083704471588135,
      "sampling/importance_sampling_ratio/mean": 0.9341359734535217,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.738619327545166,
      "sampling/sampling_logp_difference/mean": 0.08796980232000351,
      "step": 55,
      "step_time": 12.622455194985378
    },
    {
      "clip_ratio/high_max": 0.018880208488553762,
      "clip_ratio/high_mean": 0.009440104244276881,
      "clip_ratio/low_mean": 0.002994791720993817,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012434895848855376,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 608.0,
      "completions/mean_length": 221.4375,
      "completions/mean_terminated_length": 240.3000030517578,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1795061714947224,
      "epoch": 0.00112,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3641784191131592,
      "kl": 0.5376564711332321,
      "learning_rate": 9.990111331216204e-06,
      "loss": -0.1326,
      "num_tokens": 2189552.0,
      "reward": 0.40312498807907104,
      "reward_std": 0.7786563634872437,
      "rewards/rollout_reward_func/mean": 0.40312498807907104,
      "rewards/rollout_reward_func/std": 0.8088200688362122,
      "sampling/importance_sampling_ratio/max": 1.8173494338989258,
      "sampling/importance_sampling_ratio/mean": 0.6639854907989502,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 24.921098709106445,
      "sampling/sampling_logp_difference/mean": 0.1388252079486847,
      "step": 56,
      "step_time": 12.920008219021838
    },
    {
      "clip_ratio/high_max": 0.01953125,
      "clip_ratio/high_mean": 0.009765625,
      "clip_ratio/low_mean": 0.0020926339784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01185825897846371,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 689.0,
      "completions/max_terminated_length": 689.0,
      "completions/mean_length": 213.28125,
      "completions/mean_terminated_length": 213.76190185546875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.961669560521841,
      "epoch": 0.00114,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.571563720703125,
      "kl": 0.7163866071496159,
      "learning_rate": 9.989098234045254e-06,
      "loss": 0.0373,
      "num_tokens": 2229890.0,
      "reward": 0.29569011926651,
      "reward_std": 0.9190771579742432,
      "rewards/rollout_reward_func/mean": 0.29569011926651,
      "rewards/rollout_reward_func/std": 0.8838240504264832,
      "sampling/importance_sampling_ratio/max": 2.887221574783325,
      "sampling/importance_sampling_ratio/mean": 0.9558820724487305,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.4540443420410156,
      "sampling/sampling_logp_difference/mean": 0.10127431154251099,
      "step": 57,
      "step_time": 13.012004989970592
    },
    {
      "clip_ratio/high_max": 0.02176339295692742,
      "clip_ratio/high_mean": 0.012526433332823217,
      "clip_ratio/low_mean": 0.0031317349057644606,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015658168238587677,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 270.53125,
      "completions/mean_terminated_length": 186.1666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9453736878931522,
      "epoch": 0.00116,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4450411796569824,
      "kl": 0.9419204645091668,
      "learning_rate": 9.988035815640953e-06,
      "loss": -0.1407,
      "num_tokens": 2271766.0,
      "reward": 0.34562498331069946,
      "reward_std": 0.8157999515533447,
      "rewards/rollout_reward_func/mean": 0.34562498331069946,
      "rewards/rollout_reward_func/std": 0.807844340801239,
      "sampling/importance_sampling_ratio/max": 2.9548535346984863,
      "sampling/importance_sampling_ratio/mean": 0.9399468898773193,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9999890327453613,
      "sampling/sampling_logp_difference/mean": 0.09195321798324585,
      "step": 58,
      "step_time": 12.663164175959537
    },
    {
      "clip_ratio/high_max": 0.016373311169445515,
      "clip_ratio/high_mean": 0.008186655584722757,
      "clip_ratio/low_mean": 0.0011574074160307646,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009344063000753522,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 652.0,
      "completions/max_terminated_length": 652.0,
      "completions/mean_length": 302.4375,
      "completions/mean_terminated_length": 321.45001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9753099232912064,
      "epoch": 0.00118,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5897505283355713,
      "kl": 0.3615207166876644,
      "learning_rate": 9.986924090017322e-06,
      "loss": -0.2381,
      "num_tokens": 2316344.0,
      "reward": 0.13241125643253326,
      "reward_std": 0.8944839239120483,
      "rewards/rollout_reward_func/mean": 0.13241125643253326,
      "rewards/rollout_reward_func/std": 0.895727276802063,
      "sampling/importance_sampling_ratio/max": 2.3224706649780273,
      "sampling/importance_sampling_ratio/mean": 0.6902827024459839,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.5303726196289062,
      "sampling/sampling_logp_difference/mean": 0.09990397840738297,
      "step": 59,
      "step_time": 13.478291145016556
    },
    {
      "clip_ratio/high_max": 0.015390624990686774,
      "clip_ratio/high_mean": 0.007695312495343387,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007695312495343387,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 700.0,
      "completions/max_terminated_length": 700.0,
      "completions/mean_length": 220.3125,
      "completions/mean_terminated_length": 216.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0352301597595215,
      "epoch": 0.0012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.213038444519043,
      "kl": 0.13710439787246287,
      "learning_rate": 9.985763071838774e-06,
      "loss": -0.059,
      "num_tokens": 2355793.0,
      "reward": 0.41343751549720764,
      "reward_std": 0.646430253982544,
      "rewards/rollout_reward_func/mean": 0.41343751549720764,
      "rewards/rollout_reward_func/std": 0.7645221948623657,
      "sampling/importance_sampling_ratio/max": 2.2757949829101562,
      "sampling/importance_sampling_ratio/mean": 0.8368750810623169,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 21.570724487304688,
      "sampling/sampling_logp_difference/mean": 0.23323720693588257,
      "step": 60,
      "step_time": 12.706386449965066
    },
    {
      "clip_ratio/high_max": 0.070766129065305,
      "clip_ratio/high_mean": 0.0353830645326525,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03761520818807185,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 653.0,
      "completions/mean_length": 266.28125,
      "completions/mean_terminated_length": 212.52174377441406,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8858621306717396,
      "epoch": 0.00122,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.474869728088379,
      "kl": 0.3407137815374881,
      "learning_rate": 9.98455277641992e-06,
      "loss": -0.1562,
      "num_tokens": 2397354.0,
      "reward": 0.546875,
      "reward_std": 0.6911813020706177,
      "rewards/rollout_reward_func/mean": 0.546875,
      "rewards/rollout_reward_func/std": 0.7259006500244141,
      "sampling/importance_sampling_ratio/max": 2.623645067214966,
      "sampling/importance_sampling_ratio/mean": 1.031089425086975,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.5755870342254639,
      "sampling/sampling_logp_difference/mean": 0.0995551124215126,
      "step": 61,
      "step_time": 12.433704744980787
    },
    {
      "clip_ratio/high_max": 0.013241775333881378,
      "clip_ratio/high_mean": 0.006620887666940689,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008574012666940689,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 683.0,
      "completions/mean_length": 204.78125,
      "completions/mean_terminated_length": 171.1999969482422,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9909212440252304,
      "epoch": 0.00124,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.78271484375,
      "kl": 0.9711711439304054,
      "learning_rate": 9.983293219725379e-06,
      "loss": 0.1997,
      "num_tokens": 2436557.0,
      "reward": 0.5012500286102295,
      "reward_std": 0.6808013916015625,
      "rewards/rollout_reward_func/mean": 0.5012500286102295,
      "rewards/rollout_reward_func/std": 0.7392618060112,
      "sampling/importance_sampling_ratio/max": 2.975165367126465,
      "sampling/importance_sampling_ratio/mean": 0.8823078274726868,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.1972947120666504,
      "sampling/sampling_logp_difference/mean": 0.10405464470386505,
      "step": 62,
      "step_time": 13.425610810998478
    },
    {
      "clip_ratio/high_max": 0.004872563760727644,
      "clip_ratio/high_mean": 0.002436281880363822,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004389406880363822,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 596.0,
      "completions/mean_length": 306.5,
      "completions/mean_terminated_length": 213.3333282470703,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8561898916959763,
      "epoch": 0.00126,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5629407167434692,
      "kl": 0.2325179111212492,
      "learning_rate": 9.981984418369552e-06,
      "loss": -0.3027,
      "num_tokens": 2480112.0,
      "reward": 0.37476465106010437,
      "reward_std": 0.8081490993499756,
      "rewards/rollout_reward_func/mean": 0.37476465106010437,
      "rewards/rollout_reward_func/std": 0.775314211845398,
      "sampling/importance_sampling_ratio/max": 2.6873745918273926,
      "sampling/importance_sampling_ratio/mean": 0.752242386341095,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.3590269088745117,
      "sampling/sampling_logp_difference/mean": 0.09907381236553192,
      "step": 63,
      "step_time": 13.095424739003647
    },
    {
      "clip_ratio/high_max": 0.014973958488553762,
      "clip_ratio/high_mean": 0.007486979244276881,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007486979244276881,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 673.0,
      "completions/max_terminated_length": 384.0,
      "completions/mean_length": 191.1875,
      "completions/mean_terminated_length": 89.2272720336914,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8625967614352703,
      "epoch": 0.00128,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.89760160446167,
      "kl": 4.500828737160191,
      "learning_rate": 9.980626389616414e-06,
      "loss": -0.1461,
      "num_tokens": 2518828.0,
      "reward": 0.371978223323822,
      "reward_std": 0.9388980865478516,
      "rewards/rollout_reward_func/mean": 0.371978223323822,
      "rewards/rollout_reward_func/std": 0.896711528301239,
      "sampling/importance_sampling_ratio/max": 2.9135513305664062,
      "sampling/importance_sampling_ratio/mean": 0.885046124458313,
      "sampling/importance_sampling_ratio/min": 1.5349984738149969e-12,
      "sampling/sampling_logp_difference/max": 19.479324340820312,
      "sampling/sampling_logp_difference/mean": 0.1780552864074707,
      "step": 64,
      "step_time": 12.560219528022571
    },
    {
      "clip_ratio/high_max": 0.011067708488553762,
      "clip_ratio/high_mean": 0.005533854244276881,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005533854244276881,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 643.0,
      "completions/max_terminated_length": 643.0,
      "completions/mean_length": 271.53125,
      "completions/mean_terminated_length": 260.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8901497535407543,
      "epoch": 0.0013,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.263439655303955,
      "kl": 1.8229574370197952,
      "learning_rate": 9.979219151379285e-06,
      "loss": -0.1698,
      "num_tokens": 2561781.0,
      "reward": 0.3394154906272888,
      "reward_std": 0.8372423648834229,
      "rewards/rollout_reward_func/mean": 0.3394154906272888,
      "rewards/rollout_reward_func/std": 0.8138791918754578,
      "sampling/importance_sampling_ratio/max": 2.1945595741271973,
      "sampling/importance_sampling_ratio/mean": 0.7200146913528442,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.2597999572753906,
      "sampling/sampling_logp_difference/mean": 0.10942533612251282,
      "step": 65,
      "step_time": 13.629006222967291
    },
    {
      "clip_ratio/high_max": 0.013671875,
      "clip_ratio/high_mean": 0.0068359375,
      "clip_ratio/low_mean": 0.0032738096779212356,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010109747177921236,
      "completions/clipped_ratio": 0.40625,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 646.0,
      "completions/mean_length": 231.09375,
      "completions/mean_terminated_length": 215.63157653808594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9038160145282745,
      "epoch": 0.00132,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.857741594314575,
      "kl": 0.7688989997841418,
      "learning_rate": 9.977762722220585e-06,
      "loss": -0.4312,
      "num_tokens": 2602607.0,
      "reward": 0.2881249785423279,
      "reward_std": 0.9365736842155457,
      "rewards/rollout_reward_func/mean": 0.2881249785423279,
      "rewards/rollout_reward_func/std": 0.9431191086769104,
      "sampling/importance_sampling_ratio/max": 2.709365129470825,
      "sampling/importance_sampling_ratio/mean": 0.7999600172042847,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 20.481657028198242,
      "sampling/sampling_logp_difference/mean": 0.25599008798599243,
      "step": 66,
      "step_time": 12.542682128972956
    },
    {
      "clip_ratio/high_max": 0.027556818444281816,
      "clip_ratio/high_mean": 0.013778409222140908,
      "clip_ratio/low_mean": 0.0035156250232830644,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017294034245423973,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 695.0,
      "completions/max_terminated_length": 695.0,
      "completions/mean_length": 190.0625,
      "completions/mean_terminated_length": 201.1666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8019825387746096,
      "epoch": 0.00134,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5905554294586182,
      "kl": 2.8811870981007814,
      "learning_rate": 9.976257121351602e-06,
      "loss": -0.1229,
      "num_tokens": 2640426.0,
      "reward": 0.5129157304763794,
      "reward_std": 0.7467684745788574,
      "rewards/rollout_reward_func/mean": 0.5129157304763794,
      "rewards/rollout_reward_func/std": 0.8029112815856934,
      "sampling/importance_sampling_ratio/max": 2.734023332595825,
      "sampling/importance_sampling_ratio/mean": 0.9010581374168396,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.2284789085388184,
      "sampling/sampling_logp_difference/mean": 0.11680151522159576,
      "step": 67,
      "step_time": 12.198405891962466
    },
    {
      "clip_ratio/high_max": 0.022135416977107525,
      "clip_ratio/high_mean": 0.011067708488553762,
      "clip_ratio/low_mean": 0.00555889424867928,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0166266025044024,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 633.0,
      "completions/max_terminated_length": 595.0,
      "completions/mean_length": 149.3125,
      "completions/mean_terminated_length": 164.63636779785156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8003146685659885,
      "epoch": 0.00136,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0209877490997314,
      "kl": 0.5229525612667203,
      "learning_rate": 9.974702368632228e-06,
      "loss": -0.1678,
      "num_tokens": 2677224.0,
      "reward": 0.7565624713897705,
      "reward_std": 0.4051898717880249,
      "rewards/rollout_reward_func/mean": 0.7565624713897705,
      "rewards/rollout_reward_func/std": 0.49128279089927673,
      "sampling/importance_sampling_ratio/max": 2.775141716003418,
      "sampling/importance_sampling_ratio/mean": 1.0002084970474243,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.6006546020507812,
      "sampling/sampling_logp_difference/mean": 0.12797600030899048,
      "step": 68,
      "step_time": 13.030739766996703
    },
    {
      "clip_ratio/high_max": 0.005859375,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.004179687472060323,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007109374972060323,
      "completions/clipped_ratio": 0.53125,
      "completions/max_length": 688.0,
      "completions/max_terminated_length": 688.0,
      "completions/mean_length": 236.625,
      "completions/mean_terminated_length": 242.53334045410156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8374966233968735,
      "epoch": 0.00138,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.375986337661743,
      "kl": 1.590686159208417,
      "learning_rate": 9.973098484570702e-06,
      "loss": -0.4082,
      "num_tokens": 2718164.0,
      "reward": 0.41843751072883606,
      "reward_std": 0.7843682765960693,
      "rewards/rollout_reward_func/mean": 0.41843751072883606,
      "rewards/rollout_reward_func/std": 0.8153926730155945,
      "sampling/importance_sampling_ratio/max": 2.8736321926116943,
      "sampling/importance_sampling_ratio/mean": 0.8742094039916992,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.0816571712493896,
      "sampling/sampling_logp_difference/mean": 0.10882984101772308,
      "step": 69,
      "step_time": 12.666442746005487
    },
    {
      "clip_ratio/high_max": 0.036771616665646434,
      "clip_ratio/high_mean": 0.021315495832823217,
      "clip_ratio/low_mean": 0.0010416667209938169,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022357162553817034,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 584.0,
      "completions/mean_length": 232.53125,
      "completions/mean_terminated_length": 189.6666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8198818881064653,
      "epoch": 0.0014,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.1643879413604736,
      "kl": 0.48179814126342535,
      "learning_rate": 9.97144549032334e-06,
      "loss": -0.0138,
      "num_tokens": 2759234.0,
      "reward": 0.6484375,
      "reward_std": 0.5274016857147217,
      "rewards/rollout_reward_func/mean": 0.6484375,
      "rewards/rollout_reward_func/std": 0.5904132127761841,
      "sampling/importance_sampling_ratio/max": 2.703444242477417,
      "sampling/importance_sampling_ratio/mean": 0.9320708513259888,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9965975284576416,
      "sampling/sampling_logp_difference/mean": 0.12269452959299088,
      "step": 70,
      "step_time": 12.979443570991862
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.00390625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0048828125,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 180.875,
      "completions/mean_terminated_length": 178.59091186523438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7967657577246428,
      "epoch": 0.00142,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.351608991622925,
      "kl": 0.8479240862652659,
      "learning_rate": 9.969743407694254e-06,
      "loss": -0.0997,
      "num_tokens": 2797264.0,
      "reward": 0.553827166557312,
      "reward_std": 0.8138425350189209,
      "rewards/rollout_reward_func/mean": 0.553827166557312,
      "rewards/rollout_reward_func/std": 0.7898650765419006,
      "sampling/importance_sampling_ratio/max": 2.6495373249053955,
      "sampling/importance_sampling_ratio/mean": 0.832921028137207,
      "sampling/importance_sampling_ratio/min": 0.08351821452379227,
      "sampling/sampling_logp_difference/max": 1.8291492462158203,
      "sampling/sampling_logp_difference/mean": 0.12324561178684235,
      "step": 71,
      "step_time": 12.694148611961282
    },
    {
      "clip_ratio/high_max": 0.02855145698413253,
      "clip_ratio/high_mean": 0.014275728492066264,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016228853492066264,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 704.0,
      "completions/max_terminated_length": 684.0,
      "completions/mean_length": 228.53125,
      "completions/mean_terminated_length": 172.0,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.6983378380537033,
      "epoch": 0.00144,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.954265832901001,
      "kl": 0.30631584487855434,
      "learning_rate": 9.967992259135063e-06,
      "loss": -0.3238,
      "num_tokens": 2838247.0,
      "reward": 0.5742862224578857,
      "reward_std": 0.5754244327545166,
      "rewards/rollout_reward_func/mean": 0.5742862224578857,
      "rewards/rollout_reward_func/std": 0.5693974494934082,
      "sampling/importance_sampling_ratio/max": 2.944709300994873,
      "sampling/importance_sampling_ratio/mean": 0.9444066286087036,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 22.735857009887695,
      "sampling/sampling_logp_difference/mean": 0.2083752155303955,
      "step": 72,
      "step_time": 12.713693763958872
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0012019231216982007,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0012019231216982007,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 681.0,
      "completions/max_terminated_length": 681.0,
      "completions/mean_length": 261.65625,
      "completions/mean_terminated_length": 290.5769348144531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7697948589920998,
      "epoch": 0.00146,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7889959812164307,
      "kl": 0.8839123575016856,
      "learning_rate": 9.9661920677446e-06,
      "loss": 0.0362,
      "num_tokens": 2879263.0,
      "reward": 0.4964737296104431,
      "reward_std": 0.6955374479293823,
      "rewards/rollout_reward_func/mean": 0.4964737296104431,
      "rewards/rollout_reward_func/std": 0.7395352125167847,
      "sampling/importance_sampling_ratio/max": 2.2933902740478516,
      "sampling/importance_sampling_ratio/mean": 0.9242541790008545,
      "sampling/importance_sampling_ratio/min": 0.15822435915470123,
      "sampling/sampling_logp_difference/max": 1.8012161254882812,
      "sampling/sampling_logp_difference/mean": 0.10938294231891632,
      "step": 73,
      "step_time": 12.798804676014697
    },
    {
      "clip_ratio/high_max": 0.007634943351149559,
      "clip_ratio/high_mean": 0.0038174716755747795,
      "clip_ratio/low_mean": 0.004248903598636389,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008066375274211168,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 638.0,
      "completions/max_terminated_length": 633.0,
      "completions/mean_length": 313.78125,
      "completions/mean_terminated_length": 308.5714416503906,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6707625761628151,
      "epoch": 0.00148,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9844763278961182,
      "kl": 0.567382971290499,
      "learning_rate": 9.964342857268609e-06,
      "loss": -0.3504,
      "num_tokens": 2923027.0,
      "reward": 0.4493773877620697,
      "reward_std": 0.7608656883239746,
      "rewards/rollout_reward_func/mean": 0.4493773877620697,
      "rewards/rollout_reward_func/std": 0.8006519079208374,
      "sampling/importance_sampling_ratio/max": 1.9908028841018677,
      "sampling/importance_sampling_ratio/mean": 0.8166816234588623,
      "sampling/importance_sampling_ratio/min": 0.12356176227331161,
      "sampling/sampling_logp_difference/max": 1.8197897672653198,
      "sampling/sampling_logp_difference/mean": 0.10586657375097275,
      "step": 74,
      "step_time": 13.361965022966615
    },
    {
      "clip_ratio/high_max": 0.008370535913854837,
      "clip_ratio/high_mean": 0.0056057225447148085,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013418222544714808,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 667.0,
      "completions/mean_length": 251.3125,
      "completions/mean_terminated_length": 236.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7620927523821592,
      "epoch": 0.0015,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3982799053192139,
      "kl": 0.8253440069966018,
      "learning_rate": 9.962444652099422e-06,
      "loss": -0.2392,
      "num_tokens": 2965046.0,
      "reward": 0.6586612462997437,
      "reward_std": 0.5930125117301941,
      "rewards/rollout_reward_func/mean": 0.6586612462997437,
      "rewards/rollout_reward_func/std": 0.6729342937469482,
      "sampling/importance_sampling_ratio/max": 2.4055728912353516,
      "sampling/importance_sampling_ratio/mean": 1.0064224004745483,
      "sampling/importance_sampling_ratio/min": 0.14169640839099884,
      "sampling/sampling_logp_difference/max": 1.8282921314239502,
      "sampling/sampling_logp_difference/mean": 0.1034165620803833,
      "step": 75,
      "step_time": 12.215856175942463
    },
    {
      "clip_ratio/high_max": 0.027198204305022955,
      "clip_ratio/high_mean": 0.01672410243190825,
      "clip_ratio/low_mean": 0.0032402074430137873,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019964309874922037,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 633.0,
      "completions/max_terminated_length": 633.0,
      "completions/mean_length": 228.4375,
      "completions/mean_terminated_length": 229.11111450195312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7328853234648705,
      "epoch": 0.00152,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3260223865509033,
      "kl": 0.5603991616517305,
      "learning_rate": 9.960497477275653e-06,
      "loss": -0.1621,
      "num_tokens": 3005985.0,
      "reward": 0.5231876969337463,
      "reward_std": 0.7136449813842773,
      "rewards/rollout_reward_func/mean": 0.5231876969337463,
      "rewards/rollout_reward_func/std": 0.7197718620300293,
      "sampling/importance_sampling_ratio/max": 2.5988528728485107,
      "sampling/importance_sampling_ratio/mean": 0.7472135424613953,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9123153686523438,
      "sampling/sampling_logp_difference/mean": 0.12195128202438354,
      "step": 76,
      "step_time": 12.63992480802699
    },
    {
      "clip_ratio/high_max": 0.0204315478913486,
      "clip_ratio/high_mean": 0.0102157739456743,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0102157739456743,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 628.0,
      "completions/max_terminated_length": 603.0,
      "completions/mean_length": 218.3125,
      "completions/mean_terminated_length": 173.4166717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7499921750277281,
      "epoch": 0.00154,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.5184857845306396,
      "kl": 6.860477710142732,
      "learning_rate": 9.95850135848185e-06,
      "loss": -0.1547,
      "num_tokens": 3046127.0,
      "reward": 0.4831250309944153,
      "reward_std": 0.6934801340103149,
      "rewards/rollout_reward_func/mean": 0.4831250309944153,
      "rewards/rollout_reward_func/std": 0.8207055926322937,
      "sampling/importance_sampling_ratio/max": 2.429649829864502,
      "sampling/importance_sampling_ratio/mean": 0.8476479649543762,
      "sampling/importance_sampling_ratio/min": 0.06514973193407059,
      "sampling/sampling_logp_difference/max": 2.7263875007629395,
      "sampling/sampling_logp_difference/mean": 0.12946997582912445,
      "step": 77,
      "step_time": 12.557037811973714
    },
    {
      "clip_ratio/high_max": 0.018136160913854837,
      "clip_ratio/high_mean": 0.009068080456927419,
      "clip_ratio/low_mean": 0.0234375,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03250558045692742,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 655.0,
      "completions/max_terminated_length": 655.0,
      "completions/mean_length": 235.40625,
      "completions/mean_terminated_length": 210.92593383789062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7341149486601353,
      "epoch": 0.00156,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2427520751953125,
      "kl": 1.4391632091719657,
      "learning_rate": 9.956456322048174e-06,
      "loss": -0.2573,
      "num_tokens": 3087323.0,
      "reward": 0.6034375429153442,
      "reward_std": 0.6326199769973755,
      "rewards/rollout_reward_func/mean": 0.6034375429153442,
      "rewards/rollout_reward_func/std": 0.7361916303634644,
      "sampling/importance_sampling_ratio/max": 2.654374837875366,
      "sampling/importance_sampling_ratio/mean": 0.9959803819656372,
      "sampling/importance_sampling_ratio/min": 0.16198402643203735,
      "sampling/sampling_logp_difference/max": 1.6022305488586426,
      "sampling/sampling_logp_difference/mean": 0.10324783623218536,
      "step": 78,
      "step_time": 12.72153523293673
    },
    {
      "clip_ratio/high_max": 0.007195723708719015,
      "clip_ratio/high_mean": 0.0035978618543595076,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0035978618543595076,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 661.0,
      "completions/max_terminated_length": 661.0,
      "completions/mean_length": 235.78125,
      "completions/mean_terminated_length": 244.96429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6718597169965506,
      "epoch": 0.00158,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1995981931686401,
      "kl": 1.2177076172083616,
      "learning_rate": 9.954362394950035e-06,
      "loss": -0.1127,
      "num_tokens": 3128884.0,
      "reward": 0.4243749976158142,
      "reward_std": 0.7534654140472412,
      "rewards/rollout_reward_func/mean": 0.4243749976158142,
      "rewards/rollout_reward_func/std": 0.7597874999046326,
      "sampling/importance_sampling_ratio/max": 2.5804975032806396,
      "sampling/importance_sampling_ratio/mean": 0.7397943139076233,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 22.11829948425293,
      "sampling/sampling_logp_difference/mean": 0.20973658561706543,
      "step": 79,
      "step_time": 13.134983102994738
    },
    {
      "clip_ratio/high_max": 0.013237847248092294,
      "clip_ratio/high_mean": 0.006618923624046147,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006618923624046147,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 167.25,
      "completions/mean_terminated_length": 161.67857360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6738289967179298,
      "epoch": 0.0016,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6807355880737305,
      "kl": 0.5036687101237476,
      "learning_rate": 9.952219604807746e-06,
      "loss": -0.1995,
      "num_tokens": 3166527.0,
      "reward": 0.5837500095367432,
      "reward_std": 0.5173101425170898,
      "rewards/rollout_reward_func/mean": 0.5837500095367432,
      "rewards/rollout_reward_func/std": 0.6248083114624023,
      "sampling/importance_sampling_ratio/max": 2.014082670211792,
      "sampling/importance_sampling_ratio/mean": 0.9195514917373657,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 21.893178939819336,
      "sampling/sampling_logp_difference/mean": 0.2006417065858841,
      "step": 80,
      "step_time": 12.076362965002772
    },
    {
      "clip_ratio/high_max": 0.010731938760727644,
      "clip_ratio/high_mean": 0.005365969380363822,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005365969380363822,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 648.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 196.625,
      "completions/mean_terminated_length": 188.4615478515625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8147512897849083,
      "epoch": 0.00162,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5948408842086792,
      "kl": 2.992323379032314,
      "learning_rate": 9.950027979886159e-06,
      "loss": -0.1701,
      "num_tokens": 3205137.0,
      "reward": 0.18991126120090485,
      "reward_std": 0.8059802055358887,
      "rewards/rollout_reward_func/mean": 0.18991126120090485,
      "rewards/rollout_reward_func/std": 0.8809428811073303,
      "sampling/importance_sampling_ratio/max": 2.6819353103637695,
      "sampling/importance_sampling_ratio/mean": 0.7559325695037842,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 5.223290920257568,
      "sampling/sampling_logp_difference/mean": 0.15525344014167786,
      "step": 81,
      "step_time": 12.11735813798441
    },
    {
      "clip_ratio/high_max": 0.0334755782969296,
      "clip_ratio/high_mean": 0.0167377891484648,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0167377891484648,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 573.0,
      "completions/max_terminated_length": 573.0,
      "completions/mean_length": 173.625,
      "completions/mean_terminated_length": 174.34483337402344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7447240278124809,
      "epoch": 0.00164,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1534223556518555,
      "kl": 1.0318080270662904,
      "learning_rate": 9.947787549094288e-06,
      "loss": -0.1915,
      "num_tokens": 3241935.0,
      "reward": 0.7315624952316284,
      "reward_std": 0.4662872850894928,
      "rewards/rollout_reward_func/mean": 0.7315624952316284,
      "rewards/rollout_reward_func/std": 0.5894289612770081,
      "sampling/importance_sampling_ratio/max": 2.7592639923095703,
      "sampling/importance_sampling_ratio/mean": 1.087034821510315,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 20.11916732788086,
      "sampling/sampling_logp_difference/mean": 0.2131710797548294,
      "step": 82,
      "step_time": 13.113543270985247
    },
    {
      "clip_ratio/high_max": 0.023828125093132257,
      "clip_ratio/high_mean": 0.011914062546566129,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011914062546566129,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 629.0,
      "completions/max_terminated_length": 561.0,
      "completions/mean_length": 209.34375,
      "completions/mean_terminated_length": 214.75999450683594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8027353100478649,
      "epoch": 0.00166,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0700441598892212,
      "kl": 0.4077621651813388,
      "learning_rate": 9.945498341984929e-06,
      "loss": -0.1294,
      "num_tokens": 3280582.0,
      "reward": 0.4577237367630005,
      "reward_std": 0.7519221305847168,
      "rewards/rollout_reward_func/mean": 0.4577237367630005,
      "rewards/rollout_reward_func/std": 0.8285698890686035,
      "sampling/importance_sampling_ratio/max": 2.8300137519836426,
      "sampling/importance_sampling_ratio/mean": 0.9165946841239929,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.4819607734680176,
      "sampling/sampling_logp_difference/mean": 0.11464811861515045,
      "step": 83,
      "step_time": 11.90185412199935
    },
    {
      "clip_ratio/high_max": 0.010291820857673883,
      "clip_ratio/high_mean": 0.0065663650166243315,
      "clip_ratio/low_mean": 0.0017361111240461469,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008302476140670478,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 314.40625,
      "completions/mean_terminated_length": 336.7200012207031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8331865593791008,
      "epoch": 0.00168,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2566524744033813,
      "kl": 0.6445068642497063,
      "learning_rate": 9.943160388754274e-06,
      "loss": -0.2064,
      "num_tokens": 3324385.0,
      "reward": 0.3810194730758667,
      "reward_std": 0.7791692018508911,
      "rewards/rollout_reward_func/mean": 0.3810194730758667,
      "rewards/rollout_reward_func/std": 0.8424921035766602,
      "sampling/importance_sampling_ratio/max": 2.2634100914001465,
      "sampling/importance_sampling_ratio/mean": 0.7179601192474365,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 22.815942764282227,
      "sampling/sampling_logp_difference/mean": 0.17926472425460815,
      "step": 84,
      "step_time": 12.520836708965362
    },
    {
      "clip_ratio/high_max": 0.010655899299308658,
      "clip_ratio/high_mean": 0.006529872654937208,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006529872654937208,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 671.0,
      "completions/max_terminated_length": 671.0,
      "completions/mean_length": 244.84375,
      "completions/mean_terminated_length": 244.59259033203125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7324507683515549,
      "epoch": 0.0017,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6409313678741455,
      "kl": 0.4419076587073505,
      "learning_rate": 9.940773720241503e-06,
      "loss": -0.2416,
      "num_tokens": 3366066.0,
      "reward": 0.6290624737739563,
      "reward_std": 0.7171441316604614,
      "rewards/rollout_reward_func/mean": 0.6290624737739563,
      "rewards/rollout_reward_func/std": 0.7091674208641052,
      "sampling/importance_sampling_ratio/max": 2.609598398208618,
      "sampling/importance_sampling_ratio/mean": 1.0305142402648926,
      "sampling/importance_sampling_ratio/min": 0.0802537277340889,
      "sampling/sampling_logp_difference/max": 1.2795006036758423,
      "sampling/sampling_logp_difference/mean": 0.10222354531288147,
      "step": 85,
      "step_time": 13.678101476980373
    },
    {
      "clip_ratio/high_max": 0.022281280951574445,
      "clip_ratio/high_mean": 0.011140640475787222,
      "clip_ratio/low_mean": 0.017578125,
      "clip_ratio/low_min": 0.00390625,
      "clip_ratio/region_mean": 0.028718765475787222,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 710.0,
      "completions/max_terminated_length": 599.0,
      "completions/mean_length": 171.09375,
      "completions/mean_terminated_length": 136.95652770996094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7768513895571232,
      "epoch": 0.00172,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3423579931259155,
      "kl": 2.1093540601432323,
      "learning_rate": 9.938338367928391e-06,
      "loss": -0.1843,
      "num_tokens": 3404125.0,
      "reward": 0.36522376537323,
      "reward_std": 0.8468011617660522,
      "rewards/rollout_reward_func/mean": 0.36522376537323,
      "rewards/rollout_reward_func/std": 0.8413136005401611,
      "sampling/importance_sampling_ratio/max": 2.9960618019104004,
      "sampling/importance_sampling_ratio/mean": 0.7917765378952026,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.095489501953125,
      "sampling/sampling_logp_difference/mean": 0.12895825505256653,
      "step": 86,
      "step_time": 12.200203202010016
    },
    {
      "clip_ratio/high_max": 0.007378472248092294,
      "clip_ratio/high_mean": 0.003689236124046147,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003689236124046147,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 655.0,
      "completions/max_terminated_length": 655.0,
      "completions/mean_length": 209.84375,
      "completions/mean_terminated_length": 212.82144165039062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8031869065016508,
      "epoch": 0.00174,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6591393947601318,
      "kl": 0.8687427015975118,
      "learning_rate": 9.935854363938876e-06,
      "loss": -0.162,
      "num_tokens": 3444135.0,
      "reward": 0.4231273829936981,
      "reward_std": 0.8604464530944824,
      "rewards/rollout_reward_func/mean": 0.4231273829936981,
      "rewards/rollout_reward_func/std": 0.8474411964416504,
      "sampling/importance_sampling_ratio/max": 2.6027350425720215,
      "sampling/importance_sampling_ratio/mean": 0.8642632961273193,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.482572078704834,
      "sampling/sampling_logp_difference/mean": 0.12166670709848404,
      "step": 87,
      "step_time": 12.058846924992395
    },
    {
      "clip_ratio/high_max": 0.0067471591755747795,
      "clip_ratio/high_mean": 0.0033735795877873898,
      "clip_ratio/low_mean": 0.033333333441987634,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03670691302977502,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 250.625,
      "completions/mean_terminated_length": 255.92593383789062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7056362517178059,
      "epoch": 0.00176,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.673019528388977,
      "kl": 2.419476170092821,
      "learning_rate": 9.933321741038655e-06,
      "loss": -0.2967,
      "num_tokens": 3486189.0,
      "reward": 0.5303751826286316,
      "reward_std": 0.7324895858764648,
      "rewards/rollout_reward_func/mean": 0.5303751826286316,
      "rewards/rollout_reward_func/std": 0.7408851981163025,
      "sampling/importance_sampling_ratio/max": 2.8106367588043213,
      "sampling/importance_sampling_ratio/mean": 1.1917905807495117,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.764371395111084,
      "sampling/sampling_logp_difference/mean": 0.1021614670753479,
      "step": 88,
      "step_time": 13.112865573988529
    },
    {
      "clip_ratio/high_max": 0.006310096243396401,
      "clip_ratio/high_mean": 0.0031550481216982007,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031550481216982007,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 240.375,
      "completions/mean_terminated_length": 237.13043212890625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7641278728842735,
      "epoch": 0.00178,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7174426317214966,
      "kl": 0.5125272599980235,
      "learning_rate": 9.930740532634733e-06,
      "loss": -0.463,
      "num_tokens": 3526568.0,
      "reward": 0.5509804487228394,
      "reward_std": 0.7659791111946106,
      "rewards/rollout_reward_func/mean": 0.5509804487228394,
      "rewards/rollout_reward_func/std": 0.7403309345245361,
      "sampling/importance_sampling_ratio/max": 2.771024227142334,
      "sampling/importance_sampling_ratio/mean": 1.2283291816711426,
      "sampling/importance_sampling_ratio/min": 0.09599928557872772,
      "sampling/sampling_logp_difference/max": 1.8884501457214355,
      "sampling/sampling_logp_difference/mean": 0.10920237004756927,
      "step": 89,
      "step_time": 12.248761897018994
    },
    {
      "clip_ratio/high_max": 0.008463541744276881,
      "clip_ratio/high_mean": 0.004231770872138441,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004231770872138441,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1215.0,
      "completions/max_terminated_length": 1054.0,
      "completions/mean_length": 242.28125,
      "completions/mean_terminated_length": 206.92308044433594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7404716908931732,
      "epoch": 0.0018,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2462239265441895,
      "kl": 0.6246154974214733,
      "learning_rate": 9.928110772774995e-06,
      "loss": -0.2455,
      "num_tokens": 3566744.0,
      "reward": 0.5738271474838257,
      "reward_std": 0.6476214528083801,
      "rewards/rollout_reward_func/mean": 0.5738271474838257,
      "rewards/rollout_reward_func/std": 0.7407666444778442,
      "sampling/importance_sampling_ratio/max": 2.7574243545532227,
      "sampling/importance_sampling_ratio/mean": 1.002872347831726,
      "sampling/importance_sampling_ratio/min": 1.4057662801004968e-25,
      "sampling/sampling_logp_difference/max": 26.69582748413086,
      "sampling/sampling_logp_difference/mean": 0.34837689995765686,
      "step": 90,
      "step_time": 15.045528464979725
    },
    {
      "clip_ratio/high_max": 0.011211445089429617,
      "clip_ratio/high_mean": 0.0056057225447148085,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0065822850447148085,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 246.84375,
      "completions/mean_terminated_length": 264.15386962890625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6944178678095341,
      "epoch": 0.00182,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5883519649505615,
      "kl": 1.3103416776284575,
      "learning_rate": 9.92543249614775e-06,
      "loss": -0.2645,
      "num_tokens": 3607493.0,
      "reward": 0.283300518989563,
      "reward_std": 0.8887693881988525,
      "rewards/rollout_reward_func/mean": 0.283300518989563,
      "rewards/rollout_reward_func/std": 0.8880862593650818,
      "sampling/importance_sampling_ratio/max": 2.5734336376190186,
      "sampling/importance_sampling_ratio/mean": 0.9032967686653137,
      "sampling/importance_sampling_ratio/min": 0.05662206560373306,
      "sampling/sampling_logp_difference/max": 2.3724136352539062,
      "sampling/sampling_logp_difference/mean": 0.11580298095941544,
      "step": 91,
      "step_time": 12.980732258001808
    },
    {
      "clip_ratio/high_max": 0.0190328904427588,
      "clip_ratio/high_mean": 0.0095164452213794,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0095164452213794,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 266.3125,
      "completions/mean_terminated_length": 256.10345458984375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7503110319375992,
      "epoch": 0.00184,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.455946922302246,
      "kl": 2.4899320725817233,
      "learning_rate": 9.922705738081279e-06,
      "loss": -0.1819,
      "num_tokens": 3649901.0,
      "reward": 0.30124998092651367,
      "reward_std": 0.8564956188201904,
      "rewards/rollout_reward_func/mean": 0.30124998092651367,
      "rewards/rollout_reward_func/std": 0.8870856761932373,
      "sampling/importance_sampling_ratio/max": 2.345909833908081,
      "sampling/importance_sampling_ratio/mean": 0.7614245414733887,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.130732774734497,
      "sampling/sampling_logp_difference/mean": 0.14194367825984955,
      "step": 92,
      "step_time": 12.325108670993359
    },
    {
      "clip_ratio/high_max": 0.01122106472030282,
      "clip_ratio/high_mean": 0.00561053236015141,
      "clip_ratio/low_mean": 0.0011574074160307646,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006767939776182175,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 624.0,
      "completions/mean_length": 228.4375,
      "completions/mean_terminated_length": 209.33334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6080380193889141,
      "epoch": 0.00186,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3837730884552002,
      "kl": 0.42082537710666656,
      "learning_rate": 9.919930534543365e-06,
      "loss": -0.4066,
      "num_tokens": 3689589.0,
      "reward": 0.46557676792144775,
      "reward_std": 0.7582623958587646,
      "rewards/rollout_reward_func/mean": 0.46557676792144775,
      "rewards/rollout_reward_func/std": 0.8248792290687561,
      "sampling/importance_sampling_ratio/max": 2.5285041332244873,
      "sampling/importance_sampling_ratio/mean": 1.178090214729309,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.4125723838806152,
      "sampling/sampling_logp_difference/mean": 0.0957946851849556,
      "step": 93,
      "step_time": 13.186322327994276
    },
    {
      "clip_ratio/high_max": 0.010716413147747517,
      "clip_ratio/high_mean": 0.005358206573873758,
      "clip_ratio/low_mean": 0.005843211198225617,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011201417772099376,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 661.0,
      "completions/mean_length": 214.15625,
      "completions/mean_terminated_length": 208.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7977820262312889,
      "epoch": 0.00188,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4264417886734009,
      "kl": 0.2735514813102782,
      "learning_rate": 9.917106922140814e-06,
      "loss": -0.1537,
      "num_tokens": 3728819.0,
      "reward": 0.5238126516342163,
      "reward_std": 0.6420454382896423,
      "rewards/rollout_reward_func/mean": 0.5238126516342163,
      "rewards/rollout_reward_func/std": 0.7291461825370789,
      "sampling/importance_sampling_ratio/max": 2.6931703090667725,
      "sampling/importance_sampling_ratio/mean": 0.7968823313713074,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.2306733131408691,
      "sampling/sampling_logp_difference/mean": 0.0947219580411911,
      "step": 94,
      "step_time": 12.477565846013022
    },
    {
      "clip_ratio/high_max": 0.013257576152682304,
      "clip_ratio/high_mean": 0.006628788076341152,
      "clip_ratio/low_mean": 0.024479166720993817,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03110795479733497,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 757.0,
      "completions/max_terminated_length": 757.0,
      "completions/mean_length": 223.28125,
      "completions/mean_terminated_length": 218.61538696289062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7100971452891827,
      "epoch": 0.0019,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9933600425720215,
      "kl": 1.5450700679793954,
      "learning_rate": 9.914234938118988e-06,
      "loss": -0.2451,
      "num_tokens": 3768562.0,
      "reward": 0.40655332803726196,
      "reward_std": 0.8774359226226807,
      "rewards/rollout_reward_func/mean": 0.40655332803726196,
      "rewards/rollout_reward_func/std": 0.8680322766304016,
      "sampling/importance_sampling_ratio/max": 2.857481002807617,
      "sampling/importance_sampling_ratio/mean": 0.8239586353302002,
      "sampling/importance_sampling_ratio/min": 0.11413362622261047,
      "sampling/sampling_logp_difference/max": 1.5585308074951172,
      "sampling/sampling_logp_difference/mean": 0.12393781542778015,
      "step": 95,
      "step_time": 11.988063458004035
    },
    {
      "clip_ratio/high_max": 0.04146739118732512,
      "clip_ratio/high_mean": 0.02073369559366256,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02073369559366256,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 642.0,
      "completions/mean_length": 167.5625,
      "completions/mean_terminated_length": 154.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7700920794159174,
      "epoch": 0.00192,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1747682094573975,
      "kl": 3.163258579093963,
      "learning_rate": 9.911314620361294e-06,
      "loss": -0.2132,
      "num_tokens": 3806508.0,
      "reward": 0.2088271528482437,
      "reward_std": 0.8599115610122681,
      "rewards/rollout_reward_func/mean": 0.2088271528482437,
      "rewards/rollout_reward_func/std": 0.9058598875999451,
      "sampling/importance_sampling_ratio/max": 2.820152997970581,
      "sampling/importance_sampling_ratio/mean": 0.7280029058456421,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.0662856101989746,
      "sampling/sampling_logp_difference/mean": 0.13139429688453674,
      "step": 96,
      "step_time": 12.93814704399847
    },
    {
      "clip_ratio/high_max": 0.020833333721384406,
      "clip_ratio/high_mean": 0.011393229360692203,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013625372317619622,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 883.0,
      "completions/max_terminated_length": 883.0,
      "completions/mean_length": 204.65625,
      "completions/mean_terminated_length": 199.27586364746094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5706040132790804,
      "epoch": 0.00194,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.3406214714050293,
      "kl": 1.284589326940477,
      "learning_rate": 9.908346007388698e-06,
      "loss": -0.2078,
      "num_tokens": 3845865.0,
      "reward": 0.2280791848897934,
      "reward_std": 0.8255096673965454,
      "rewards/rollout_reward_func/mean": 0.2280791848897934,
      "rewards/rollout_reward_func/std": 0.9260514378547668,
      "sampling/importance_sampling_ratio/max": 2.789428949356079,
      "sampling/importance_sampling_ratio/mean": 1.0984184741973877,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.6697278022766113,
      "sampling/sampling_logp_difference/mean": 0.12373289465904236,
      "step": 97,
      "step_time": 12.572131911001634
    },
    {
      "clip_ratio/high_max": 0.006465517450124025,
      "clip_ratio/high_mean": 0.0032327587250620127,
      "clip_ratio/low_mean": 0.0013020833721384406,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004534842097200453,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 697.0,
      "completions/max_terminated_length": 697.0,
      "completions/mean_length": 262.34375,
      "completions/mean_terminated_length": 231.0357208251953,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5886645764112473,
      "epoch": 0.00196,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.318421721458435,
      "kl": 1.5845300648361444,
      "learning_rate": 9.905329138359213e-06,
      "loss": -0.3555,
      "num_tokens": 3887749.0,
      "reward": 0.27195215225219727,
      "reward_std": 0.8258433938026428,
      "rewards/rollout_reward_func/mean": 0.27195215225219727,
      "rewards/rollout_reward_func/std": 0.8419767618179321,
      "sampling/importance_sampling_ratio/max": 2.6245768070220947,
      "sampling/importance_sampling_ratio/mean": 0.9098495244979858,
      "sampling/importance_sampling_ratio/min": 0.14829480648040771,
      "sampling/sampling_logp_difference/max": 1.611341953277588,
      "sampling/sampling_logp_difference/mean": 0.11805026233196259,
      "step": 98,
      "step_time": 12.424927090018173
    },
    {
      "clip_ratio/high_max": 0.030097853858023882,
      "clip_ratio/high_mean": 0.015048926929011941,
      "clip_ratio/low_mean": 0.00390625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01895517692901194,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 698.0,
      "completions/mean_length": 217.875,
      "completions/mean_terminated_length": 239.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7615592665970325,
      "epoch": 0.00198,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.557417392730713,
      "kl": 0.4209024319425225,
      "learning_rate": 9.902264053067381e-06,
      "loss": 0.4631,
      "num_tokens": 3926991.0,
      "reward": 0.6725000143051147,
      "reward_std": 0.4900071620941162,
      "rewards/rollout_reward_func/mean": 0.6725000143051147,
      "rewards/rollout_reward_func/std": 0.5951578617095947,
      "sampling/importance_sampling_ratio/max": 2.9877216815948486,
      "sampling/importance_sampling_ratio/mean": 0.8923699855804443,
      "sampling/importance_sampling_ratio/min": 3.4235528301084262e-22,
      "sampling/sampling_logp_difference/max": 22.605154037475586,
      "sampling/sampling_logp_difference/mean": 0.2666288912296295,
      "step": 99,
      "step_time": 13.540776068984997
    },
    {
      "clip_ratio/high_max": 0.02100745076313615,
      "clip_ratio/high_mean": 0.010503725381568074,
      "clip_ratio/low_mean": 0.0024038462433964014,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012907571624964476,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 253.90625,
      "completions/mean_terminated_length": 228.8518524169922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7094956208020449,
      "epoch": 0.002,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5161370038986206,
      "kl": 0.46191434538923204,
      "learning_rate": 9.899150791943747e-06,
      "loss": 0.0174,
      "num_tokens": 3967917.0,
      "reward": 0.65625,
      "reward_std": 0.4902026653289795,
      "rewards/rollout_reward_func/mean": 0.65625,
      "rewards/rollout_reward_func/std": 0.47861188650131226,
      "sampling/importance_sampling_ratio/max": 2.6472437381744385,
      "sampling/importance_sampling_ratio/mean": 0.859967827796936,
      "sampling/importance_sampling_ratio/min": 3.2179332237606585e-15,
      "sampling/sampling_logp_difference/max": 21.820323944091797,
      "sampling/sampling_logp_difference/mean": 0.16814789175987244,
      "step": 100,
      "step_time": 12.513988562961458
    },
    {
      "clip_ratio/high_max": 0.012823660857975483,
      "clip_ratio/high_mean": 0.0064118304289877415,
      "clip_ratio/low_mean": 0.004199604853056371,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010611435282044113,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 676.0,
      "completions/max_terminated_length": 676.0,
      "completions/mean_length": 188.6875,
      "completions/mean_terminated_length": 188.75999450683594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6434148550033569,
      "epoch": 0.00202,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.553330659866333,
      "kl": 0.37580074719153345,
      "learning_rate": 9.895989396054334e-06,
      "loss": -0.1432,
      "num_tokens": 4007496.0,
      "reward": 0.6418750286102295,
      "reward_std": 0.6180306673049927,
      "rewards/rollout_reward_func/mean": 0.6418750286102295,
      "rewards/rollout_reward_func/std": 0.6490809321403503,
      "sampling/importance_sampling_ratio/max": 2.62202787399292,
      "sampling/importance_sampling_ratio/mean": 0.9788997173309326,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 24.05610466003418,
      "sampling/sampling_logp_difference/mean": 0.21181684732437134,
      "step": 101,
      "step_time": 11.927557628994691
    },
    {
      "clip_ratio/high_max": 0.033517466858029366,
      "clip_ratio/high_mean": 0.016758733429014683,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016758733429014683,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 630.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 248.46875,
      "completions/mean_terminated_length": 253.00001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5771082267165184,
      "epoch": 0.00204,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.901191771030426,
      "kl": 2.413372104987502,
      "learning_rate": 9.892779907100084e-06,
      "loss": -0.1517,
      "num_tokens": 4049329.0,
      "reward": 0.589529275894165,
      "reward_std": 0.6257125735282898,
      "rewards/rollout_reward_func/mean": 0.589529275894165,
      "rewards/rollout_reward_func/std": 0.6829493045806885,
      "sampling/importance_sampling_ratio/max": 2.8764286041259766,
      "sampling/importance_sampling_ratio/mean": 0.7275078296661377,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.6653485298156738,
      "sampling/sampling_logp_difference/mean": 0.12106862664222717,
      "step": 102,
      "step_time": 12.953762051969534
    },
    {
      "clip_ratio/high_max": 0.02467078692279756,
      "clip_ratio/high_mean": 0.01233539346139878,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01233539346139878,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 615.0,
      "completions/max_terminated_length": 615.0,
      "completions/mean_length": 182.40625,
      "completions/mean_terminated_length": 134.57693481445312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6567448135465384,
      "epoch": 0.00206,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4715605974197388,
      "kl": 0.9032106902450323,
      "learning_rate": 9.889522367416332e-06,
      "loss": 0.0714,
      "num_tokens": 4086810.0,
      "reward": 0.7146874666213989,
      "reward_std": 0.5373616218566895,
      "rewards/rollout_reward_func/mean": 0.7146874666213989,
      "rewards/rollout_reward_func/std": 0.5724986791610718,
      "sampling/importance_sampling_ratio/max": 2.368659019470215,
      "sampling/importance_sampling_ratio/mean": 0.8652793765068054,
      "sampling/importance_sampling_ratio/min": 0.11478550732135773,
      "sampling/sampling_logp_difference/max": 1.5299646854400635,
      "sampling/sampling_logp_difference/mean": 0.11990825831890106,
      "step": 103,
      "step_time": 12.0926686389721
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0052083334885537624,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0071614584885537624,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1179.0,
      "completions/max_terminated_length": 537.0,
      "completions/mean_length": 239.53125,
      "completions/mean_terminated_length": 208.8181915283203,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6131645981222391,
      "epoch": 0.00208,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2063007354736328,
      "kl": 1.0365421082824469,
      "learning_rate": 9.886216819972226e-06,
      "loss": -0.2467,
      "num_tokens": 4127325.0,
      "reward": 0.6953125,
      "reward_std": 0.5979020595550537,
      "rewards/rollout_reward_func/mean": 0.6953125,
      "rewards/rollout_reward_func/std": 0.6410877108573914,
      "sampling/importance_sampling_ratio/max": 2.6564548015594482,
      "sampling/importance_sampling_ratio/mean": 0.8722463846206665,
      "sampling/importance_sampling_ratio/min": 8.643476986314135e-19,
      "sampling/sampling_logp_difference/max": 22.087188720703125,
      "sampling/sampling_logp_difference/mean": 0.17038825154304504,
      "step": 104,
      "step_time": 15.719284081977094
    },
    {
      "clip_ratio/high_max": 0.022111313184723258,
      "clip_ratio/high_mean": 0.011055656592361629,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01886815659236163,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 612.0,
      "completions/max_terminated_length": 612.0,
      "completions/mean_length": 219.8125,
      "completions/mean_terminated_length": 220.89654541015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6003642845898867,
      "epoch": 0.0021,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9536707997322083,
      "kl": 1.4665878238156438,
      "learning_rate": 9.882863308370175e-06,
      "loss": -0.1333,
      "num_tokens": 4167848.0,
      "reward": 0.4069521129131317,
      "reward_std": 0.8597943186759949,
      "rewards/rollout_reward_func/mean": 0.4069521129131317,
      "rewards/rollout_reward_func/std": 0.8571972846984863,
      "sampling/importance_sampling_ratio/max": 2.3856396675109863,
      "sampling/importance_sampling_ratio/mean": 0.781065821647644,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 22.742168426513672,
      "sampling/sampling_logp_difference/mean": 0.23159033060073853,
      "step": 105,
      "step_time": 12.338313121013925
    },
    {
      "clip_ratio/high_max": 0.013932291883975267,
      "clip_ratio/high_mean": 0.006966145941987634,
      "clip_ratio/low_mean": 0.0034722222480922937,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010438368190079927,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 646.0,
      "completions/max_terminated_length": 646.0,
      "completions/mean_length": 213.21875,
      "completions/mean_terminated_length": 186.45834350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5893869213759899,
      "epoch": 0.00212,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.405883312225342,
      "kl": 1.6882341541349888,
      "learning_rate": 9.879461876845255e-06,
      "loss": -0.0189,
      "num_tokens": 4207792.0,
      "reward": 0.6143773794174194,
      "reward_std": 0.5898290872573853,
      "rewards/rollout_reward_func/mean": 0.6143773794174194,
      "rewards/rollout_reward_func/std": 0.6628314852714539,
      "sampling/importance_sampling_ratio/max": 2.9538767337799072,
      "sampling/importance_sampling_ratio/mean": 1.0308585166931152,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.1309187412261963,
      "sampling/sampling_logp_difference/mean": 0.11610356718301773,
      "step": 106,
      "step_time": 12.1989235659712
    },
    {
      "clip_ratio/high_max": 0.020955993561074138,
      "clip_ratio/high_mean": 0.010477996780537069,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010477996780537069,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 829.0,
      "completions/max_terminated_length": 829.0,
      "completions/mean_length": 297.75,
      "completions/mean_terminated_length": 312.8077087402344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7342859022319317,
      "epoch": 0.00214,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.746052622795105,
      "kl": 3.2415115358307958,
      "learning_rate": 9.876012570264658e-06,
      "loss": -0.0316,
      "num_tokens": 4250246.0,
      "reward": 0.5646874904632568,
      "reward_std": 0.6263034343719482,
      "rewards/rollout_reward_func/mean": 0.5646874904632568,
      "rewards/rollout_reward_func/std": 0.6226840019226074,
      "sampling/importance_sampling_ratio/max": 2.321662425994873,
      "sampling/importance_sampling_ratio/mean": 0.7790735363960266,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.8138909339904785,
      "sampling/sampling_logp_difference/mean": 0.11841538548469543,
      "step": 107,
      "step_time": 14.109198086996912
    },
    {
      "clip_ratio/high_max": 0.012607024516910315,
      "clip_ratio/high_mean": 0.0072800746420398355,
      "clip_ratio/low_mean": 0.0020926339784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009372708620503545,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 643.0,
      "completions/max_terminated_length": 643.0,
      "completions/mean_length": 265.5,
      "completions/mean_terminated_length": 223.6666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6088640987873077,
      "epoch": 0.00216,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3715074062347412,
      "kl": 1.9075782867148519,
      "learning_rate": 9.872515434127064e-06,
      "loss": -0.2704,
      "num_tokens": 4291841.0,
      "reward": 0.539728045463562,
      "reward_std": 0.8497440814971924,
      "rewards/rollout_reward_func/mean": 0.539728045463562,
      "rewards/rollout_reward_func/std": 0.8308035731315613,
      "sampling/importance_sampling_ratio/max": 2.502061605453491,
      "sampling/importance_sampling_ratio/mean": 0.8490312099456787,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 17.7230281829834,
      "sampling/sampling_logp_difference/mean": 0.15530318021774292,
      "step": 108,
      "step_time": 12.471900792996166
    },
    {
      "clip_ratio/high_max": 0.01282531931065023,
      "clip_ratio/high_mean": 0.007107104058377445,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007107104058377445,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 242.6875,
      "completions/mean_terminated_length": 230.9091033935547,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7376490626484156,
      "epoch": 0.00218,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3223967552185059,
      "kl": 0.8489520575385541,
      "learning_rate": 9.86897051456206e-06,
      "loss": 0.0172,
      "num_tokens": 4332966.0,
      "reward": 0.7178124189376831,
      "reward_std": 0.6001530289649963,
      "rewards/rollout_reward_func/mean": 0.7178124189376831,
      "rewards/rollout_reward_func/std": 0.5883806943893433,
      "sampling/importance_sampling_ratio/max": 2.6324374675750732,
      "sampling/importance_sampling_ratio/mean": 0.656146764755249,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 22.133209228515625,
      "sampling/sampling_logp_difference/mean": 0.18011602759361267,
      "step": 109,
      "step_time": 12.263386043981882
    },
    {
      "clip_ratio/high_max": 0.01677533728070557,
      "clip_ratio/high_mean": 0.008387668640352786,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009364231140352786,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 267.8125,
      "completions/mean_terminated_length": 256.5517272949219,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5541024021804333,
      "epoch": 0.0022,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6340843439102173,
      "kl": 0.7144495164975524,
      "learning_rate": 9.865377858329539e-06,
      "loss": -0.321,
      "num_tokens": 4375775.0,
      "reward": 0.5224999785423279,
      "reward_std": 0.6987348794937134,
      "rewards/rollout_reward_func/mean": 0.5224999785423279,
      "rewards/rollout_reward_func/std": 0.7214769124984741,
      "sampling/importance_sampling_ratio/max": 2.4963066577911377,
      "sampling/importance_sampling_ratio/mean": 1.0880424976348877,
      "sampling/importance_sampling_ratio/min": 0.12554031610488892,
      "sampling/sampling_logp_difference/max": 1.5245091915130615,
      "sampling/sampling_logp_difference/mean": 0.09587530046701431,
      "step": 110,
      "step_time": 13.321327931014821
    },
    {
      "clip_ratio/high_max": 0.012125039240345359,
      "clip_ratio/high_mean": 0.006062519620172679,
      "clip_ratio/low_mean": 0.003874798770993948,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009937318391166627,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 197.0,
      "completions/mean_terminated_length": 199.1666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7315361313521862,
      "epoch": 0.00222,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.862235188484192,
      "kl": 0.7362443548627198,
      "learning_rate": 9.861737512819058e-06,
      "loss": -0.1074,
      "num_tokens": 4414167.0,
      "reward": 0.7495003938674927,
      "reward_std": 0.5005481243133545,
      "rewards/rollout_reward_func/mean": 0.7495003938674927,
      "rewards/rollout_reward_func/std": 0.5885781645774841,
      "sampling/importance_sampling_ratio/max": 2.5709290504455566,
      "sampling/importance_sampling_ratio/mean": 0.9914208650588989,
      "sampling/importance_sampling_ratio/min": 0.052233729511499405,
      "sampling/sampling_logp_difference/max": 1.500720500946045,
      "sampling/sampling_logp_difference/mean": 0.10679277777671814,
      "step": 111,
      "step_time": 12.05650240102841
    },
    {
      "clip_ratio/high_max": 0.01989293983206153,
      "clip_ratio/high_mean": 0.009946469916030765,
      "clip_ratio/low_mean": 0.0034722222480922937,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013418692164123058,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 557.0,
      "completions/max_terminated_length": 557.0,
      "completions/mean_length": 148.1875,
      "completions/mean_terminated_length": 133.60714721679688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5741560757160187,
      "epoch": 0.00224,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3956197500228882,
      "kl": 2.159604270476848,
      "learning_rate": 9.858049526049241e-06,
      "loss": -0.3192,
      "num_tokens": 4450330.0,
      "reward": 0.5050404667854309,
      "reward_std": 0.7398421764373779,
      "rewards/rollout_reward_func/mean": 0.5050404667854309,
      "rewards/rollout_reward_func/std": 0.8665554523468018,
      "sampling/importance_sampling_ratio/max": 2.722776412963867,
      "sampling/importance_sampling_ratio/mean": 0.9386013746261597,
      "sampling/importance_sampling_ratio/min": 0.15175260603427887,
      "sampling/sampling_logp_difference/max": 1.5585861206054688,
      "sampling/sampling_logp_difference/mean": 0.12841570377349854,
      "step": 112,
      "step_time": 11.51292866501899
    },
    {
      "clip_ratio/high_max": 0.020617319736629725,
      "clip_ratio/high_mean": 0.010308659868314862,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010308659868314862,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 288.46875,
      "completions/mean_terminated_length": 277.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.661247968673706,
      "epoch": 0.00226,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3955721855163574,
      "kl": 1.807788292877376,
      "learning_rate": 9.85431394666712e-06,
      "loss": -0.0953,
      "num_tokens": 4493640.0,
      "reward": 0.24428626894950867,
      "reward_std": 0.8097410202026367,
      "rewards/rollout_reward_func/mean": 0.24428626894950867,
      "rewards/rollout_reward_func/std": 0.8603294491767883,
      "sampling/importance_sampling_ratio/max": 2.0207204818725586,
      "sampling/importance_sampling_ratio/mean": 0.572285532951355,
      "sampling/importance_sampling_ratio/min": 0.09000113606452942,
      "sampling/sampling_logp_difference/max": 2.0281753540039062,
      "sampling/sampling_logp_difference/mean": 0.11512648314237595,
      "step": 113,
      "step_time": 13.181385390038486
    },
    {
      "clip_ratio/high_max": 0.020312500186264515,
      "clip_ratio/high_mean": 0.010807291837409139,
      "clip_ratio/low_mean": 0.016015625093132257,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.026822916930541396,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 890.0,
      "completions/max_terminated_length": 890.0,
      "completions/mean_length": 238.5625,
      "completions/mean_terminated_length": 221.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5439982134848833,
      "epoch": 0.00228,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5643229484558105,
      "kl": 0.5985625116154552,
      "learning_rate": 9.850530823947519e-06,
      "loss": -0.3578,
      "num_tokens": 4533638.0,
      "reward": 0.5145148634910583,
      "reward_std": 0.7315468192100525,
      "rewards/rollout_reward_func/mean": 0.5145148634910583,
      "rewards/rollout_reward_func/std": 0.7693302631378174,
      "sampling/importance_sampling_ratio/max": 2.9787991046905518,
      "sampling/importance_sampling_ratio/mean": 1.1645063161849976,
      "sampling/importance_sampling_ratio/min": 0.24126574397087097,
      "sampling/sampling_logp_difference/max": 1.2448192834854126,
      "sampling/sampling_logp_difference/mean": 0.10473307967185974,
      "step": 114,
      "step_time": 12.545894338953076
    },
    {
      "clip_ratio/high_max": 0.010416666744276881,
      "clip_ratio/high_mean": 0.005208333372138441,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005208333372138441,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 910.0,
      "completions/max_terminated_length": 910.0,
      "completions/mean_length": 198.25,
      "completions/mean_terminated_length": 197.4666748046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6643406376242638,
      "epoch": 0.0023,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.469751000404358,
      "kl": 1.9632383896969259,
      "learning_rate": 9.846700207792381e-06,
      "loss": -0.3221,
      "num_tokens": 4572556.0,
      "reward": 0.3181876540184021,
      "reward_std": 0.9158343076705933,
      "rewards/rollout_reward_func/mean": 0.3181876540184021,
      "rewards/rollout_reward_func/std": 0.897193431854248,
      "sampling/importance_sampling_ratio/max": 2.489990234375,
      "sampling/importance_sampling_ratio/mean": 0.908228874206543,
      "sampling/importance_sampling_ratio/min": 0.07885687053203583,
      "sampling/sampling_logp_difference/max": 1.690739631652832,
      "sampling/sampling_logp_difference/mean": 0.11611685901880264,
      "step": 115,
      "step_time": 12.79871818202082
    },
    {
      "clip_ratio/high_max": 0.020746237598359585,
      "clip_ratio/high_mean": 0.010373118799179792,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010373118799179792,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 182.71875,
      "completions/mean_terminated_length": 186.8518524169922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6035721097141504,
      "epoch": 0.00232,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.570186734199524,
      "kl": 0.7998226340860128,
      "learning_rate": 9.842822148730122e-06,
      "loss": 0.0562,
      "num_tokens": 4610869.0,
      "reward": 0.6690624952316284,
      "reward_std": 0.6062224507331848,
      "rewards/rollout_reward_func/mean": 0.6690624952316284,
      "rewards/rollout_reward_func/std": 0.6256731152534485,
      "sampling/importance_sampling_ratio/max": 2.381884813308716,
      "sampling/importance_sampling_ratio/mean": 0.8981492519378662,
      "sampling/importance_sampling_ratio/min": 7.301455438195346e-18,
      "sampling/sampling_logp_difference/max": 24.574201583862305,
      "sampling/sampling_logp_difference/mean": 0.30190619826316833,
      "step": 116,
      "step_time": 12.707192223024322
    },
    {
      "clip_ratio/high_max": 0.008572583319619298,
      "clip_ratio/high_mean": 0.004286291659809649,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004286291659809649,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 784.0,
      "completions/max_terminated_length": 784.0,
      "completions/mean_length": 218.34375,
      "completions/mean_terminated_length": 192.45834350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6102747768163681,
      "epoch": 0.00234,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5297832489013672,
      "kl": 0.9913459960371256,
      "learning_rate": 9.838896697914964e-06,
      "loss": -0.0082,
      "num_tokens": 4649387.0,
      "reward": 0.6443749666213989,
      "reward_std": 0.6503565311431885,
      "rewards/rollout_reward_func/mean": 0.6443749666213989,
      "rewards/rollout_reward_func/std": 0.6732824444770813,
      "sampling/importance_sampling_ratio/max": 2.9204659461975098,
      "sampling/importance_sampling_ratio/mean": 0.8741087913513184,
      "sampling/importance_sampling_ratio/min": 3.4577563243458713e-22,
      "sampling/sampling_logp_difference/max": 20.470239639282227,
      "sampling/sampling_logp_difference/mean": 0.205940842628479,
      "step": 117,
      "step_time": 12.338153807009803
    },
    {
      "clip_ratio/high_max": 0.026692708488553762,
      "clip_ratio/high_mean": 0.013346354244276881,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013346354244276881,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 611.0,
      "completions/max_terminated_length": 609.0,
      "completions/mean_length": 233.03125,
      "completions/mean_terminated_length": 209.3333282470703,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5747595466673374,
      "epoch": 0.00236,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0736844539642334,
      "kl": 0.860448757186532,
      "learning_rate": 9.834923907126256e-06,
      "loss": -0.2634,
      "num_tokens": 4690379.0,
      "reward": 0.3800404965877533,
      "reward_std": 0.8480271100997925,
      "rewards/rollout_reward_func/mean": 0.3800404965877533,
      "rewards/rollout_reward_func/std": 0.8568060398101807,
      "sampling/importance_sampling_ratio/max": 2.347238779067993,
      "sampling/importance_sampling_ratio/mean": 0.9888460636138916,
      "sampling/importance_sampling_ratio/min": 0.1337958723306656,
      "sampling/sampling_logp_difference/max": 1.5637531280517578,
      "sampling/sampling_logp_difference/mean": 0.11304305493831635,
      "step": 118,
      "step_time": 12.076213972002733
    },
    {
      "clip_ratio/high_max": 0.02760416711680591,
      "clip_ratio/high_mean": 0.013802083558402956,
      "clip_ratio/low_mean": 0.005078125046566129,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018880208604969084,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1168.0,
      "completions/max_terminated_length": 1168.0,
      "completions/mean_length": 262.34375,
      "completions/mean_terminated_length": 270.66668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6773894298821688,
      "epoch": 0.00238,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1693671941757202,
      "kl": 3.8758230339735746,
      "learning_rate": 9.830903828767796e-06,
      "loss": -0.1555,
      "num_tokens": 4730205.0,
      "reward": 0.4128149151802063,
      "reward_std": 0.8677624464035034,
      "rewards/rollout_reward_func/mean": 0.4128149151802063,
      "rewards/rollout_reward_func/std": 0.8834648728370667,
      "sampling/importance_sampling_ratio/max": 2.0952157974243164,
      "sampling/importance_sampling_ratio/mean": 0.8471798896789551,
      "sampling/importance_sampling_ratio/min": 0.1091524139046669,
      "sampling/sampling_logp_difference/max": 2.2162728309631348,
      "sampling/sampling_logp_difference/mean": 0.10763879120349884,
      "step": 119,
      "step_time": 15.067942087014671
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0020833334419876337,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004036458441987634,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 234.5625,
      "completions/mean_terminated_length": 217.56668090820312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6005770359188318,
      "epoch": 0.0024,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3420857191085815,
      "kl": 1.6011292450129986,
      "learning_rate": 9.826836515867131e-06,
      "loss": -0.2283,
      "num_tokens": 4771200.0,
      "reward": 0.38084876537323,
      "reward_std": 0.9035135507583618,
      "rewards/rollout_reward_func/mean": 0.38084876537323,
      "rewards/rollout_reward_func/std": 0.8711024522781372,
      "sampling/importance_sampling_ratio/max": 2.538489580154419,
      "sampling/importance_sampling_ratio/mean": 0.780072808265686,
      "sampling/importance_sampling_ratio/min": 2.9541549793066224e-06,
      "sampling/sampling_logp_difference/max": 12.559476852416992,
      "sampling/sampling_logp_difference/mean": 0.18990649282932281,
      "step": 120,
      "step_time": 12.129922917039949
    },
    {
      "clip_ratio/high_max": 0.017505338182672858,
      "clip_ratio/high_mean": 0.010054752579890192,
      "clip_ratio/low_mean": 0.008653846336528659,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01870859880000353,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 239.03125,
      "completions/mean_terminated_length": 239.6206817626953,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6056597772985697,
      "epoch": 0.00242,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0071914196014404,
      "kl": 1.003183025866747,
      "learning_rate": 9.822722022074871e-06,
      "loss": -0.234,
      "num_tokens": 4811835.0,
      "reward": 0.49022376537323,
      "reward_std": 0.7984931468963623,
      "rewards/rollout_reward_func/mean": 0.49022376537323,
      "rewards/rollout_reward_func/std": 0.7992748618125916,
      "sampling/importance_sampling_ratio/max": 2.2431094646453857,
      "sampling/importance_sampling_ratio/mean": 0.8240576982498169,
      "sampling/importance_sampling_ratio/min": 0.10133003443479538,
      "sampling/sampling_logp_difference/max": 1.4228968620300293,
      "sampling/sampling_logp_difference/mean": 0.11849194765090942,
      "step": 121,
      "step_time": 12.43095824001648
    },
    {
      "clip_ratio/high_max": 0.0176743664778769,
      "clip_ratio/high_mean": 0.00883718323893845,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00883718323893845,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 648.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 223.28125,
      "completions/mean_terminated_length": 245.84616088867188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6875145640224218,
      "epoch": 0.00244,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3970439434051514,
      "kl": 1.8368692575022578,
      "learning_rate": 9.818560401663972e-06,
      "loss": -0.2427,
      "num_tokens": 4852420.0,
      "reward": 0.2421874850988388,
      "reward_std": 0.8073561191558838,
      "rewards/rollout_reward_func/mean": 0.2421874850988388,
      "rewards/rollout_reward_func/std": 0.8458694219589233,
      "sampling/importance_sampling_ratio/max": 2.235051393508911,
      "sampling/importance_sampling_ratio/mean": 0.8621902465820312,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 20.750011444091797,
      "sampling/sampling_logp_difference/mean": 0.1937955915927887,
      "step": 122,
      "step_time": 13.259493217963609
    },
    {
      "clip_ratio/high_max": 0.032031250186264515,
      "clip_ratio/high_mean": 0.016015625093132257,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016015625093132257,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 1027.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 212.0625,
      "completions/mean_terminated_length": 197.48275756835938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6344220135360956,
      "epoch": 0.00246,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2317733764648438,
      "kl": 1.4187929313629866,
      "learning_rate": 9.814351709529018e-06,
      "loss": -0.1178,
      "num_tokens": 4891555.0,
      "reward": 0.4538271725177765,
      "reward_std": 0.7479486465454102,
      "rewards/rollout_reward_func/mean": 0.4538271725177765,
      "rewards/rollout_reward_func/std": 0.8347770571708679,
      "sampling/importance_sampling_ratio/max": 2.0446979999542236,
      "sampling/importance_sampling_ratio/mean": 0.8217203617095947,
      "sampling/importance_sampling_ratio/min": 0.21478402614593506,
      "sampling/sampling_logp_difference/max": 1.450711727142334,
      "sampling/sampling_logp_difference/mean": 0.11312085390090942,
      "step": 123,
      "step_time": 12.931650742000784
    },
    {
      "clip_ratio/high_max": 0.04817708337213844,
      "clip_ratio/high_mean": 0.02408854168606922,
      "clip_ratio/low_mean": 0.004924242617562413,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029012784361839294,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 792.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 272.6875,
      "completions/mean_terminated_length": 225.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6526379194110632,
      "epoch": 0.00248,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2313592433929443,
      "kl": 1.2024640869349241,
      "learning_rate": 9.8100960011855e-06,
      "loss": -0.0641,
      "num_tokens": 4932780.0,
      "reward": 0.460077166557312,
      "reward_std": 0.657999575138092,
      "rewards/rollout_reward_func/mean": 0.460077166557312,
      "rewards/rollout_reward_func/std": 0.6850254535675049,
      "sampling/importance_sampling_ratio/max": 2.322357177734375,
      "sampling/importance_sampling_ratio/mean": 0.8623349666595459,
      "sampling/importance_sampling_ratio/min": 3.1347480167198682e-09,
      "sampling/sampling_logp_difference/max": 19.1875,
      "sampling/sampling_logp_difference/mean": 0.16718262434005737,
      "step": 124,
      "step_time": 13.194154031021753
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00390625,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 184.78125,
      "completions/mean_terminated_length": 197.0416717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8216367550194263,
      "epoch": 0.0025,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8231768608093262,
      "kl": 0.7560070715844631,
      "learning_rate": 9.805793332769095e-06,
      "loss": -0.2898,
      "num_tokens": 4971593.0,
      "reward": 0.4815624952316284,
      "reward_std": 0.7379949688911438,
      "rewards/rollout_reward_func/mean": 0.4815624952316284,
      "rewards/rollout_reward_func/std": 0.7966020703315735,
      "sampling/importance_sampling_ratio/max": 2.750918388366699,
      "sampling/importance_sampling_ratio/mean": 0.9809617400169373,
      "sampling/importance_sampling_ratio/min": 0.1406843662261963,
      "sampling/sampling_logp_difference/max": 1.4772448539733887,
      "sampling/sampling_logp_difference/mean": 0.1050536260008812,
      "step": 125,
      "step_time": 11.908214923998457
    },
    {
      "clip_ratio/high_max": 0.03020833432674408,
      "clip_ratio/high_mean": 0.01510416716337204,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01510416716337204,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 215.09375,
      "completions/mean_terminated_length": 198.11538696289062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6429573837667704,
      "epoch": 0.00252,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1516865491867065,
      "kl": 0.9412194807082415,
      "learning_rate": 9.801443761034903e-06,
      "loss": -0.1713,
      "num_tokens": 5011155.0,
      "reward": 0.6181249618530273,
      "reward_std": 0.6449759006500244,
      "rewards/rollout_reward_func/mean": 0.6181249618530273,
      "rewards/rollout_reward_func/std": 0.6594349145889282,
      "sampling/importance_sampling_ratio/max": 2.1796646118164062,
      "sampling/importance_sampling_ratio/mean": 0.9723384976387024,
      "sampling/importance_sampling_ratio/min": 0.11083625257015228,
      "sampling/sampling_logp_difference/max": 1.3877840042114258,
      "sampling/sampling_logp_difference/mean": 0.10613994300365448,
      "step": 126,
      "step_time": 12.104509336029878
    },
    {
      "clip_ratio/high_max": 0.0116257444024086,
      "clip_ratio/high_mean": 0.0058128722012043,
      "clip_ratio/low_mean": 0.005621693329885602,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011434565531089902,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 184.46875,
      "completions/mean_terminated_length": 172.96551513671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.648771433159709,
      "epoch": 0.00254,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8943966031074524,
      "kl": 1.3389656338840723,
      "learning_rate": 9.797047343356718e-06,
      "loss": -0.1714,
      "num_tokens": 5049862.0,
      "reward": 0.2708892822265625,
      "reward_std": 0.914330244064331,
      "rewards/rollout_reward_func/mean": 0.2708892822265625,
      "rewards/rollout_reward_func/std": 0.890661358833313,
      "sampling/importance_sampling_ratio/max": 2.0942628383636475,
      "sampling/importance_sampling_ratio/mean": 0.8756117224693298,
      "sampling/importance_sampling_ratio/min": 0.1762433797121048,
      "sampling/sampling_logp_difference/max": 1.6368169784545898,
      "sampling/sampling_logp_difference/mean": 0.11091165244579315,
      "step": 127,
      "step_time": 12.674710178020177
    },
    {
      "clip_ratio/high_max": 0.029542707605287433,
      "clip_ratio/high_mean": 0.014771353802643716,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014771353802643716,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 979.0,
      "completions/max_terminated_length": 979.0,
      "completions/mean_length": 255.0,
      "completions/mean_terminated_length": 205.7777862548828,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6836048122495413,
      "epoch": 0.00256,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4561699628829956,
      "kl": 1.0490720616653562,
      "learning_rate": 9.792604137726259e-06,
      "loss": -0.2862,
      "num_tokens": 5091936.0,
      "reward": 0.49281251430511475,
      "reward_std": 0.7243983149528503,
      "rewards/rollout_reward_func/mean": 0.49281251430511475,
      "rewards/rollout_reward_func/std": 0.7757877707481384,
      "sampling/importance_sampling_ratio/max": 2.3856074810028076,
      "sampling/importance_sampling_ratio/mean": 0.9340779781341553,
      "sampling/importance_sampling_ratio/min": 0.1580251157283783,
      "sampling/sampling_logp_difference/max": 1.5031366348266602,
      "sampling/sampling_logp_difference/mean": 0.09966586530208588,
      "step": 128,
      "step_time": 13.253396825981326
    },
    {
      "clip_ratio/high_max": 0.05104166688397527,
      "clip_ratio/high_mean": 0.025520833441987634,
      "clip_ratio/low_mean": 0.0012499999720603228,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.026770833414047956,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 540.0,
      "completions/max_terminated_length": 517.0,
      "completions/mean_length": 177.03125,
      "completions/mean_terminated_length": 165.32257080078125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6167433243244886,
      "epoch": 0.00258,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1610621213912964,
      "kl": 2.234186192974448,
      "learning_rate": 9.788114202752415e-06,
      "loss": -0.1995,
      "num_tokens": 5129879.0,
      "reward": 0.5171011686325073,
      "reward_std": 0.7728384733200073,
      "rewards/rollout_reward_func/mean": 0.5171011686325073,
      "rewards/rollout_reward_func/std": 0.832011878490448,
      "sampling/importance_sampling_ratio/max": 2.886399745941162,
      "sampling/importance_sampling_ratio/mean": 1.01292884349823,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 26.81975555419922,
      "sampling/sampling_logp_difference/mean": 0.36164331436157227,
      "step": 129,
      "step_time": 11.332059527019737
    },
    {
      "clip_ratio/high_max": 0.006138392956927419,
      "clip_ratio/high_mean": 0.0030691964784637094,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01088169647846371,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 642.0,
      "completions/max_terminated_length": 642.0,
      "completions/mean_length": 195.5625,
      "completions/mean_terminated_length": 175.75001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6128010582178831,
      "epoch": 0.0026,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1921699047088623,
      "kl": 1.4134037801995873,
      "learning_rate": 9.78357759766046e-06,
      "loss": -0.0349,
      "num_tokens": 5167987.0,
      "reward": 0.4276176393032074,
      "reward_std": 0.8117365837097168,
      "rewards/rollout_reward_func/mean": 0.4276176393032074,
      "rewards/rollout_reward_func/std": 0.8175442218780518,
      "sampling/importance_sampling_ratio/max": 2.3745265007019043,
      "sampling/importance_sampling_ratio/mean": 0.9005454778671265,
      "sampling/importance_sampling_ratio/min": 0.17290550470352173,
      "sampling/sampling_logp_difference/max": 1.5555922985076904,
      "sampling/sampling_logp_difference/mean": 0.09913434833288193,
      "step": 130,
      "step_time": 13.0957756130083
    },
    {
      "clip_ratio/high_max": 0.01764881261624396,
      "clip_ratio/high_mean": 0.00882440630812198,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00980096880812198,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 621.0,
      "completions/mean_length": 304.1875,
      "completions/mean_terminated_length": 241.9499969482422,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6418006736785173,
      "epoch": 0.00262,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9741240739822388,
      "kl": 0.7293591406196356,
      "learning_rate": 9.778994382291283e-06,
      "loss": -0.1625,
      "num_tokens": 5213446.0,
      "reward": 0.5787500143051147,
      "reward_std": 0.6704323291778564,
      "rewards/rollout_reward_func/mean": 0.5787500143051147,
      "rewards/rollout_reward_func/std": 0.6760976910591125,
      "sampling/importance_sampling_ratio/max": 2.117593765258789,
      "sampling/importance_sampling_ratio/mean": 0.8564802408218384,
      "sampling/importance_sampling_ratio/min": 7.367339164578202e-27,
      "sampling/sampling_logp_difference/max": 26.804672241210938,
      "sampling/sampling_logp_difference/mean": 0.17015990614891052,
      "step": 131,
      "step_time": 13.137926662981045
    },
    {
      "clip_ratio/high_max": 0.009030522778630257,
      "clip_ratio/high_mean": 0.00587395706679672,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008478123811073601,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 655.0,
      "completions/max_terminated_length": 655.0,
      "completions/mean_length": 209.4375,
      "completions/mean_terminated_length": 188.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5177624570205808,
      "epoch": 0.00264,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8715291619300842,
      "kl": 1.0429232064634562,
      "learning_rate": 9.774364617100596e-06,
      "loss": -0.1365,
      "num_tokens": 5252642.0,
      "reward": 0.5431274175643921,
      "reward_std": 0.7374885082244873,
      "rewards/rollout_reward_func/mean": 0.5431274175643921,
      "rewards/rollout_reward_func/std": 0.7446845173835754,
      "sampling/importance_sampling_ratio/max": 1.792188048362732,
      "sampling/importance_sampling_ratio/mean": 0.8710232973098755,
      "sampling/importance_sampling_ratio/min": 0.08479435741901398,
      "sampling/sampling_logp_difference/max": 1.6029052734375,
      "sampling/sampling_logp_difference/mean": 0.0992274284362793,
      "step": 132,
      "step_time": 11.927098404950812
    },
    {
      "clip_ratio/high_max": 0.02640568115748465,
      "clip_ratio/high_mean": 0.014404763584025204,
      "clip_ratio/low_mean": 0.005440848413854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019845611765049398,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 656.0,
      "completions/mean_length": 221.375,
      "completions/mean_terminated_length": 214.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5215325988829136,
      "epoch": 0.00266,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2297946214675903,
      "kl": 1.0241317888721824,
      "learning_rate": 9.769688363158127e-06,
      "loss": -0.0491,
      "num_tokens": 5292477.0,
      "reward": 0.5670987367630005,
      "reward_std": 0.6610325574874878,
      "rewards/rollout_reward_func/mean": 0.5670987367630005,
      "rewards/rollout_reward_func/std": 0.6842064261436462,
      "sampling/importance_sampling_ratio/max": 2.453320026397705,
      "sampling/importance_sampling_ratio/mean": 0.9189397096633911,
      "sampling/importance_sampling_ratio/min": 0.10618587583303452,
      "sampling/sampling_logp_difference/max": 1.639390468597412,
      "sampling/sampling_logp_difference/mean": 0.10205963253974915,
      "step": 133,
      "step_time": 13.22707321299822
    },
    {
      "clip_ratio/high_max": 0.011781753972172737,
      "clip_ratio/high_mean": 0.0058908769860863686,
      "clip_ratio/low_mean": 0.0032402074430137873,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009131084196269512,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1124.0,
      "completions/max_terminated_length": 691.0,
      "completions/mean_length": 198.34375,
      "completions/mean_terminated_length": 148.9615478515625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.47952658869326115,
      "epoch": 0.00268,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2264533042907715,
      "kl": 0.5732358945533633,
      "learning_rate": 9.76496568214683e-06,
      "loss": 0.0033,
      "num_tokens": 5331272.0,
      "reward": 0.5868749618530273,
      "reward_std": 0.6274101734161377,
      "rewards/rollout_reward_func/mean": 0.5868749618530273,
      "rewards/rollout_reward_func/std": 0.6741419434547424,
      "sampling/importance_sampling_ratio/max": 1.9974626302719116,
      "sampling/importance_sampling_ratio/mean": 0.7634527087211609,
      "sampling/importance_sampling_ratio/min": 2.9275372951569414e-15,
      "sampling/sampling_logp_difference/max": 21.42192840576172,
      "sampling/sampling_logp_difference/mean": 0.28915226459503174,
      "step": 134,
      "step_time": 13.709758492972469
    },
    {
      "clip_ratio/high_max": 0.01578947389498353,
      "clip_ratio/high_mean": 0.007894736947491765,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007894736947491765,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 227.0,
      "completions/mean_terminated_length": 223.21739196777344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5428877267986536,
      "epoch": 0.0027,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1166995763778687,
      "kl": 1.4720135079696774,
      "learning_rate": 9.760196636362058e-06,
      "loss": -0.1168,
      "num_tokens": 5372135.0,
      "reward": 0.5703529715538025,
      "reward_std": 0.6986875534057617,
      "rewards/rollout_reward_func/mean": 0.5703529715538025,
      "rewards/rollout_reward_func/std": 0.7425456047058105,
      "sampling/importance_sampling_ratio/max": 2.541121482849121,
      "sampling/importance_sampling_ratio/mean": 1.0585870742797852,
      "sampling/importance_sampling_ratio/min": 0.13707053661346436,
      "sampling/sampling_logp_difference/max": 1.974228858947754,
      "sampling/sampling_logp_difference/mean": 0.08682049810886383,
      "step": 135,
      "step_time": 12.319846764963586
    },
    {
      "clip_ratio/high_max": 0.015340909361839294,
      "clip_ratio/high_mean": 0.007670454680919647,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012134740594774485,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 647.0,
      "completions/mean_length": 212.78125,
      "completions/mean_terminated_length": 203.27586364746094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4992398517206311,
      "epoch": 0.00272,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7938188314437866,
      "kl": 4.2364822290837765,
      "learning_rate": 9.755381288710751e-06,
      "loss": -0.0867,
      "num_tokens": 5411033.0,
      "reward": 0.6309374570846558,
      "reward_std": 0.6301780939102173,
      "rewards/rollout_reward_func/mean": 0.6309374570846558,
      "rewards/rollout_reward_func/std": 0.6714709401130676,
      "sampling/importance_sampling_ratio/max": 2.2715749740600586,
      "sampling/importance_sampling_ratio/mean": 0.8617445230484009,
      "sampling/importance_sampling_ratio/min": 0.07226257026195526,
      "sampling/sampling_logp_difference/max": 1.7538838386535645,
      "sampling/sampling_logp_difference/mean": 0.10158240795135498,
      "step": 136,
      "step_time": 12.834243483026512
    },
    {
      "clip_ratio/high_max": 0.034599570324644446,
      "clip_ratio/high_mean": 0.018787880311720073,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020741005311720073,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 710.0,
      "completions/max_terminated_length": 656.0,
      "completions/mean_length": 230.875,
      "completions/mean_terminated_length": 194.51852416992188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5266514848917723,
      "epoch": 0.00274,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0352489948272705,
      "kl": 0.5302538257092237,
      "learning_rate": 9.750519702710598e-06,
      "loss": 0.1128,
      "num_tokens": 5450933.0,
      "reward": 0.8149999976158142,
      "reward_std": 0.39556366205215454,
      "rewards/rollout_reward_func/mean": 0.8149999976158142,
      "rewards/rollout_reward_func/std": 0.3982866406440735,
      "sampling/importance_sampling_ratio/max": 1.9585896730422974,
      "sampling/importance_sampling_ratio/mean": 0.9910277724266052,
      "sampling/importance_sampling_ratio/min": 0.25698766112327576,
      "sampling/sampling_logp_difference/max": 1.3432090282440186,
      "sampling/sampling_logp_difference/mean": 0.09187475591897964,
      "step": 137,
      "step_time": 12.33281235495815
    },
    {
      "clip_ratio/high_max": 0.019531250232830644,
      "clip_ratio/high_mean": 0.009765625116415322,
      "clip_ratio/low_mean": 0.003412844380363822,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013178469496779144,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 635.0,
      "completions/mean_length": 238.59375,
      "completions/mean_terminated_length": 215.6666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.580316374078393,
      "epoch": 0.00276,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2399874925613403,
      "kl": 0.9797670152038336,
      "learning_rate": 9.745611942489202e-06,
      "loss": -0.1071,
      "num_tokens": 5491270.0,
      "reward": 0.6906249523162842,
      "reward_std": 0.526070237159729,
      "rewards/rollout_reward_func/mean": 0.6906249523162842,
      "rewards/rollout_reward_func/std": 0.6133931875228882,
      "sampling/importance_sampling_ratio/max": 2.447577714920044,
      "sampling/importance_sampling_ratio/mean": 0.9695064425468445,
      "sampling/importance_sampling_ratio/min": 3.230656762914536e-19,
      "sampling/sampling_logp_difference/max": 22.801429748535156,
      "sampling/sampling_logp_difference/mean": 0.23036324977874756,
      "step": 138,
      "step_time": 12.705131782015087
    },
    {
      "clip_ratio/high_max": 0.01047979830764234,
      "clip_ratio/high_mean": 0.00523989915382117,
      "clip_ratio/low_mean": 0.0048828125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01012271165382117,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 204.09375,
      "completions/mean_terminated_length": 204.86668395996094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4801871906965971,
      "epoch": 0.00278,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8954254984855652,
      "kl": 0.491910757496953,
      "learning_rate": 9.740658072783244e-06,
      "loss": -0.1277,
      "num_tokens": 5530736.0,
      "reward": 0.5712499618530273,
      "reward_std": 0.656201958656311,
      "rewards/rollout_reward_func/mean": 0.5712499618530273,
      "rewards/rollout_reward_func/std": 0.6720250844955444,
      "sampling/importance_sampling_ratio/max": 2.587134838104248,
      "sampling/importance_sampling_ratio/mean": 1.20135498046875,
      "sampling/importance_sampling_ratio/min": 3.364864749349733e-20,
      "sampling/sampling_logp_difference/max": 22.42864418029785,
      "sampling/sampling_logp_difference/mean": 0.20854787528514862,
      "step": 139,
      "step_time": 12.112629684022977
    },
    {
      "clip_ratio/high_max": 0.0020833334419876337,
      "clip_ratio/high_mean": 0.0010416667209938169,
      "clip_ratio/low_mean": 0.004288383177481592,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0053300498984754086,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 1332.0,
      "completions/max_terminated_length": 683.0,
      "completions/mean_length": 241.84375,
      "completions/mean_terminated_length": 224.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5333880074322224,
      "epoch": 0.0028,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1854441165924072,
      "kl": 1.115496621467173,
      "learning_rate": 9.73565815893761e-06,
      "loss": -0.0465,
      "num_tokens": 5571493.0,
      "reward": 0.48625001311302185,
      "reward_std": 0.7618280649185181,
      "rewards/rollout_reward_func/mean": 0.48625001311302185,
      "rewards/rollout_reward_func/std": 0.7344198822975159,
      "sampling/importance_sampling_ratio/max": 1.9577609300613403,
      "sampling/importance_sampling_ratio/mean": 0.8557852506637573,
      "sampling/importance_sampling_ratio/min": 1.2312042010909074e-11,
      "sampling/sampling_logp_difference/max": 20.73401641845703,
      "sampling/sampling_logp_difference/mean": 0.16074255108833313,
      "step": 140,
      "step_time": 15.27622019201226
    },
    {
      "clip_ratio/high_max": 0.04121767240576446,
      "clip_ratio/high_mean": 0.02060883620288223,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02256196120288223,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 656.0,
      "completions/mean_length": 187.375,
      "completions/mean_terminated_length": 163.4615478515625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5156266205012798,
      "epoch": 0.00282,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4966633319854736,
      "kl": 1.0850589098408818,
      "learning_rate": 9.730612266904548e-06,
      "loss": 0.0689,
      "num_tokens": 5610886.0,
      "reward": 0.5503125190734863,
      "reward_std": 0.5755362510681152,
      "rewards/rollout_reward_func/mean": 0.5503125190734863,
      "rewards/rollout_reward_func/std": 0.5597550272941589,
      "sampling/importance_sampling_ratio/max": 2.6279077529907227,
      "sampling/importance_sampling_ratio/mean": 0.9387043714523315,
      "sampling/importance_sampling_ratio/min": 3.2315179851138964e-05,
      "sampling/sampling_logp_difference/max": 9.130290985107422,
      "sampling/sampling_logp_difference/mean": 0.11546764522790909,
      "step": 141,
      "step_time": 12.611397152009886
    },
    {
      "clip_ratio/high_max": 0.011931818444281816,
      "clip_ratio/high_mean": 0.005965909222140908,
      "clip_ratio/low_mean": 0.0014204545877873898,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007386363809928298,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 659.0,
      "completions/mean_length": 237.28125,
      "completions/mean_terminated_length": 245.34616088867188,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5693657789379358,
      "epoch": 0.00284,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.513933539390564,
      "kl": 0.5774479769170284,
      "learning_rate": 9.72552046324278e-06,
      "loss": 0.1651,
      "num_tokens": 5650581.0,
      "reward": 0.5374999642372131,
      "reward_std": 0.5726516842842102,
      "rewards/rollout_reward_func/mean": 0.5374999642372131,
      "rewards/rollout_reward_func/std": 0.5635544061660767,
      "sampling/importance_sampling_ratio/max": 2.208584785461426,
      "sampling/importance_sampling_ratio/mean": 0.9609779119491577,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7062911987304688,
      "sampling/sampling_logp_difference/mean": 0.09156946837902069,
      "step": 142,
      "step_time": 12.56438390898984
    },
    {
      "clip_ratio/high_max": 0.016834077890962362,
      "clip_ratio/high_mean": 0.009393601561896503,
      "clip_ratio/low_mean": 0.002864583395421505,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012258184957318008,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 837.0,
      "completions/max_terminated_length": 626.0,
      "completions/mean_length": 257.3125,
      "completions/mean_terminated_length": 231.0399932861328,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6585570108145475,
      "epoch": 0.00286,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9988731741905212,
      "kl": 0.34728113655000925,
      "learning_rate": 9.72038281511664e-06,
      "loss": 0.059,
      "num_tokens": 5692062.0,
      "reward": 0.6050000190734863,
      "reward_std": 0.5230768918991089,
      "rewards/rollout_reward_func/mean": 0.6050000190734863,
      "rewards/rollout_reward_func/std": 0.5645894408226013,
      "sampling/importance_sampling_ratio/max": 1.6738818883895874,
      "sampling/importance_sampling_ratio/mean": 0.8190377354621887,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.071516513824463,
      "sampling/sampling_logp_difference/mean": 0.08486485481262207,
      "step": 143,
      "step_time": 12.944209793990012
    },
    {
      "clip_ratio/high_max": 0.013805318623781204,
      "clip_ratio/high_mean": 0.006902659311890602,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006902659311890602,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 685.0,
      "completions/max_terminated_length": 647.0,
      "completions/mean_length": 255.5625,
      "completions/mean_terminated_length": 269.6521911621094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5951343905180693,
      "epoch": 0.00288,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2271524667739868,
      "kl": 2.399850253947079,
      "learning_rate": 9.715199390295181e-06,
      "loss": -0.218,
      "num_tokens": 5733142.0,
      "reward": 0.38066792488098145,
      "reward_std": 0.7707634568214417,
      "rewards/rollout_reward_func/mean": 0.38066792488098145,
      "rewards/rollout_reward_func/std": 0.7335361242294312,
      "sampling/importance_sampling_ratio/max": 2.0271692276000977,
      "sampling/importance_sampling_ratio/mean": 0.8048043847084045,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7327580451965332,
      "sampling/sampling_logp_difference/mean": 0.10093681514263153,
      "step": 144,
      "step_time": 12.87183133000508
    },
    {
      "clip_ratio/high_max": 0.022296262672170997,
      "clip_ratio/high_mean": 0.011148131336085498,
      "clip_ratio/low_mean": 0.00435014208778739,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015498273307457566,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 643.0,
      "completions/max_terminated_length": 643.0,
      "completions/mean_length": 212.21875,
      "completions/mean_terminated_length": 229.27999877929688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6448405459523201,
      "epoch": 0.0029,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6568541526794434,
      "kl": 0.5055196369066834,
      "learning_rate": 9.70997025715128e-06,
      "loss": -0.0675,
      "num_tokens": 5773718.0,
      "reward": 0.45834875106811523,
      "reward_std": 0.7073843479156494,
      "rewards/rollout_reward_func/mean": 0.45834875106811523,
      "rewards/rollout_reward_func/std": 0.7265693545341492,
      "sampling/importance_sampling_ratio/max": 2.0912163257598877,
      "sampling/importance_sampling_ratio/mean": 0.8310059309005737,
      "sampling/importance_sampling_ratio/min": 1.0700430903298627e-16,
      "sampling/sampling_logp_difference/max": 19.637615203857422,
      "sampling/sampling_logp_difference/mean": 0.21294377744197845,
      "step": 145,
      "step_time": 12.089974620001158
    },
    {
      "clip_ratio/high_max": 0.03806830337271094,
      "clip_ratio/high_mean": 0.020392847363837063,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021369409863837063,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 647.0,
      "completions/mean_length": 226.59375,
      "completions/mean_terminated_length": 226.42308044433594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7506681475788355,
      "epoch": 0.00292,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6393438577651978,
      "kl": 0.4183855322189629,
      "learning_rate": 9.704695484660736e-06,
      "loss": 0.0793,
      "num_tokens": 5814600.0,
      "reward": 0.7672646045684814,
      "reward_std": 0.423237681388855,
      "rewards/rollout_reward_func/mean": 0.7672646045684814,
      "rewards/rollout_reward_func/std": 0.5058237314224243,
      "sampling/importance_sampling_ratio/max": 2.665134906768799,
      "sampling/importance_sampling_ratio/mean": 0.8882358074188232,
      "sampling/importance_sampling_ratio/min": 8.866882402750661e-23,
      "sampling/sampling_logp_difference/max": 27.64430809020996,
      "sampling/sampling_logp_difference/mean": 0.4175644516944885,
      "step": 146,
      "step_time": 12.317353604987147
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.006983901723288,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006983901723288,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 892.0,
      "completions/max_terminated_length": 892.0,
      "completions/mean_length": 213.0,
      "completions/mean_terminated_length": 226.1333465576172,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5585642103105783,
      "epoch": 0.00294,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.831347942352295,
      "kl": 0.8243753733113408,
      "learning_rate": 9.699375142401365e-06,
      "loss": -0.2648,
      "num_tokens": 5852829.0,
      "reward": 0.4770553708076477,
      "reward_std": 0.6833381652832031,
      "rewards/rollout_reward_func/mean": 0.4770553708076477,
      "rewards/rollout_reward_func/std": 0.7316402196884155,
      "sampling/importance_sampling_ratio/max": 2.239927053451538,
      "sampling/importance_sampling_ratio/mean": 1.0179672241210938,
      "sampling/importance_sampling_ratio/min": 0.1471695899963379,
      "sampling/sampling_logp_difference/max": 1.9455175399780273,
      "sampling/sampling_logp_difference/mean": 0.09774789214134216,
      "step": 147,
      "step_time": 13.433196441968903
    },
    {
      "clip_ratio/high_max": 0.013599537080153823,
      "clip_ratio/high_mean": 0.0067997685400769114,
      "clip_ratio/low_mean": 0.0011160714784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007915839902125299,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1129.0,
      "completions/max_terminated_length": 1129.0,
      "completions/mean_length": 229.5,
      "completions/mean_terminated_length": 221.7916717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6419080253690481,
      "epoch": 0.00296,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.186741828918457,
      "kl": 0.5629236744716763,
      "learning_rate": 9.694009300552081e-06,
      "loss": -0.1341,
      "num_tokens": 5893342.0,
      "reward": 0.5981249809265137,
      "reward_std": 0.5759212374687195,
      "rewards/rollout_reward_func/mean": 0.5981249809265137,
      "rewards/rollout_reward_func/std": 0.6027273535728455,
      "sampling/importance_sampling_ratio/max": 2.0248653888702393,
      "sampling/importance_sampling_ratio/mean": 0.7832241058349609,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 16.1875,
      "sampling/sampling_logp_difference/mean": 0.11777827143669128,
      "step": 148,
      "step_time": 13.729297392012086
    },
    {
      "clip_ratio/high_max": 0.003289473708719015,
      "clip_ratio/high_mean": 0.0016447368543595076,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0026212993543595076,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 1116.0,
      "completions/max_terminated_length": 667.0,
      "completions/mean_length": 216.5,
      "completions/mean_terminated_length": 181.7777862548828,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6181398537009954,
      "epoch": 0.00298,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.3164448738098145,
      "kl": 0.34297334775328636,
      "learning_rate": 9.68859802989196e-06,
      "loss": 0.1742,
      "num_tokens": 5933052.0,
      "reward": 0.8674999475479126,
      "reward_std": 0.23397208750247955,
      "rewards/rollout_reward_func/mean": 0.8674999475479126,
      "rewards/rollout_reward_func/std": 0.33361169695854187,
      "sampling/importance_sampling_ratio/max": 2.305260181427002,
      "sampling/importance_sampling_ratio/mean": 1.1210858821868896,
      "sampling/importance_sampling_ratio/min": 0.07800392806529999,
      "sampling/sampling_logp_difference/max": 0.9536380767822266,
      "sampling/sampling_logp_difference/mean": 0.08070062100887299,
      "step": 149,
      "step_time": 13.831906781051657
    },
    {
      "clip_ratio/high_max": 0.007211538730189204,
      "clip_ratio/high_mean": 0.003605769365094602,
      "clip_ratio/low_mean": 0.005876068491488695,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009481837856583297,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 808.0,
      "completions/max_terminated_length": 656.0,
      "completions/mean_length": 281.46875,
      "completions/mean_terminated_length": 240.7777862548828,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6283114366233349,
      "epoch": 0.003,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.054165244102478,
      "kl": 0.5733157526701689,
      "learning_rate": 9.683141401799326e-06,
      "loss": -0.0543,
      "num_tokens": 5975932.0,
      "reward": 0.4390625059604645,
      "reward_std": 0.8177142143249512,
      "rewards/rollout_reward_func/mean": 0.4390625059604645,
      "rewards/rollout_reward_func/std": 0.8098002672195435,
      "sampling/importance_sampling_ratio/max": 2.846447706222534,
      "sampling/importance_sampling_ratio/mean": 0.9145864844322205,
      "sampling/importance_sampling_ratio/min": 0.15574544668197632,
      "sampling/sampling_logp_difference/max": 1.4315319061279297,
      "sampling/sampling_logp_difference/mean": 0.09104137122631073,
      "step": 150,
      "step_time": 13.504170472995611
    },
    {
      "clip_ratio/high_max": 0.009542410960420966,
      "clip_ratio/high_mean": 0.004771205480210483,
      "clip_ratio/low_mean": 0.0030651914421468973,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00783639692235738,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 678.0,
      "completions/max_terminated_length": 678.0,
      "completions/mean_length": 231.53125,
      "completions/mean_terminated_length": 230.25001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7033243291079998,
      "epoch": 0.00302,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.932028889656067,
      "kl": 2.340105996467173,
      "learning_rate": 9.677639488250787e-06,
      "loss": 0.0027,
      "num_tokens": 6014982.0,
      "reward": 0.408518522977829,
      "reward_std": 0.733809232711792,
      "rewards/rollout_reward_func/mean": 0.408518522977829,
      "rewards/rollout_reward_func/std": 0.740959644317627,
      "sampling/importance_sampling_ratio/max": 1.62697172164917,
      "sampling/importance_sampling_ratio/mean": 0.7090562582015991,
      "sampling/importance_sampling_ratio/min": 3.915998486581345e-15,
      "sampling/sampling_logp_difference/max": 19.468786239624023,
      "sampling/sampling_logp_difference/mean": 0.262198269367218,
      "step": 151,
      "step_time": 11.762964030000148
    },
    {
      "clip_ratio/high_max": 0.006510416744276881,
      "clip_ratio/high_mean": 0.0032552083721384406,
      "clip_ratio/low_mean": 0.002921195700764656,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006176404072903097,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 675.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 198.1875,
      "completions/mean_terminated_length": 193.8518524169922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5682809539139271,
      "epoch": 0.00304,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6728379726409912,
      "kl": 0.9160190196707845,
      "learning_rate": 9.672092361820306e-06,
      "loss": -0.1578,
      "num_tokens": 6053818.0,
      "reward": 0.5441275835037231,
      "reward_std": 0.5824833512306213,
      "rewards/rollout_reward_func/mean": 0.5441275835037231,
      "rewards/rollout_reward_func/std": 0.7976074814796448,
      "sampling/importance_sampling_ratio/max": 2.4656546115875244,
      "sampling/importance_sampling_ratio/mean": 1.0299019813537598,
      "sampling/importance_sampling_ratio/min": 0.12011483311653137,
      "sampling/sampling_logp_difference/max": 1.583970069885254,
      "sampling/sampling_logp_difference/mean": 0.09314249455928802,
      "step": 152,
      "step_time": 12.697374891009531
    },
    {
      "clip_ratio/high_max": 0.018359375651925802,
      "clip_ratio/high_mean": 0.010156250325962901,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010156250325962901,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 563.0,
      "completions/max_terminated_length": 563.0,
      "completions/mean_length": 213.71875,
      "completions/mean_terminated_length": 186.25001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6385288722813129,
      "epoch": 0.00306,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2162017822265625,
      "kl": 0.8507515387609601,
      "learning_rate": 9.666500095678226e-06,
      "loss": -0.11,
      "num_tokens": 6094368.0,
      "reward": 0.5640624761581421,
      "reward_std": 0.6583585143089294,
      "rewards/rollout_reward_func/mean": 0.5640624761581421,
      "rewards/rollout_reward_func/std": 0.6641407608985901,
      "sampling/importance_sampling_ratio/max": 2.690807342529297,
      "sampling/importance_sampling_ratio/mean": 1.0759166479110718,
      "sampling/importance_sampling_ratio/min": 6.920134296706237e-07,
      "sampling/sampling_logp_difference/max": 5.725097179412842,
      "sampling/sampling_logp_difference/mean": 0.1284565031528473,
      "step": 153,
      "step_time": 12.587266943970462
    },
    {
      "clip_ratio/high_max": 0.029131081886589527,
      "clip_ratio/high_mean": 0.014565540943294764,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014565540943294764,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 792.0,
      "completions/max_terminated_length": 642.0,
      "completions/mean_length": 208.1875,
      "completions/mean_terminated_length": 170.36000061035156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6497909501194954,
      "epoch": 0.00308,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4286279678344727,
      "kl": 0.475322219543159,
      "learning_rate": 9.660862763590322e-06,
      "loss": -0.0039,
      "num_tokens": 6133809.0,
      "reward": 0.6228148937225342,
      "reward_std": 0.7126596570014954,
      "rewards/rollout_reward_func/mean": 0.6228148937225342,
      "rewards/rollout_reward_func/std": 0.7152525782585144,
      "sampling/importance_sampling_ratio/max": 2.8425827026367188,
      "sampling/importance_sampling_ratio/mean": 0.9635320901870728,
      "sampling/importance_sampling_ratio/min": 0.15776567161083221,
      "sampling/sampling_logp_difference/max": 1.0000925064086914,
      "sampling/sampling_logp_difference/mean": 0.08134534955024719,
      "step": 154,
      "step_time": 12.53454512997996
    },
    {
      "clip_ratio/high_max": 0.01903431979008019,
      "clip_ratio/high_mean": 0.009517159895040095,
      "clip_ratio/low_mean": 0.0072265625931322575,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016743722488172352,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 594.0,
      "completions/max_terminated_length": 594.0,
      "completions/mean_length": 201.90625,
      "completions/mean_terminated_length": 206.90000915527344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5872972588986158,
      "epoch": 0.0031,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.9927358627319336,
      "kl": 0.49799636378884315,
      "learning_rate": 9.655180439916814e-06,
      "loss": -0.0427,
      "num_tokens": 6173238.0,
      "reward": 0.7797646522521973,
      "reward_std": 0.42842286825180054,
      "rewards/rollout_reward_func/mean": 0.7797646522521973,
      "rewards/rollout_reward_func/std": 0.5074214339256287,
      "sampling/importance_sampling_ratio/max": 2.5467658042907715,
      "sampling/importance_sampling_ratio/mean": 1.0081865787506104,
      "sampling/importance_sampling_ratio/min": 3.2559504447135957e-16,
      "sampling/sampling_logp_difference/max": 25.224346160888672,
      "sampling/sampling_logp_difference/mean": 0.15465500950813293,
      "step": 155,
      "step_time": 12.476585128999432
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0031250000465661287,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005078125046566129,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 585.0,
      "completions/max_terminated_length": 585.0,
      "completions/mean_length": 216.0,
      "completions/mean_terminated_length": 223.70370483398438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6426145546138287,
      "epoch": 0.00312,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0875636339187622,
      "kl": 0.4497987683862448,
      "learning_rate": 9.64945319961139e-06,
      "loss": -0.1905,
      "num_tokens": 6214275.0,
      "reward": 0.6665624976158142,
      "reward_std": 0.5188271999359131,
      "rewards/rollout_reward_func/mean": 0.6665624976158142,
      "rewards/rollout_reward_func/std": 0.6710684895515442,
      "sampling/importance_sampling_ratio/max": 2.04431414604187,
      "sampling/importance_sampling_ratio/mean": 0.8580808043479919,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 26.875003814697266,
      "sampling/sampling_logp_difference/mean": 0.3067754805088043,
      "step": 156,
      "step_time": 12.864865207040566
    },
    {
      "clip_ratio/high_max": 0.015653121285140514,
      "clip_ratio/high_mean": 0.007826560642570257,
      "clip_ratio/low_mean": 0.004166666883975267,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011993227526545525,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 601.0,
      "completions/max_terminated_length": 601.0,
      "completions/mean_length": 160.90625,
      "completions/mean_terminated_length": 165.5806427001953,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.6242186240851879,
      "epoch": 0.00314,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1470022201538086,
      "kl": 0.4036520952358842,
      "learning_rate": 9.643681118220224e-06,
      "loss": 0.0044,
      "num_tokens": 6250881.0,
      "reward": 0.69593745470047,
      "reward_std": 0.5855188369750977,
      "rewards/rollout_reward_func/mean": 0.69593745470047,
      "rewards/rollout_reward_func/std": 0.599342942237854,
      "sampling/importance_sampling_ratio/max": 2.778062105178833,
      "sampling/importance_sampling_ratio/mean": 1.0153595209121704,
      "sampling/importance_sampling_ratio/min": 3.0066522730722023e-13,
      "sampling/sampling_logp_difference/max": 18.556039810180664,
      "sampling/sampling_logp_difference/mean": 0.15778356790542603,
      "step": 157,
      "step_time": 11.787645642980351
    },
    {
      "clip_ratio/high_max": 0.005291005363687873,
      "clip_ratio/high_mean": 0.0026455026818439364,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0026455026818439364,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 646.0,
      "completions/max_terminated_length": 646.0,
      "completions/mean_length": 179.25,
      "completions/mean_terminated_length": 181.9666748046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5210995767265558,
      "epoch": 0.00316,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6982234716415405,
      "kl": 0.9037424707785249,
      "learning_rate": 9.637864271880972e-06,
      "loss": -0.1858,
      "num_tokens": 6287355.0,
      "reward": 0.6512384414672852,
      "reward_std": 0.761395275592804,
      "rewards/rollout_reward_func/mean": 0.6512384414672852,
      "rewards/rollout_reward_func/std": 0.7329332828521729,
      "sampling/importance_sampling_ratio/max": 2.1109540462493896,
      "sampling/importance_sampling_ratio/mean": 0.9472735524177551,
      "sampling/importance_sampling_ratio/min": 0.1849900335073471,
      "sampling/sampling_logp_difference/max": 1.6734731197357178,
      "sampling/sampling_logp_difference/mean": 0.0849253237247467,
      "step": 158,
      "step_time": 12.479325327003608
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0048828125,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 222.53125,
      "completions/mean_terminated_length": 194.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7059572283178568,
      "epoch": 0.00318,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3048707246780396,
      "kl": 1.1110294272657484,
      "learning_rate": 9.632002737321768e-06,
      "loss": -0.326,
      "num_tokens": 6326952.0,
      "reward": 0.3487904965877533,
      "reward_std": 0.9287893176078796,
      "rewards/rollout_reward_func/mean": 0.3487904965877533,
      "rewards/rollout_reward_func/std": 0.9054158926010132,
      "sampling/importance_sampling_ratio/max": 2.417271137237549,
      "sampling/importance_sampling_ratio/mean": 0.9429965019226074,
      "sampling/importance_sampling_ratio/min": 0.2349337935447693,
      "sampling/sampling_logp_difference/max": 1.5389084815979004,
      "sampling/sampling_logp_difference/mean": 0.08810003101825714,
      "step": 159,
      "step_time": 12.45117519798805
    },
    {
      "clip_ratio/high_max": 0.012500000186264515,
      "clip_ratio/high_mean": 0.0062500000931322575,
      "clip_ratio/low_mean": 0.005078125046566129,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011328125139698386,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 218.96875,
      "completions/mean_terminated_length": 192.9629669189453,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.4995936043560505,
      "epoch": 0.0032,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8756972551345825,
      "kl": 0.5228236699476838,
      "learning_rate": 9.626096591860215e-06,
      "loss": -0.0179,
      "num_tokens": 6366309.0,
      "reward": 0.5048418045043945,
      "reward_std": 0.6796785593032837,
      "rewards/rollout_reward_func/mean": 0.5048418045043945,
      "rewards/rollout_reward_func/std": 0.6995879411697388,
      "sampling/importance_sampling_ratio/max": 2.6296703815460205,
      "sampling/importance_sampling_ratio/mean": 1.1948684453964233,
      "sampling/importance_sampling_ratio/min": 1.658333412581984e-16,
      "sampling/sampling_logp_difference/max": 22.117448806762695,
      "sampling/sampling_logp_difference/mean": 0.15599307417869568,
      "step": 160,
      "step_time": 11.992413977030083
    },
    {
      "clip_ratio/high_max": 0.011451511178165674,
      "clip_ratio/high_mean": 0.005725755589082837,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005725755589082837,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 603.0,
      "completions/max_terminated_length": 603.0,
      "completions/mean_length": 136.625,
      "completions/mean_terminated_length": 136.06668090820312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6318340767174959,
      "epoch": 0.00322,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8889428973197937,
      "kl": 0.9561359947547317,
      "learning_rate": 9.620145913402366e-06,
      "loss": -0.1899,
      "num_tokens": 6403356.0,
      "reward": 0.5773417949676514,
      "reward_std": 0.7856334447860718,
      "rewards/rollout_reward_func/mean": 0.5773417949676514,
      "rewards/rollout_reward_func/std": 0.7714837193489075,
      "sampling/importance_sampling_ratio/max": 1.7133136987686157,
      "sampling/importance_sampling_ratio/mean": 0.7404782176017761,
      "sampling/importance_sampling_ratio/min": 3.685342821719878e-22,
      "sampling/sampling_logp_difference/max": 27.165964126586914,
      "sampling/sampling_logp_difference/mean": 0.4487418234348297,
      "step": 161,
      "step_time": 12.751903334006784
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004185267956927419,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 621.0,
      "completions/max_terminated_length": 621.0,
      "completions/mean_length": 193.625,
      "completions/mean_terminated_length": 209.00001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6539102606475353,
      "epoch": 0.00324,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2899484634399414,
      "kl": 4.842572920024395,
      "learning_rate": 9.614150780441687e-06,
      "loss": -0.1954,
      "num_tokens": 6441903.0,
      "reward": 0.4540625214576721,
      "reward_std": 0.7212908267974854,
      "rewards/rollout_reward_func/mean": 0.4540625214576721,
      "rewards/rollout_reward_func/std": 0.7868633270263672,
      "sampling/importance_sampling_ratio/max": 2.0364341735839844,
      "sampling/importance_sampling_ratio/mean": 0.9261319637298584,
      "sampling/importance_sampling_ratio/min": 0.1833428293466568,
      "sampling/sampling_logp_difference/max": 1.6149730682373047,
      "sampling/sampling_logp_difference/mean": 0.09070277214050293,
      "step": 162,
      "step_time": 11.77563173697854
    },
    {
      "clip_ratio/high_max": 0.0210767665412277,
      "clip_ratio/high_mean": 0.01053838327061385,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01053838327061385,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 593.0,
      "completions/max_terminated_length": 563.0,
      "completions/mean_length": 242.125,
      "completions/mean_terminated_length": 233.1785888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5451669413596392,
      "epoch": 0.00326,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.346292018890381,
      "kl": 1.0485903392545879,
      "learning_rate": 9.608111272058044e-06,
      "loss": -0.0194,
      "num_tokens": 6482322.0,
      "reward": 0.7868750095367432,
      "reward_std": 0.5697559118270874,
      "rewards/rollout_reward_func/mean": 0.7868750095367432,
      "rewards/rollout_reward_func/std": 0.591910183429718,
      "sampling/importance_sampling_ratio/max": 2.235002040863037,
      "sampling/importance_sampling_ratio/mean": 0.9645026326179504,
      "sampling/importance_sampling_ratio/min": 0.18357418477535248,
      "sampling/sampling_logp_difference/max": 1.6208257675170898,
      "sampling/sampling_logp_difference/mean": 0.08051663637161255,
      "step": 163,
      "step_time": 11.95449039999221
    },
    {
      "clip_ratio/high_max": 0.006414473755285144,
      "clip_ratio/high_mean": 0.003207236877642572,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003207236877642572,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 675.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 238.84375,
      "completions/mean_terminated_length": 244.9666748046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5281211491674185,
      "epoch": 0.00328,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9561030864715576,
      "kl": 1.5573012195527554,
      "learning_rate": 9.602027467916626e-06,
      "loss": -0.1546,
      "num_tokens": 6523370.0,
      "reward": 0.43937498331069946,
      "reward_std": 0.8622356057167053,
      "rewards/rollout_reward_func/mean": 0.43937498331069946,
      "rewards/rollout_reward_func/std": 0.8380081057548523,
      "sampling/importance_sampling_ratio/max": 2.1034114360809326,
      "sampling/importance_sampling_ratio/mean": 0.9557827711105347,
      "sampling/importance_sampling_ratio/min": 0.08538348972797394,
      "sampling/sampling_logp_difference/max": 1.485581398010254,
      "sampling/sampling_logp_difference/mean": 0.08948000520467758,
      "step": 164,
      "step_time": 12.963660976005485
    },
    {
      "clip_ratio/high_max": 0.017578125465661287,
      "clip_ratio/high_mean": 0.011021205922588706,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011997768422588706,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 685.0,
      "completions/max_terminated_length": 685.0,
      "completions/mean_length": 200.875,
      "completions/mean_terminated_length": 200.4615478515625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5930871665477753,
      "epoch": 0.0033,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2620450258255005,
      "kl": 0.6747720595449209,
      "learning_rate": 9.595899448266928e-06,
      "loss": -0.1127,
      "num_tokens": 6561863.0,
      "reward": 0.687580943107605,
      "reward_std": 0.5047495365142822,
      "rewards/rollout_reward_func/mean": 0.687580943107605,
      "rewards/rollout_reward_func/std": 0.5907595157623291,
      "sampling/importance_sampling_ratio/max": 2.12589693069458,
      "sampling/importance_sampling_ratio/mean": 0.8168007135391235,
      "sampling/importance_sampling_ratio/min": 6.025827538634404e-15,
      "sampling/sampling_logp_difference/max": 23.687612533569336,
      "sampling/sampling_logp_difference/mean": 0.4299688935279846,
      "step": 165,
      "step_time": 11.937944646007963
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.005440848413854837,
      "clip_ratio/low_min": 0.001953125,
      "clip_ratio/region_mean": 0.007393973413854837,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 238.59375,
      "completions/mean_terminated_length": 225.39999389648438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5889310110360384,
      "epoch": 0.00332,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9688844680786133,
      "kl": 1.8328271862119436,
      "learning_rate": 9.589727293941669e-06,
      "loss": -0.2768,
      "num_tokens": 6602060.0,
      "reward": 0.37254050374031067,
      "reward_std": 0.7893848419189453,
      "rewards/rollout_reward_func/mean": 0.37254050374031067,
      "rewards/rollout_reward_func/std": 0.8218263983726501,
      "sampling/importance_sampling_ratio/max": 2.166555881500244,
      "sampling/importance_sampling_ratio/mean": 0.8915714025497437,
      "sampling/importance_sampling_ratio/min": 9.658892456877342e-16,
      "sampling/sampling_logp_difference/max": 19.468765258789062,
      "sampling/sampling_logp_difference/mean": 0.21576252579689026,
      "step": 166,
      "step_time": 12.821771912000258
    },
    {
      "clip_ratio/high_max": 0.01171875,
      "clip_ratio/high_mean": 0.005859375,
      "clip_ratio/low_mean": 0.025390625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 584.0,
      "completions/max_terminated_length": 584.0,
      "completions/mean_length": 220.28125,
      "completions/mean_terminated_length": 235.0357208251953,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6141235437244177,
      "epoch": 0.00334,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6322352886199951,
      "kl": 1.6807297486811876,
      "learning_rate": 9.583511086355738e-06,
      "loss": -0.2015,
      "num_tokens": 6643450.0,
      "reward": 0.381290465593338,
      "reward_std": 0.799941897392273,
      "rewards/rollout_reward_func/mean": 0.381290465593338,
      "rewards/rollout_reward_func/std": 0.8110588788986206,
      "sampling/importance_sampling_ratio/max": 2.475778818130493,
      "sampling/importance_sampling_ratio/mean": 1.0223231315612793,
      "sampling/importance_sampling_ratio/min": 3.4768539526636754e-23,
      "sampling/sampling_logp_difference/max": 21.225595474243164,
      "sampling/sampling_logp_difference/mean": 0.22400686144828796,
      "step": 167,
      "step_time": 12.229573454023921
    },
    {
      "clip_ratio/high_max": 0.028831845615059137,
      "clip_ratio/high_mean": 0.017796331318095326,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018772893818095326,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 599.0,
      "completions/mean_length": 291.28125,
      "completions/mean_terminated_length": 252.27999877929688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6098360568284988,
      "epoch": 0.00336,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2481215000152588,
      "kl": 0.5118815079331398,
      "learning_rate": 9.577250907505117e-06,
      "loss": -0.1855,
      "num_tokens": 6686181.0,
      "reward": 0.6834374666213989,
      "reward_std": 0.5138853788375854,
      "rewards/rollout_reward_func/mean": 0.6834374666213989,
      "rewards/rollout_reward_func/std": 0.5946568846702576,
      "sampling/importance_sampling_ratio/max": 1.933505892753601,
      "sampling/importance_sampling_ratio/mean": 0.9270507097244263,
      "sampling/importance_sampling_ratio/min": 0.31895744800567627,
      "sampling/sampling_logp_difference/max": 1.228039264678955,
      "sampling/sampling_logp_difference/mean": 0.08163121342658997,
      "step": 168,
      "step_time": 12.439842129984754
    },
    {
      "clip_ratio/high_max": 0.010844494332559407,
      "clip_ratio/high_mean": 0.005422247166279703,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005422247166279703,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 900.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 247.375,
      "completions/mean_terminated_length": 216.40740966796875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5805898308753967,
      "epoch": 0.00338,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.412109613418579,
      "kl": 1.6751377992331982,
      "learning_rate": 9.570946839965795e-06,
      "loss": -0.1013,
      "num_tokens": 6725965.0,
      "reward": 0.5603751540184021,
      "reward_std": 0.6792531609535217,
      "rewards/rollout_reward_func/mean": 0.5603751540184021,
      "rewards/rollout_reward_func/std": 0.7369281649589539,
      "sampling/importance_sampling_ratio/max": 2.6315906047821045,
      "sampling/importance_sampling_ratio/mean": 1.0196866989135742,
      "sampling/importance_sampling_ratio/min": 0.1364004760980606,
      "sampling/sampling_logp_difference/max": 1.4056603908538818,
      "sampling/sampling_logp_difference/mean": 0.08216319978237152,
      "step": 169,
      "step_time": 12.99604926999018
    },
    {
      "clip_ratio/high_max": 0.016608297592028975,
      "clip_ratio/high_mean": 0.008304148796014488,
      "clip_ratio/low_mean": 0.010416666860692203,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01872081565670669,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 656.0,
      "completions/mean_length": 182.96875,
      "completions/mean_terminated_length": 159.8928680419922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6093393974006176,
      "epoch": 0.0034,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8998780846595764,
      "kl": 0.9295055195689201,
      "learning_rate": 9.564598966892683e-06,
      "loss": -0.1229,
      "num_tokens": 6765524.0,
      "reward": 0.3416975140571594,
      "reward_std": 0.8389285802841187,
      "rewards/rollout_reward_func/mean": 0.3416975140571594,
      "rewards/rollout_reward_func/std": 0.8216835260391235,
      "sampling/importance_sampling_ratio/max": 1.7307997941970825,
      "sampling/importance_sampling_ratio/mean": 0.8249015808105469,
      "sampling/importance_sampling_ratio/min": 1.7066148665969158e-22,
      "sampling/sampling_logp_difference/max": 27.172016143798828,
      "sampling/sampling_logp_difference/mean": 0.25124943256378174,
      "step": 170,
      "step_time": 12.745819260991993
    },
    {
      "clip_ratio/high_max": 0.005629595601931214,
      "clip_ratio/high_mean": 0.002814797800965607,
      "clip_ratio/low_mean": 0.017578125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020392922800965607,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 695.0,
      "completions/max_terminated_length": 695.0,
      "completions/mean_length": 293.8125,
      "completions/mean_terminated_length": 288.3448181152344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6060059294104576,
      "epoch": 0.00342,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4411683082580566,
      "kl": 1.7018772410228848,
      "learning_rate": 9.558207372018519e-06,
      "loss": -0.2315,
      "num_tokens": 6808839.0,
      "reward": 0.46611785888671875,
      "reward_std": 0.586765706539154,
      "rewards/rollout_reward_func/mean": 0.46611785888671875,
      "rewards/rollout_reward_func/std": 0.7949056625366211,
      "sampling/importance_sampling_ratio/max": 1.978875756263733,
      "sampling/importance_sampling_ratio/mean": 0.8585664629936218,
      "sampling/importance_sampling_ratio/min": 0.20861868560314178,
      "sampling/sampling_logp_difference/max": 1.3022489547729492,
      "sampling/sampling_logp_difference/mean": 0.08260895311832428,
      "step": 171,
      "step_time": 12.325890433014138
    },
    {
      "clip_ratio/high_max": 0.005842391401529312,
      "clip_ratio/high_mean": 0.002921195700764656,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002921195700764656,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 638.0,
      "completions/mean_length": 237.5625,
      "completions/mean_terminated_length": 231.00001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5098046790808439,
      "epoch": 0.00344,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.4714841842651367,
      "kl": 1.753780035302043,
      "learning_rate": 9.551772139652762e-06,
      "loss": 0.078,
      "num_tokens": 6848962.0,
      "reward": 0.5620987415313721,
      "reward_std": 0.6436513662338257,
      "rewards/rollout_reward_func/mean": 0.5620987415313721,
      "rewards/rollout_reward_func/std": 0.6798839569091797,
      "sampling/importance_sampling_ratio/max": 2.4833290576934814,
      "sampling/importance_sampling_ratio/mean": 1.0881335735321045,
      "sampling/importance_sampling_ratio/min": 0.1326587051153183,
      "sampling/sampling_logp_difference/max": 1.2535791397094727,
      "sampling/sampling_logp_difference/mean": 0.0836268961429596,
      "step": 172,
      "step_time": 12.189697175024776
    },
    {
      "clip_ratio/high_max": 0.014322916977107525,
      "clip_ratio/high_mean": 0.0071614584885537624,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0071614584885537624,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 811.0,
      "completions/max_terminated_length": 811.0,
      "completions/mean_length": 310.65625,
      "completions/mean_terminated_length": 282.8571472167969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5864296276122332,
      "epoch": 0.00346,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8648294806480408,
      "kl": 1.396013897843659,
      "learning_rate": 9.545293354680477e-06,
      "loss": -0.1798,
      "num_tokens": 6892658.0,
      "reward": 0.6169376373291016,
      "reward_std": 0.6859169006347656,
      "rewards/rollout_reward_func/mean": 0.6169376373291016,
      "rewards/rollout_reward_func/std": 0.6900351047515869,
      "sampling/importance_sampling_ratio/max": 1.7732200622558594,
      "sampling/importance_sampling_ratio/mean": 0.7665269374847412,
      "sampling/importance_sampling_ratio/min": 5.07109361098289e-18,
      "sampling/sampling_logp_difference/max": 21.813140869140625,
      "sampling/sampling_logp_difference/mean": 0.15407614409923553,
      "step": 173,
      "step_time": 13.18019387597451
    },
    {
      "clip_ratio/high_max": 0.02009637444280088,
      "clip_ratio/high_mean": 0.01004818722140044,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01200131222140044,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 647.0,
      "completions/mean_length": 190.25,
      "completions/mean_terminated_length": 197.42857360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5285902433097363,
      "epoch": 0.00348,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1867773532867432,
      "kl": 0.7869226029142737,
      "learning_rate": 9.538771102561212e-06,
      "loss": 0.0342,
      "num_tokens": 6930356.0,
      "reward": 0.7125405073165894,
      "reward_std": 0.43634650111198425,
      "rewards/rollout_reward_func/mean": 0.7125405073165894,
      "rewards/rollout_reward_func/std": 0.5294061899185181,
      "sampling/importance_sampling_ratio/max": 1.5447808504104614,
      "sampling/importance_sampling_ratio/mean": 0.779205858707428,
      "sampling/importance_sampling_ratio/min": 3.1227461096028334e-17,
      "sampling/sampling_logp_difference/max": 20.406782150268555,
      "sampling/sampling_logp_difference/mean": 0.2531192898750305,
      "step": 174,
      "step_time": 11.865168097021524
    },
    {
      "clip_ratio/high_max": 0.0400000000372529,
      "clip_ratio/high_mean": 0.02000000001862645,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02000000001862645,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 667.0,
      "completions/max_terminated_length": 667.0,
      "completions/mean_length": 226.875,
      "completions/mean_terminated_length": 206.40740966796875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5276930816471577,
      "epoch": 0.0035,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.31081223487854,
      "kl": 0.35314200073480606,
      "learning_rate": 9.53220546932789e-06,
      "loss": -0.06,
      "num_tokens": 6969787.0,
      "reward": 0.79749995470047,
      "reward_std": 0.4195187985897064,
      "rewards/rollout_reward_func/mean": 0.79749995470047,
      "rewards/rollout_reward_func/std": 0.4180368483066559,
      "sampling/importance_sampling_ratio/max": 2.2622950077056885,
      "sampling/importance_sampling_ratio/mean": 0.9426403641700745,
      "sampling/importance_sampling_ratio/min": 4.296868610254023e-06,
      "sampling/sampling_logp_difference/max": 12.78437328338623,
      "sampling/sampling_logp_difference/mean": 0.09832639992237091,
      "step": 175,
      "step_time": 12.640595517979818
    },
    {
      "clip_ratio/high_max": 0.02575231483206153,
      "clip_ratio/high_mean": 0.012876157416030765,
      "clip_ratio/low_mean": 0.00390625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016782407416030765,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 594.0,
      "completions/max_terminated_length": 594.0,
      "completions/mean_length": 203.03125,
      "completions/mean_terminated_length": 179.9615478515625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5207838732749224,
      "epoch": 0.00352,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1103922128677368,
      "kl": 0.6533062632661313,
      "learning_rate": 9.525596541585646e-06,
      "loss": 0.0255,
      "num_tokens": 7009336.0,
      "reward": 0.7518750429153442,
      "reward_std": 0.443362295627594,
      "rewards/rollout_reward_func/mean": 0.7518750429153442,
      "rewards/rollout_reward_func/std": 0.446400910615921,
      "sampling/importance_sampling_ratio/max": 2.274035930633545,
      "sampling/importance_sampling_ratio/mean": 0.8899685144424438,
      "sampling/importance_sampling_ratio/min": 5.287497206874858e-21,
      "sampling/sampling_logp_difference/max": 21.17436981201172,
      "sampling/sampling_logp_difference/mean": 0.17463864386081696,
      "step": 176,
      "step_time": 12.607181970000966
    },
    {
      "clip_ratio/high_max": 0.0034722222480922937,
      "clip_ratio/high_mean": 0.0017361111240461469,
      "clip_ratio/low_mean": 0.01893682067748159,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02067293180152774,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 620.0,
      "completions/max_terminated_length": 566.0,
      "completions/mean_length": 235.46875,
      "completions/mean_terminated_length": 228.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5687723346054554,
      "epoch": 0.00354,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3131235837936401,
      "kl": 1.273395037278533,
      "learning_rate": 9.518944406510707e-06,
      "loss": -0.2013,
      "num_tokens": 7049982.0,
      "reward": 0.5934664011001587,
      "reward_std": 0.7731683850288391,
      "rewards/rollout_reward_func/mean": 0.5934664011001587,
      "rewards/rollout_reward_func/std": 0.7967554926872253,
      "sampling/importance_sampling_ratio/max": 2.013603687286377,
      "sampling/importance_sampling_ratio/mean": 1.0102488994598389,
      "sampling/importance_sampling_ratio/min": 0.20647786557674408,
      "sampling/sampling_logp_difference/max": 1.3961691856384277,
      "sampling/sampling_logp_difference/mean": 0.06925071775913239,
      "step": 177,
      "step_time": 12.280920007047825
    },
    {
      "clip_ratio/high_max": 0.014843750046566129,
      "clip_ratio/high_mean": 0.007421875023283064,
      "clip_ratio/low_mean": 0.0016447368543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009066611877642572,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 649.0,
      "completions/max_terminated_length": 649.0,
      "completions/mean_length": 289.96875,
      "completions/mean_terminated_length": 306.96295166015625,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.567922180518508,
      "epoch": 0.00356,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.615379810333252,
      "kl": 0.3425546446815133,
      "learning_rate": 9.512249151849229e-06,
      "loss": -0.0458,
      "num_tokens": 7092733.0,
      "reward": 0.6347501277923584,
      "reward_std": 0.5351201295852661,
      "rewards/rollout_reward_func/mean": 0.6347501277923584,
      "rewards/rollout_reward_func/std": 0.6230155825614929,
      "sampling/importance_sampling_ratio/max": 2.2039856910705566,
      "sampling/importance_sampling_ratio/mean": 1.1244844198226929,
      "sampling/importance_sampling_ratio/min": 0.1714639961719513,
      "sampling/sampling_logp_difference/max": 1.1202280521392822,
      "sampling/sampling_logp_difference/mean": 0.05906744673848152,
      "step": 178,
      "step_time": 13.340312599000754
    },
    {
      "clip_ratio/high_max": 0.01622111350297928,
      "clip_ratio/high_mean": 0.00811055675148964,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00811055675148964,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 990.0,
      "completions/max_terminated_length": 647.0,
      "completions/mean_length": 302.28125,
      "completions/mean_terminated_length": 288.9000244140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5950467269867659,
      "epoch": 0.00358,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1777673959732056,
      "kl": 1.322785799857229,
      "learning_rate": 9.50551086591615e-06,
      "loss": -0.0672,
      "num_tokens": 7134995.0,
      "reward": 0.4737017750740051,
      "reward_std": 0.7216402292251587,
      "rewards/rollout_reward_func/mean": 0.4737017750740051,
      "rewards/rollout_reward_func/std": 0.7460048794746399,
      "sampling/importance_sampling_ratio/max": 1.5238367319107056,
      "sampling/importance_sampling_ratio/mean": 0.8359778523445129,
      "sampling/importance_sampling_ratio/min": 3.703902251572799e-08,
      "sampling/sampling_logp_difference/max": 17.562501907348633,
      "sampling/sampling_logp_difference/mean": 0.12688526511192322,
      "step": 179,
      "step_time": 13.122614931009593
    },
    {
      "clip_ratio/high_max": 0.01442945096641779,
      "clip_ratio/high_mean": 0.007214725483208895,
      "clip_ratio/low_mean": 0.01704545458778739,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.024260180653072894,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 267.75,
      "completions/mean_terminated_length": 258.64288330078125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6254449523985386,
      "epoch": 0.0036,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9993259906768799,
      "kl": 0.8624154757708311,
      "learning_rate": 9.498729637594016e-06,
      "loss": -0.1594,
      "num_tokens": 7177225.0,
      "reward": 0.4703124761581421,
      "reward_std": 0.7149204611778259,
      "rewards/rollout_reward_func/mean": 0.4703124761581421,
      "rewards/rollout_reward_func/std": 0.7249715328216553,
      "sampling/importance_sampling_ratio/max": 1.941190242767334,
      "sampling/importance_sampling_ratio/mean": 0.9535425305366516,
      "sampling/importance_sampling_ratio/min": 7.72347477314729e-14,
      "sampling/sampling_logp_difference/max": 23.505964279174805,
      "sampling/sampling_logp_difference/mean": 0.13156738877296448,
      "step": 180,
      "step_time": 12.264473582006758
    },
    {
      "clip_ratio/high_max": 0.004166666883975267,
      "clip_ratio/high_mean": 0.0020833334419876337,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0020833334419876337,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1036.0,
      "completions/max_terminated_length": 1036.0,
      "completions/mean_length": 274.3125,
      "completions/mean_terminated_length": 277.3333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5600980054587126,
      "epoch": 0.00362,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.672725796699524,
      "kl": 1.1422591521404684,
      "learning_rate": 9.491905556331809e-06,
      "loss": -0.044,
      "num_tokens": 7219488.0,
      "reward": 0.6421898603439331,
      "reward_std": 0.5830928087234497,
      "rewards/rollout_reward_func/mean": 0.6421898603439331,
      "rewards/rollout_reward_func/std": 0.6245945692062378,
      "sampling/importance_sampling_ratio/max": 2.504362106323242,
      "sampling/importance_sampling_ratio/mean": 1.058903694152832,
      "sampling/importance_sampling_ratio/min": 1.1067334521703762e-13,
      "sampling/sampling_logp_difference/max": 18.627723693847656,
      "sampling/sampling_logp_difference/mean": 0.13569439947605133,
      "step": 181,
      "step_time": 13.953537189983763
    },
    {
      "clip_ratio/high_max": 0.010650785407051444,
      "clip_ratio/high_mean": 0.005325392703525722,
      "clip_ratio/low_mean": 0.005425347248092294,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010750739951618016,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 299.5,
      "completions/mean_terminated_length": 269.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5549353305250406,
      "epoch": 0.00364,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0464526414871216,
      "kl": 0.500805826857686,
      "learning_rate": 9.485038712143778e-06,
      "loss": -0.1472,
      "num_tokens": 7262976.0,
      "reward": 0.6037499904632568,
      "reward_std": 0.5743985176086426,
      "rewards/rollout_reward_func/mean": 0.6037499904632568,
      "rewards/rollout_reward_func/std": 0.6125607490539551,
      "sampling/importance_sampling_ratio/max": 1.831450343132019,
      "sampling/importance_sampling_ratio/mean": 0.9855327606201172,
      "sampling/importance_sampling_ratio/min": 0.24585458636283875,
      "sampling/sampling_logp_difference/max": 1.0293011665344238,
      "sampling/sampling_logp_difference/mean": 0.06947404146194458,
      "step": 182,
      "step_time": 12.332429420974222
    },
    {
      "clip_ratio/high_max": 0.006119791883975267,
      "clip_ratio/high_mean": 0.0030598959419876337,
      "clip_ratio/low_mean": 0.017187500023283064,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020247395732440054,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 688.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 307.8125,
      "completions/mean_terminated_length": 283.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5713469218462706,
      "epoch": 0.00366,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7186897993087769,
      "kl": 0.8019076678901911,
      "learning_rate": 9.478129195608238e-06,
      "loss": -0.0649,
      "num_tokens": 7307086.0,
      "reward": 0.4689737558364868,
      "reward_std": 0.6707086563110352,
      "rewards/rollout_reward_func/mean": 0.4689737558364868,
      "rewards/rollout_reward_func/std": 0.7036197185516357,
      "sampling/importance_sampling_ratio/max": 2.441429376602173,
      "sampling/importance_sampling_ratio/mean": 1.0366175174713135,
      "sampling/importance_sampling_ratio/min": 3.7558434988062216e-26,
      "sampling/sampling_logp_difference/max": 25.52202796936035,
      "sampling/sampling_logp_difference/mean": 0.1893504112958908,
      "step": 183,
      "step_time": 12.916372195046279
    },
    {
      "clip_ratio/high_max": 0.012296262430027127,
      "clip_ratio/high_mean": 0.006148131215013564,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006148131215013564,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1134.0,
      "completions/max_terminated_length": 611.0,
      "completions/mean_length": 243.9375,
      "completions/mean_terminated_length": 203.19232177734375,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.638567304238677,
      "epoch": 0.00368,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3909178972244263,
      "kl": 0.5191398691385984,
      "learning_rate": 9.471177097866384e-06,
      "loss": -0.11,
      "num_tokens": 7347463.0,
      "reward": 0.5765625238418579,
      "reward_std": 0.7141919136047363,
      "rewards/rollout_reward_func/mean": 0.5765625238418579,
      "rewards/rollout_reward_func/std": 0.6862126588821411,
      "sampling/importance_sampling_ratio/max": 1.9582726955413818,
      "sampling/importance_sampling_ratio/mean": 0.909564733505249,
      "sampling/importance_sampling_ratio/min": 0.14938926696777344,
      "sampling/sampling_logp_difference/max": 1.0604438781738281,
      "sampling/sampling_logp_difference/mean": 0.07333233952522278,
      "step": 184,
      "step_time": 14.091833217011299
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.0038543642731383443,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006784051773138344,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 580.0,
      "completions/max_terminated_length": 580.0,
      "completions/mean_length": 223.5,
      "completions/mean_terminated_length": 210.6538543701172,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5837820526212454,
      "epoch": 0.0037,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4144370555877686,
      "kl": 0.7925476990640163,
      "learning_rate": 9.464182510621085e-06,
      "loss": -0.2247,
      "num_tokens": 7386568.0,
      "reward": 0.6515624523162842,
      "reward_std": 0.5351539254188538,
      "rewards/rollout_reward_func/mean": 0.6515624523162842,
      "rewards/rollout_reward_func/std": 0.6106321811676025,
      "sampling/importance_sampling_ratio/max": 1.8691085577011108,
      "sampling/importance_sampling_ratio/mean": 0.9294899106025696,
      "sampling/importance_sampling_ratio/min": 0.2194405496120453,
      "sampling/sampling_logp_difference/max": 1.2376221418380737,
      "sampling/sampling_logp_difference/mean": 0.06910160183906555,
      "step": 185,
      "step_time": 12.041593529982492
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0020833334419876337,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0020833334419876337,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 268.0625,
      "completions/mean_terminated_length": 274.1481628417969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6851822938770056,
      "epoch": 0.00372,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6169766187667847,
      "kl": 2.0097445189021528,
      "learning_rate": 9.457145526135673e-06,
      "loss": -0.0456,
      "num_tokens": 7429295.0,
      "reward": 0.546875,
      "reward_std": 0.47788330912590027,
      "rewards/rollout_reward_func/mean": 0.546875,
      "rewards/rollout_reward_func/std": 0.6169353127479553,
      "sampling/importance_sampling_ratio/max": 1.7818255424499512,
      "sampling/importance_sampling_ratio/mean": 0.8511656522750854,
      "sampling/importance_sampling_ratio/min": 1.4460955645585294e-20,
      "sampling/sampling_logp_difference/max": 22.144132614135742,
      "sampling/sampling_logp_difference/mean": 0.26597291231155396,
      "step": 186,
      "step_time": 13.132937572008814
    },
    {
      "clip_ratio/high_max": 0.0020833334419876337,
      "clip_ratio/high_mean": 0.0010416667209938169,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002018229220993817,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 604.0,
      "completions/max_terminated_length": 604.0,
      "completions/mean_length": 124.15625,
      "completions/mean_terminated_length": 116.13793182373047,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5823411177843809,
      "epoch": 0.00374,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0815694332122803,
      "kl": 0.8742435676977038,
      "learning_rate": 9.450066237232736e-06,
      "loss": -0.1519,
      "num_tokens": 7463311.0,
      "reward": 0.6302237510681152,
      "reward_std": 0.6122280955314636,
      "rewards/rollout_reward_func/mean": 0.6302237510681152,
      "rewards/rollout_reward_func/std": 0.6696856021881104,
      "sampling/importance_sampling_ratio/max": 2.6652073860168457,
      "sampling/importance_sampling_ratio/mean": 1.0546953678131104,
      "sampling/importance_sampling_ratio/min": 1.0193296207106073e-09,
      "sampling/sampling_logp_difference/max": 10.757355690002441,
      "sampling/sampling_logp_difference/mean": 0.12849129736423492,
      "step": 187,
      "step_time": 11.896128285981831
    },
    {
      "clip_ratio/high_max": 0.025303819682449102,
      "clip_ratio/high_mean": 0.012651909841224551,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012651909841224551,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 1041.0,
      "completions/max_terminated_length": 1041.0,
      "completions/mean_length": 254.59375,
      "completions/mean_terminated_length": 248.3333282470703,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.6102704927325249,
      "epoch": 0.00376,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2257951498031616,
      "kl": 1.4862774121575058,
      "learning_rate": 9.442944737292872e-06,
      "loss": 0.0076,
      "num_tokens": 7503493.0,
      "reward": 0.7047907114028931,
      "reward_std": 0.6852116584777832,
      "rewards/rollout_reward_func/mean": 0.7047907114028931,
      "rewards/rollout_reward_func/std": 0.6659271717071533,
      "sampling/importance_sampling_ratio/max": 1.837454080581665,
      "sampling/importance_sampling_ratio/mean": 0.8761759996414185,
      "sampling/importance_sampling_ratio/min": 0.19186674058437347,
      "sampling/sampling_logp_difference/max": 1.577216625213623,
      "sampling/sampling_logp_difference/mean": 0.06978730857372284,
      "step": 188,
      "step_time": 13.51416221402178
    },
    {
      "clip_ratio/high_max": 0.013315515592694283,
      "clip_ratio/high_mean": 0.006657757796347141,
      "clip_ratio/low_mean": 0.0029296875,
      "clip_ratio/low_min": 0.001953125,
      "clip_ratio/region_mean": 0.009587445296347141,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 646.0,
      "completions/max_terminated_length": 646.0,
      "completions/mean_length": 269.5,
      "completions/mean_terminated_length": 277.3461608886719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.677584882825613,
      "epoch": 0.00378,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2788223028182983,
      "kl": 0.7329649403691292,
      "learning_rate": 9.435781120253486e-06,
      "loss": -0.2009,
      "num_tokens": 7544878.0,
      "reward": 0.5832045078277588,
      "reward_std": 0.7163327932357788,
      "rewards/rollout_reward_func/mean": 0.5832045078277588,
      "rewards/rollout_reward_func/std": 0.6919088959693909,
      "sampling/importance_sampling_ratio/max": 1.8099701404571533,
      "sampling/importance_sampling_ratio/mean": 0.9009370803833008,
      "sampling/importance_sampling_ratio/min": 3.009052750366107e-19,
      "sampling/sampling_logp_difference/max": 19.108732223510742,
      "sampling/sampling_logp_difference/mean": 0.16418960690498352,
      "step": 189,
      "step_time": 12.839160055998946
    },
    {
      "clip_ratio/high_max": 0.010512907756492496,
      "clip_ratio/high_mean": 0.005256453878246248,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006233016378246248,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 649.0,
      "completions/max_terminated_length": 649.0,
      "completions/mean_length": 260.34375,
      "completions/mean_terminated_length": 266.478271484375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7452936768531799,
      "epoch": 0.0038,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7833266258239746,
      "kl": 0.2633369229733944,
      "learning_rate": 9.428575480607526e-06,
      "loss": 0.0553,
      "num_tokens": 7585640.0,
      "reward": 0.6737499833106995,
      "reward_std": 0.4986758828163147,
      "rewards/rollout_reward_func/mean": 0.6737499833106995,
      "rewards/rollout_reward_func/std": 0.5448216199874878,
      "sampling/importance_sampling_ratio/max": 1.9240825176239014,
      "sampling/importance_sampling_ratio/mean": 0.8584877252578735,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.1995301246643066,
      "sampling/sampling_logp_difference/mean": 0.07063819468021393,
      "step": 190,
      "step_time": 12.796360532986
    },
    {
      "clip_ratio/high_max": 0.015625000465661287,
      "clip_ratio/high_mean": 0.007812500232830644,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007812500232830644,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1081.0,
      "completions/max_terminated_length": 1081.0,
      "completions/mean_length": 286.75,
      "completions/mean_terminated_length": 332.69232177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6620315797626972,
      "epoch": 0.00382,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5228700637817383,
      "kl": 0.3365844548679888,
      "learning_rate": 9.421327913402252e-06,
      "loss": -0.0211,
      "num_tokens": 7627228.0,
      "reward": 0.49629050493240356,
      "reward_std": 0.7927142977714539,
      "rewards/rollout_reward_func/mean": 0.49629050493240356,
      "rewards/rollout_reward_func/std": 0.77347332239151,
      "sampling/importance_sampling_ratio/max": 2.1431820392608643,
      "sampling/importance_sampling_ratio/mean": 1.003544569015503,
      "sampling/importance_sampling_ratio/min": 4.4510898439265034e-11,
      "sampling/sampling_logp_difference/max": 20.459840774536133,
      "sampling/sampling_logp_difference/mean": 0.18228664994239807,
      "step": 191,
      "step_time": 13.49295739196532
    },
    {
      "clip_ratio/high_max": 0.00957436254248023,
      "clip_ratio/high_mean": 0.005763743887655437,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005763743887655437,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 593.0,
      "completions/max_terminated_length": 587.0,
      "completions/mean_length": 277.78125,
      "completions/mean_terminated_length": 263.0400085449219,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.7009951621294022,
      "epoch": 0.00384,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9968698024749756,
      "kl": 0.25456699915230274,
      "learning_rate": 9.41403851423797e-06,
      "loss": 0.151,
      "num_tokens": 7670088.0,
      "reward": 0.6887499690055847,
      "reward_std": 0.5566283464431763,
      "rewards/rollout_reward_func/mean": 0.6887499690055847,
      "rewards/rollout_reward_func/std": 0.5942886471748352,
      "sampling/importance_sampling_ratio/max": 1.683251976966858,
      "sampling/importance_sampling_ratio/mean": 0.857994556427002,
      "sampling/importance_sampling_ratio/min": 0.333316832780838,
      "sampling/sampling_logp_difference/max": 0.7194147109985352,
      "sampling/sampling_logp_difference/mean": 0.06644241511821747,
      "step": 192,
      "step_time": 12.908514172027935
    },
    {
      "clip_ratio/high_max": 0.017711078515276313,
      "clip_ratio/high_mean": 0.008855539257638156,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010693774558603764,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1079.0,
      "completions/max_terminated_length": 1079.0,
      "completions/mean_length": 295.4375,
      "completions/mean_terminated_length": 314.0666809082031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6953090354800224,
      "epoch": 0.00386,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6745814085006714,
      "kl": 0.7507963972166181,
      "learning_rate": 9.406707379266792e-06,
      "loss": 0.0318,
      "num_tokens": 7712114.0,
      "reward": 0.737500011920929,
      "reward_std": 0.42730340361595154,
      "rewards/rollout_reward_func/mean": 0.737500011920929,
      "rewards/rollout_reward_func/std": 0.4973931610584259,
      "sampling/importance_sampling_ratio/max": 1.5105379819869995,
      "sampling/importance_sampling_ratio/mean": 0.9427314400672913,
      "sampling/importance_sampling_ratio/min": 0.4059845209121704,
      "sampling/sampling_logp_difference/max": 1.005298137664795,
      "sampling/sampling_logp_difference/mean": 0.06079141050577164,
      "step": 193,
      "step_time": 13.954165976989316
    },
    {
      "clip_ratio/high_max": 0.009027487598359585,
      "clip_ratio/high_mean": 0.004513743799179792,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004513743799179792,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1279.0,
      "completions/max_terminated_length": 1279.0,
      "completions/mean_length": 254.46875,
      "completions/mean_terminated_length": 240.50001525878906,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.691375482827425,
      "epoch": 0.00388,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3224788904190063,
      "kl": 0.32585809752345085,
      "learning_rate": 9.399334605191332e-06,
      "loss": 0.0446,
      "num_tokens": 7754376.0,
      "reward": 0.3296874761581421,
      "reward_std": 0.6671314239501953,
      "rewards/rollout_reward_func/mean": 0.3296874761581421,
      "rewards/rollout_reward_func/std": 0.7082075476646423,
      "sampling/importance_sampling_ratio/max": 2.119110107421875,
      "sampling/importance_sampling_ratio/mean": 0.7978323698043823,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 20.792531967163086,
      "sampling/sampling_logp_difference/mean": 0.2610938549041748,
      "step": 194,
      "step_time": 14.979509518991108
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 664.0,
      "completions/max_terminated_length": 637.0,
      "completions/mean_length": 231.3125,
      "completions/mean_terminated_length": 201.239990234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6908319871872663,
      "epoch": 0.0039,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3347976207733154,
      "kl": 0.5421167127788067,
      "learning_rate": 9.391920289263475e-06,
      "loss": -0.1364,
      "num_tokens": 7794993.0,
      "reward": 0.49687498807907104,
      "reward_std": 0.7279768586158752,
      "rewards/rollout_reward_func/mean": 0.49687498807907104,
      "rewards/rollout_reward_func/std": 0.7353535294532776,
      "sampling/importance_sampling_ratio/max": 2.168515920639038,
      "sampling/importance_sampling_ratio/mean": 0.9786744713783264,
      "sampling/importance_sampling_ratio/min": 1.4287254825532547e-19,
      "sampling/sampling_logp_difference/max": 23.05946922302246,
      "sampling/sampling_logp_difference/mean": 0.16554531455039978,
      "step": 195,
      "step_time": 13.32538719198783
    },
    {
      "clip_ratio/high_max": 0.026831974741071463,
      "clip_ratio/high_mean": 0.013415987370535731,
      "clip_ratio/low_mean": 0.0075041118543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02092009922489524,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 681.0,
      "completions/max_terminated_length": 681.0,
      "completions/mean_length": 221.09375,
      "completions/mean_terminated_length": 216.2916717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7566158883273602,
      "epoch": 0.00392,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3200328350067139,
      "kl": 1.4754090886563063,
      "learning_rate": 9.384464529283055e-06,
      "loss": -0.166,
      "num_tokens": 7834878.0,
      "reward": 0.518750011920929,
      "reward_std": 0.6934795379638672,
      "rewards/rollout_reward_func/mean": 0.518750011920929,
      "rewards/rollout_reward_func/std": 0.8151379823684692,
      "sampling/importance_sampling_ratio/max": 2.2366716861724854,
      "sampling/importance_sampling_ratio/mean": 0.8078794479370117,
      "sampling/importance_sampling_ratio/min": 2.439410309307277e-07,
      "sampling/sampling_logp_difference/max": 7.000133037567139,
      "sampling/sampling_logp_difference/mean": 0.11073426902294159,
      "step": 196,
      "step_time": 12.476194048984325
    },
    {
      "clip_ratio/high_max": 0.018359375186264515,
      "clip_ratio/high_mean": 0.009179687593132257,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009179687593132257,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 236.6875,
      "completions/mean_terminated_length": 240.13043212890625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7528855912387371,
      "epoch": 0.00394,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.543019413948059,
      "kl": 1.962387507315725,
      "learning_rate": 9.376967423596588e-06,
      "loss": -0.1692,
      "num_tokens": 7874897.0,
      "reward": 0.609375,
      "reward_std": 0.5258362293243408,
      "rewards/rollout_reward_func/mean": 0.609375,
      "rewards/rollout_reward_func/std": 0.6183168292045593,
      "sampling/importance_sampling_ratio/max": 2.6061995029449463,
      "sampling/importance_sampling_ratio/mean": 1.0004078149795532,
      "sampling/importance_sampling_ratio/min": 2.6523756776260666e-16,
      "sampling/sampling_logp_difference/max": 24.451330184936523,
      "sampling/sampling_logp_difference/mean": 0.21087239682674408,
      "step": 197,
      "step_time": 12.915928187037935
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.009650735300965607,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010627297800965607,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 306.34375,
      "completions/mean_terminated_length": 310.0666809082031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6710309702903032,
      "epoch": 0.00396,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4542229175567627,
      "kl": 0.5766040689777583,
      "learning_rate": 9.369429071095963e-06,
      "loss": -0.1182,
      "num_tokens": 7918438.0,
      "reward": 0.46187740564346313,
      "reward_std": 0.7194483280181885,
      "rewards/rollout_reward_func/mean": 0.46187740564346313,
      "rewards/rollout_reward_func/std": 0.7416425347328186,
      "sampling/importance_sampling_ratio/max": 1.8110023736953735,
      "sampling/importance_sampling_ratio/mean": 1.0043054819107056,
      "sampling/importance_sampling_ratio/min": 0.27346864342689514,
      "sampling/sampling_logp_difference/max": 1.2860510349273682,
      "sampling/sampling_logp_difference/mean": 0.06083624064922333,
      "step": 198,
      "step_time": 12.566039520970662
    },
    {
      "clip_ratio/high_max": 0.019097222248092294,
      "clip_ratio/high_mean": 0.009548611124046147,
      "clip_ratio/low_mean": 0.017578125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.027126736124046147,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 620.0,
      "completions/max_terminated_length": 620.0,
      "completions/mean_length": 213.5,
      "completions/mean_terminated_length": 226.6666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7496115881949663,
      "epoch": 0.00398,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5134307146072388,
      "kl": 0.8621772667393088,
      "learning_rate": 9.361849571217149e-06,
      "loss": -0.2492,
      "num_tokens": 7959200.0,
      "reward": 0.5634374618530273,
      "reward_std": 0.67543625831604,
      "rewards/rollout_reward_func/mean": 0.5634374618530273,
      "rewards/rollout_reward_func/std": 0.7788063883781433,
      "sampling/importance_sampling_ratio/max": 2.248391628265381,
      "sampling/importance_sampling_ratio/mean": 1.0704702138900757,
      "sampling/importance_sampling_ratio/min": 0.18550357222557068,
      "sampling/sampling_logp_difference/max": 1.4660024642944336,
      "sampling/sampling_logp_difference/mean": 0.06122664734721184,
      "step": 199,
      "step_time": 11.971078251983272
    },
    {
      "clip_ratio/high_max": 0.029557291883975267,
      "clip_ratio/high_mean": 0.014778645941987634,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014778645941987634,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 609.0,
      "completions/max_terminated_length": 609.0,
      "completions/mean_length": 254.4375,
      "completions/mean_terminated_length": 277.7391357421875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7618863433599472,
      "epoch": 0.004,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1966636180877686,
      "kl": 1.1770632937550545,
      "learning_rate": 9.35422902393887e-06,
      "loss": -0.1967,
      "num_tokens": 8001215.0,
      "reward": 0.6124999523162842,
      "reward_std": 0.7264729738235474,
      "rewards/rollout_reward_func/mean": 0.6124999523162842,
      "rewards/rollout_reward_func/std": 0.7120257616043091,
      "sampling/importance_sampling_ratio/max": 2.1099936962127686,
      "sampling/importance_sampling_ratio/mean": 0.8715355396270752,
      "sampling/importance_sampling_ratio/min": 6.885228467866925e-23,
      "sampling/sampling_logp_difference/max": 19.060516357421875,
      "sampling/sampling_logp_difference/mean": 0.25459349155426025,
      "step": 200,
      "step_time": 12.638067772975774
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 145.75,
      "completions/mean_terminated_length": 133.3478240966797,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7139214891940355,
      "epoch": 0.00402,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.689071774482727,
      "kl": 0.9892824669368565,
      "learning_rate": 9.346567529781298e-06,
      "loss": -0.1697,
      "num_tokens": 8037604.0,
      "reward": 0.5575000047683716,
      "reward_std": 0.6298568248748779,
      "rewards/rollout_reward_func/mean": 0.5575000047683716,
      "rewards/rollout_reward_func/std": 0.7689749598503113,
      "sampling/importance_sampling_ratio/max": 1.8210419416427612,
      "sampling/importance_sampling_ratio/mean": 0.9450453519821167,
      "sampling/importance_sampling_ratio/min": 1.1347020545393605e-10,
      "sampling/sampling_logp_difference/max": 22.312498092651367,
      "sampling/sampling_logp_difference/mean": 0.10635396093130112,
      "step": 201,
      "step_time": 12.534328038003878
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 925.0,
      "completions/max_terminated_length": 925.0,
      "completions/mean_length": 301.3125,
      "completions/mean_terminated_length": 322.67999267578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6926669366657734,
      "epoch": 0.00404,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.341220498085022,
      "kl": 0.6497242217883468,
      "learning_rate": 9.338865189804722e-06,
      "loss": -0.2084,
      "num_tokens": 8079840.0,
      "reward": 0.5903751850128174,
      "reward_std": 0.743992030620575,
      "rewards/rollout_reward_func/mean": 0.5903751850128174,
      "rewards/rollout_reward_func/std": 0.7358496189117432,
      "sampling/importance_sampling_ratio/max": 1.6265555620193481,
      "sampling/importance_sampling_ratio/mean": 0.9799441695213318,
      "sampling/importance_sampling_ratio/min": 0.17945918440818787,
      "sampling/sampling_logp_difference/max": 1.0756826400756836,
      "sampling/sampling_logp_difference/mean": 0.06630246341228485,
      "step": 202,
      "step_time": 13.467606970021734
    },
    {
      "clip_ratio/high_max": 0.007170516299083829,
      "clip_ratio/high_mean": 0.0035852581495419145,
      "clip_ratio/low_mean": 0.0022265624720603228,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005811820621602237,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 231.40625,
      "completions/mean_terminated_length": 233.6666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.71058289706707,
      "epoch": 0.00406,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.479980707168579,
      "kl": 1.435224685817957,
      "learning_rate": 9.33112210560821e-06,
      "loss": -0.1879,
      "num_tokens": 8120506.0,
      "reward": 0.4843749701976776,
      "reward_std": 0.7644779086112976,
      "rewards/rollout_reward_func/mean": 0.4843749701976776,
      "rewards/rollout_reward_func/std": 0.7504103779792786,
      "sampling/importance_sampling_ratio/max": 2.3045692443847656,
      "sampling/importance_sampling_ratio/mean": 0.881895124912262,
      "sampling/importance_sampling_ratio/min": 1.4960388609121156e-22,
      "sampling/sampling_logp_difference/max": 23.567626953125,
      "sampling/sampling_logp_difference/mean": 0.176872119307518,
      "step": 203,
      "step_time": 12.789972255006433
    },
    {
      "clip_ratio/high_max": 0.004166666883975267,
      "clip_ratio/high_mean": 0.0020833334419876337,
      "clip_ratio/low_mean": 0.005822981591336429,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007906315033324063,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 621.0,
      "completions/max_terminated_length": 621.0,
      "completions/mean_length": 202.4375,
      "completions/mean_terminated_length": 184.83334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5134226530790329,
      "epoch": 0.00408,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.370659589767456,
      "kl": 0.5504149431362748,
      "learning_rate": 9.323338379328278e-06,
      "loss": -0.1312,
      "num_tokens": 8160728.0,
      "reward": 0.44038963317871094,
      "reward_std": 0.849259614944458,
      "rewards/rollout_reward_func/mean": 0.44038963317871094,
      "rewards/rollout_reward_func/std": 0.8597383499145508,
      "sampling/importance_sampling_ratio/max": 2.2854676246643066,
      "sampling/importance_sampling_ratio/mean": 1.0562187433242798,
      "sampling/importance_sampling_ratio/min": 0.07916386425495148,
      "sampling/sampling_logp_difference/max": 1.4106597900390625,
      "sampling/sampling_logp_difference/mean": 0.07064651697874069,
      "step": 204,
      "step_time": 12.44848298704892
    },
    {
      "clip_ratio/high_max": 0.002016128972172737,
      "clip_ratio/high_mean": 0.0010080644860863686,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0010080644860863686,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 254.53125,
      "completions/mean_terminated_length": 218.0800018310547,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6200253590941429,
      "epoch": 0.0041,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3393826484680176,
      "kl": 0.4592638313770294,
      "learning_rate": 9.315514113637534e-06,
      "loss": 0.0964,
      "num_tokens": 8202196.0,
      "reward": 0.5971875190734863,
      "reward_std": 0.5818052291870117,
      "rewards/rollout_reward_func/mean": 0.5971875190734863,
      "rewards/rollout_reward_func/std": 0.6257961392402649,
      "sampling/importance_sampling_ratio/max": 2.004577159881592,
      "sampling/importance_sampling_ratio/mean": 0.9691969156265259,
      "sampling/importance_sampling_ratio/min": 1.5750482097035188e-35,
      "sampling/sampling_logp_difference/max": 27.388025283813477,
      "sampling/sampling_logp_difference/mean": 0.2455097883939743,
      "step": 205,
      "step_time": 12.293255188982585
    },
    {
      "clip_ratio/high_max": 0.004807692486792803,
      "clip_ratio/high_mean": 0.0024038462433964014,
      "clip_ratio/low_mean": 0.0013020833721384406,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003705929615534842,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 683.0,
      "completions/mean_length": 224.0,
      "completions/mean_terminated_length": 212.0370330810547,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6859660521149635,
      "epoch": 0.00412,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.647864818572998,
      "kl": 0.5149178486317396,
      "learning_rate": 9.30764941174334e-06,
      "loss": 0.0147,
      "num_tokens": 8241112.0,
      "reward": 0.6792862415313721,
      "reward_std": 0.5780168771743774,
      "rewards/rollout_reward_func/mean": 0.6792862415313721,
      "rewards/rollout_reward_func/std": 0.5927951335906982,
      "sampling/importance_sampling_ratio/max": 1.8557592630386353,
      "sampling/importance_sampling_ratio/mean": 0.9232807159423828,
      "sampling/importance_sampling_ratio/min": 0.23358872532844543,
      "sampling/sampling_logp_difference/max": 1.1538231372833252,
      "sampling/sampling_logp_difference/mean": 0.06519077718257904,
      "step": 206,
      "step_time": 12.56736287100648
    },
    {
      "clip_ratio/high_max": 0.009588068351149559,
      "clip_ratio/high_mean": 0.006632269360125065,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006632269360125065,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 230.71875,
      "completions/mean_terminated_length": 198.59259033203125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6520822569727898,
      "epoch": 0.00414,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2728863954544067,
      "kl": 1.283043802715838,
      "learning_rate": 9.299744377386424e-06,
      "loss": -0.2452,
      "num_tokens": 8281309.0,
      "reward": 0.5925771594047546,
      "reward_std": 0.6609393954277039,
      "rewards/rollout_reward_func/mean": 0.5925771594047546,
      "rewards/rollout_reward_func/std": 0.6855549216270447,
      "sampling/importance_sampling_ratio/max": 1.6838361024856567,
      "sampling/importance_sampling_ratio/mean": 0.8936470746994019,
      "sampling/importance_sampling_ratio/min": 0.14814719557762146,
      "sampling/sampling_logp_difference/max": 1.2419099807739258,
      "sampling/sampling_logp_difference/mean": 0.08170796185731888,
      "step": 207,
      "step_time": 12.642342025981634
    },
    {
      "clip_ratio/high_max": 0.011488970601931214,
      "clip_ratio/high_mean": 0.005744485300965607,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005744485300965607,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 675.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 248.96875,
      "completions/mean_terminated_length": 227.51998901367188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6344429347664118,
      "epoch": 0.00416,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5921928882598877,
      "kl": 0.4199596024118364,
      "learning_rate": 9.291799114839533e-06,
      "loss": -0.0912,
      "num_tokens": 8322122.0,
      "reward": 0.6600404977798462,
      "reward_std": 0.5320625305175781,
      "rewards/rollout_reward_func/mean": 0.6600404977798462,
      "rewards/rollout_reward_func/std": 0.6583181023597717,
      "sampling/importance_sampling_ratio/max": 2.194277048110962,
      "sampling/importance_sampling_ratio/mean": 1.0186549425125122,
      "sampling/importance_sampling_ratio/min": 0.29881006479263306,
      "sampling/sampling_logp_difference/max": 1.0909819602966309,
      "sampling/sampling_logp_difference/mean": 0.06438953429460526,
      "step": 208,
      "step_time": 12.4454219469917
    },
    {
      "clip_ratio/high_max": 0.0062806373462080956,
      "clip_ratio/high_mean": 0.0031403186731040478,
      "clip_ratio/low_mean": 0.0026212993543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005761618027463555,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 631.0,
      "completions/max_terminated_length": 631.0,
      "completions/mean_length": 245.9375,
      "completions/mean_terminated_length": 233.5806427001953,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5939568411558867,
      "epoch": 0.00418,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2543376684188843,
      "kl": 0.8774905521422625,
      "learning_rate": 9.283813728906054e-06,
      "loss": -0.0932,
      "num_tokens": 8364306.0,
      "reward": 0.5340648889541626,
      "reward_std": 0.696588397026062,
      "rewards/rollout_reward_func/mean": 0.5340648889541626,
      "rewards/rollout_reward_func/std": 0.7419484853744507,
      "sampling/importance_sampling_ratio/max": 2.11016583442688,
      "sampling/importance_sampling_ratio/mean": 0.8824775218963623,
      "sampling/importance_sampling_ratio/min": 2.1620875285244594e-19,
      "sampling/sampling_logp_difference/max": 24.82895851135254,
      "sampling/sampling_logp_difference/mean": 0.3342662751674652,
      "step": 209,
      "step_time": 12.99320673206239
    },
    {
      "clip_ratio/high_max": 0.022094253916293383,
      "clip_ratio/high_mean": 0.011047126958146691,
      "clip_ratio/low_mean": 0.0012019231216982007,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012249050079844892,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 623.0,
      "completions/max_terminated_length": 623.0,
      "completions/mean_length": 258.40625,
      "completions/mean_terminated_length": 246.9310302734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5718262530863285,
      "epoch": 0.0042,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0668410062789917,
      "kl": 0.5200041756033897,
      "learning_rate": 9.275788324918624e-06,
      "loss": -0.0898,
      "num_tokens": 8406047.0,
      "reward": 0.5992862582206726,
      "reward_std": 0.6853704452514648,
      "rewards/rollout_reward_func/mean": 0.5992862582206726,
      "rewards/rollout_reward_func/std": 0.6772705912590027,
      "sampling/importance_sampling_ratio/max": 1.9388084411621094,
      "sampling/importance_sampling_ratio/mean": 0.9772545099258423,
      "sampling/importance_sampling_ratio/min": 0.3064407706260681,
      "sampling/sampling_logp_difference/max": 1.1471762657165527,
      "sampling/sampling_logp_difference/mean": 0.06408234685659409,
      "step": 210,
      "step_time": 12.315688013026374
    },
    {
      "clip_ratio/high_max": 0.010974702658131719,
      "clip_ratio/high_mean": 0.005487351329065859,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007440476329065859,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 663.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 233.96875,
      "completions/mean_terminated_length": 223.3928680419922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6649471577256918,
      "epoch": 0.00422,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3285531997680664,
      "kl": 2.3581618079915643,
      "learning_rate": 9.267723008737745e-06,
      "loss": -0.069,
      "num_tokens": 8446497.0,
      "reward": 0.5521875023841858,
      "reward_std": 0.6480547189712524,
      "rewards/rollout_reward_func/mean": 0.5521875023841858,
      "rewards/rollout_reward_func/std": 0.732127845287323,
      "sampling/importance_sampling_ratio/max": 1.9985941648483276,
      "sampling/importance_sampling_ratio/mean": 0.9876006841659546,
      "sampling/importance_sampling_ratio/min": 0.13817660510540009,
      "sampling/sampling_logp_difference/max": 1.8979787826538086,
      "sampling/sampling_logp_difference/mean": 0.06563613563776016,
      "step": 211,
      "step_time": 12.66335095098475
    },
    {
      "clip_ratio/high_max": 0.008984375046566129,
      "clip_ratio/high_mean": 0.004492187523283064,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004492187523283064,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 590.0,
      "completions/max_terminated_length": 590.0,
      "completions/mean_length": 183.03125,
      "completions/mean_terminated_length": 159.11111450195312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6701673343777657,
      "epoch": 0.00424,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.374796986579895,
      "kl": 0.38521094247698784,
      "learning_rate": 9.259617886750395e-06,
      "loss": 0.0007,
      "num_tokens": 8484240.0,
      "reward": 0.7578125,
      "reward_std": 0.5144561529159546,
      "rewards/rollout_reward_func/mean": 0.7578125,
      "rewards/rollout_reward_func/std": 0.5852913856506348,
      "sampling/importance_sampling_ratio/max": 2.034518241882324,
      "sampling/importance_sampling_ratio/mean": 0.9353289604187012,
      "sampling/importance_sampling_ratio/min": 1.6818728213515571e-13,
      "sampling/sampling_logp_difference/max": 20.82469367980957,
      "sampling/sampling_logp_difference/mean": 0.13887982070446014,
      "step": 212,
      "step_time": 12.330998083023587
    },
    {
      "clip_ratio/high_max": 0.005425347248092294,
      "clip_ratio/high_mean": 0.002712673624046147,
      "clip_ratio/low_mean": 0.004696800722740591,
      "clip_ratio/low_min": 0.001953125,
      "clip_ratio/region_mean": 0.0074094743467867374,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 788.0,
      "completions/max_terminated_length": 788.0,
      "completions/mean_length": 191.375,
      "completions/mean_terminated_length": 163.7692413330078,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6213976871222258,
      "epoch": 0.00426,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.298917770385742,
      "kl": 0.2928885635919869,
      "learning_rate": 9.251473065868606e-06,
      "loss": 0.1134,
      "num_tokens": 8521961.0,
      "reward": 0.7837499976158142,
      "reward_std": 0.42961791157722473,
      "rewards/rollout_reward_func/mean": 0.7837499976158142,
      "rewards/rollout_reward_func/std": 0.42491552233695984,
      "sampling/importance_sampling_ratio/max": 2.3691203594207764,
      "sampling/importance_sampling_ratio/mean": 1.1620663404464722,
      "sampling/importance_sampling_ratio/min": 3.4487426248119846e-09,
      "sampling/sampling_logp_difference/max": 5.476222038269043,
      "sampling/sampling_logp_difference/mean": 0.11348774284124374,
      "step": 213,
      "step_time": 12.500003919980372
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.005642361007630825,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013454861007630825,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 636.0,
      "completions/max_terminated_length": 636.0,
      "completions/mean_length": 227.8125,
      "completions/mean_terminated_length": 219.73077392578125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6078287996351719,
      "epoch": 0.00428,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6703612804412842,
      "kl": 0.2942681345157325,
      "learning_rate": 9.243288653528075e-06,
      "loss": -0.201,
      "num_tokens": 8562425.0,
      "reward": 0.854687511920929,
      "reward_std": 0.3736533522605896,
      "rewards/rollout_reward_func/mean": 0.854687511920929,
      "rewards/rollout_reward_func/std": 0.4570309519767761,
      "sampling/importance_sampling_ratio/max": 2.424649953842163,
      "sampling/importance_sampling_ratio/mean": 0.9475359320640564,
      "sampling/importance_sampling_ratio/min": 5.25600674095017e-09,
      "sampling/sampling_logp_difference/max": 15.178810119628906,
      "sampling/sampling_logp_difference/mean": 0.17771804332733154,
      "step": 214,
      "step_time": 12.735809759993572
    },
    {
      "clip_ratio/high_max": 0.016406250186264515,
      "clip_ratio/high_mean": 0.008203125093132257,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008203125093132257,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 667.0,
      "completions/max_terminated_length": 667.0,
      "completions/mean_length": 182.96875,
      "completions/mean_terminated_length": 152.7777862548828,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6420161062851548,
      "epoch": 0.0043,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0392141342163086,
      "kl": 0.48053217260167,
      "learning_rate": 9.235064757686733e-06,
      "loss": 0.1182,
      "num_tokens": 8599559.0,
      "reward": 0.5581250190734863,
      "reward_std": 0.6161007881164551,
      "rewards/rollout_reward_func/mean": 0.5581250190734863,
      "rewards/rollout_reward_func/std": 0.6685440540313721,
      "sampling/importance_sampling_ratio/max": 1.9536081552505493,
      "sampling/importance_sampling_ratio/mean": 1.0203206539154053,
      "sampling/importance_sampling_ratio/min": 1.1266855182157087e-07,
      "sampling/sampling_logp_difference/max": 7.268720626831055,
      "sampling/sampling_logp_difference/mean": 0.09950818121433258,
      "step": 215,
      "step_time": 12.250756717985496
    },
    {
      "clip_ratio/high_max": 0.019670516485348344,
      "clip_ratio/high_mean": 0.009835258242674172,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009835258242674172,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 648.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 276.59375,
      "completions/mean_terminated_length": 249.00001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.562513068318367,
      "epoch": 0.00432,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8110514879226685,
      "kl": 0.37290279334411025,
      "learning_rate": 9.226801486823325e-06,
      "loss": -0.0782,
      "num_tokens": 8642005.0,
      "reward": 0.4871875047683716,
      "reward_std": 0.7001311182975769,
      "rewards/rollout_reward_func/mean": 0.4871875047683716,
      "rewards/rollout_reward_func/std": 0.7722244262695312,
      "sampling/importance_sampling_ratio/max": 1.8349978923797607,
      "sampling/importance_sampling_ratio/mean": 0.9941494464874268,
      "sampling/importance_sampling_ratio/min": 0.30883461236953735,
      "sampling/sampling_logp_difference/max": 1.0899786949157715,
      "sampling/sampling_logp_difference/mean": 0.06016572564840317,
      "step": 216,
      "step_time": 12.272079225964262
    },
    {
      "clip_ratio/high_max": 0.0026041667442768812,
      "clip_ratio/high_mean": 0.0013020833721384406,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0013020833721384406,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 677.0,
      "completions/max_terminated_length": 677.0,
      "completions/mean_length": 227.46875,
      "completions/mean_terminated_length": 245.8148193359375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6026113051921129,
      "epoch": 0.00434,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.1212971210479736,
      "kl": 0.2578150201588869,
      "learning_rate": 9.21849894993598e-06,
      "loss": -0.124,
      "num_tokens": 8681879.0,
      "reward": 0.7018749713897705,
      "reward_std": 0.5917739868164062,
      "rewards/rollout_reward_func/mean": 0.7018749713897705,
      "rewards/rollout_reward_func/std": 0.606823742389679,
      "sampling/importance_sampling_ratio/max": 2.1814019680023193,
      "sampling/importance_sampling_ratio/mean": 1.0980923175811768,
      "sampling/importance_sampling_ratio/min": 1.0619458441289995e-18,
      "sampling/sampling_logp_difference/max": 23.316162109375,
      "sampling/sampling_logp_difference/mean": 0.13243895769119263,
      "step": 217,
      "step_time": 12.611400474968832
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.008928571827709675,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008928571827709675,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1409.0,
      "completions/max_terminated_length": 1409.0,
      "completions/mean_length": 324.5625,
      "completions/mean_terminated_length": 326.1071472167969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6421706266701221,
      "epoch": 0.00436,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.996468186378479,
      "kl": 0.46418621204793453,
      "learning_rate": 9.21015725654077e-06,
      "loss": -0.2535,
      "num_tokens": 8726071.0,
      "reward": 0.5586612224578857,
      "reward_std": 0.7045109272003174,
      "rewards/rollout_reward_func/mean": 0.5586612224578857,
      "rewards/rollout_reward_func/std": 0.6858028173446655,
      "sampling/importance_sampling_ratio/max": 2.1262664794921875,
      "sampling/importance_sampling_ratio/mean": 0.8731014728546143,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 27.630409240722656,
      "sampling/sampling_logp_difference/mean": 0.3056909441947937,
      "step": 218,
      "step_time": 15.959710938972421
    },
    {
      "clip_ratio/high_max": 0.0457771650981158,
      "clip_ratio/high_mean": 0.0228885825490579,
      "clip_ratio/low_mean": 0.0055509868543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02843956940341741,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 237.625,
      "completions/mean_terminated_length": 240.9310302734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5308186002075672,
      "epoch": 0.00438,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5996767282485962,
      "kl": 0.374598381575197,
      "learning_rate": 9.201776516670274e-06,
      "loss": -0.2045,
      "num_tokens": 8767597.0,
      "reward": 0.4871875047683716,
      "reward_std": 0.6404141187667847,
      "rewards/rollout_reward_func/mean": 0.4871875047683716,
      "rewards/rollout_reward_func/std": 0.767565131187439,
      "sampling/importance_sampling_ratio/max": 1.749462366104126,
      "sampling/importance_sampling_ratio/mean": 1.137810230255127,
      "sampling/importance_sampling_ratio/min": 0.5700011849403381,
      "sampling/sampling_logp_difference/max": 0.8730850219726562,
      "sampling/sampling_logp_difference/mean": 0.05618341267108917,
      "step": 219,
      "step_time": 12.185516738027218
    },
    {
      "clip_ratio/high_max": 0.01605569780804217,
      "clip_ratio/high_mean": 0.008027848904021084,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008027848904021084,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 612.0,
      "completions/max_terminated_length": 612.0,
      "completions/mean_length": 215.84375,
      "completions/mean_terminated_length": 194.1724090576172,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5427979826927185,
      "epoch": 0.0044,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3245617151260376,
      "kl": 0.4378817919641733,
      "learning_rate": 9.193356840872111e-06,
      "loss": -0.0821,
      "num_tokens": 8807273.0,
      "reward": 0.6965624690055847,
      "reward_std": 0.5718454122543335,
      "rewards/rollout_reward_func/mean": 0.6965624690055847,
      "rewards/rollout_reward_func/std": 0.5943204760551453,
      "sampling/importance_sampling_ratio/max": 1.7221652269363403,
      "sampling/importance_sampling_ratio/mean": 0.9819844961166382,
      "sampling/importance_sampling_ratio/min": 0.065615713596344,
      "sampling/sampling_logp_difference/max": 1.8112425804138184,
      "sampling/sampling_logp_difference/mean": 0.0661323070526123,
      "step": 220,
      "step_time": 12.189933224930428
    },
    {
      "clip_ratio/high_max": 0.006727430620230734,
      "clip_ratio/high_mean": 0.003363715310115367,
      "clip_ratio/low_mean": 0.0166015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019965277810115367,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 864.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 302.59375,
      "completions/mean_terminated_length": 266.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5264098178595304,
      "epoch": 0.00442,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1696341037750244,
      "kl": 1.0118384696543217,
      "learning_rate": 9.184898340207502e-06,
      "loss": -0.225,
      "num_tokens": 8852026.0,
      "reward": 0.38441547751426697,
      "reward_std": 0.7865285277366638,
      "rewards/rollout_reward_func/mean": 0.38441547751426697,
      "rewards/rollout_reward_func/std": 0.8055816888809204,
      "sampling/importance_sampling_ratio/max": 2.396109104156494,
      "sampling/importance_sampling_ratio/mean": 0.9042141437530518,
      "sampling/importance_sampling_ratio/min": 1.6727430312015557e-16,
      "sampling/sampling_logp_difference/max": 18.875,
      "sampling/sampling_logp_difference/mean": 0.18899013102054596,
      "step": 221,
      "step_time": 13.350030180998147
    },
    {
      "clip_ratio/high_max": 0.010026041883975267,
      "clip_ratio/high_mean": 0.005013020941987634,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020638020941987634,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 612.0,
      "completions/max_terminated_length": 612.0,
      "completions/mean_length": 211.0625,
      "completions/mean_terminated_length": 225.51852416992188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6254293043166399,
      "epoch": 0.00444,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9070920944213867,
      "kl": 3.5525274868123233,
      "learning_rate": 9.176401126249785e-06,
      "loss": -0.1063,
      "num_tokens": 8890729.0,
      "reward": 0.6934374570846558,
      "reward_std": 0.5689823031425476,
      "rewards/rollout_reward_func/mean": 0.6934374570846558,
      "rewards/rollout_reward_func/std": 0.6060587167739868,
      "sampling/importance_sampling_ratio/max": 2.6897897720336914,
      "sampling/importance_sampling_ratio/mean": 1.0426833629608154,
      "sampling/importance_sampling_ratio/min": 1.2834652257610066e-18,
      "sampling/sampling_logp_difference/max": 13.522027969360352,
      "sampling/sampling_logp_difference/mean": 0.15187394618988037,
      "step": 222,
      "step_time": 12.010693553034798
    },
    {
      "clip_ratio/high_max": 0.0057043652050197124,
      "clip_ratio/high_mean": 0.0028521826025098562,
      "clip_ratio/low_mean": 0.0014880952658131719,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004340277868323028,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 789.0,
      "completions/max_terminated_length": 789.0,
      "completions/mean_length": 300.65625,
      "completions/mean_terminated_length": 290.9310302734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.567296925932169,
      "epoch": 0.00446,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6276799440383911,
      "kl": 0.4342463109642267,
      "learning_rate": 9.167865311082957e-06,
      "loss": 0.025,
      "num_tokens": 8933544.0,
      "reward": 0.657250165939331,
      "reward_std": 0.5556488037109375,
      "rewards/rollout_reward_func/mean": 0.657250165939331,
      "rewards/rollout_reward_func/std": 0.5498147010803223,
      "sampling/importance_sampling_ratio/max": 1.5627105236053467,
      "sampling/importance_sampling_ratio/mean": 0.985258936882019,
      "sampling/importance_sampling_ratio/min": 0.21500273048877716,
      "sampling/sampling_logp_difference/max": 1.0843048095703125,
      "sampling/sampling_logp_difference/mean": 0.06714978069067001,
      "step": 223,
      "step_time": 13.18400781499804
    },
    {
      "clip_ratio/high_max": 0.012500000186264515,
      "clip_ratio/high_mean": 0.0062500000931322575,
      "clip_ratio/low_mean": 0.0013020833721384406,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007552083465270698,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 585.0,
      "completions/max_terminated_length": 562.0,
      "completions/mean_length": 207.21875,
      "completions/mean_terminated_length": 200.8076934814453,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5419663386419415,
      "epoch": 0.00448,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3792370557785034,
      "kl": 0.4190184799954295,
      "learning_rate": 9.159291007300193e-06,
      "loss": -0.0657,
      "num_tokens": 8972753.0,
      "reward": 0.8315625190734863,
      "reward_std": 0.3395625352859497,
      "rewards/rollout_reward_func/mean": 0.8315625190734863,
      "rewards/rollout_reward_func/std": 0.39650171995162964,
      "sampling/importance_sampling_ratio/max": 2.120621681213379,
      "sampling/importance_sampling_ratio/mean": 0.9974514842033386,
      "sampling/importance_sampling_ratio/min": 0.3673342168331146,
      "sampling/sampling_logp_difference/max": 1.2140388488769531,
      "sampling/sampling_logp_difference/mean": 0.05141027644276619,
      "step": 224,
      "step_time": 12.318835829006275
    },
    {
      "clip_ratio/high_max": 0.010416666977107525,
      "clip_ratio/high_mean": 0.0052083334885537624,
      "clip_ratio/low_mean": 0.0020833334419876337,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007291666930541396,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 659.0,
      "completions/max_terminated_length": 659.0,
      "completions/mean_length": 205.59375,
      "completions/mean_terminated_length": 207.25001525878906,
      "completions/min_length": 1.0,
      "completions/min_terminated_length": 1.0,
      "entropy": 0.5675272289663553,
      "epoch": 0.0045,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2835760116577148,
      "kl": 0.7856590216979384,
      "learning_rate": 9.150678328002352e-06,
      "loss": -0.2545,
      "num_tokens": 9010467.0,
      "reward": 0.7940624952316284,
      "reward_std": 0.4024580121040344,
      "rewards/rollout_reward_func/mean": 0.7940624952316284,
      "rewards/rollout_reward_func/std": 0.5638625621795654,
      "sampling/importance_sampling_ratio/max": 2.0559513568878174,
      "sampling/importance_sampling_ratio/mean": 0.9404018521308899,
      "sampling/importance_sampling_ratio/min": 8.238298931928512e-10,
      "sampling/sampling_logp_difference/max": 20.917057037353516,
      "sampling/sampling_logp_difference/mean": 0.17014965415000916,
      "step": 225,
      "step_time": 12.312987867961056
    },
    {
      "clip_ratio/high_max": 0.009027487598359585,
      "clip_ratio/high_mean": 0.004513743799179792,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004513743799179792,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 601.0,
      "completions/mean_length": 257.28125,
      "completions/mean_terminated_length": 260.8214416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.55787668004632,
      "epoch": 0.00452,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0248669385910034,
      "kl": 0.934052468277514,
      "learning_rate": 9.1420273867965e-06,
      "loss": -0.1669,
      "num_tokens": 9052044.0,
      "reward": 0.6322280168533325,
      "reward_std": 0.710779070854187,
      "rewards/rollout_reward_func/mean": 0.6322280168533325,
      "rewards/rollout_reward_func/std": 0.7419640421867371,
      "sampling/importance_sampling_ratio/max": 2.1206467151641846,
      "sampling/importance_sampling_ratio/mean": 0.9478840231895447,
      "sampling/importance_sampling_ratio/min": 0.0835275948047638,
      "sampling/sampling_logp_difference/max": 1.4586052894592285,
      "sampling/sampling_logp_difference/mean": 0.07069359719753265,
      "step": 226,
      "step_time": 11.973949007951887
    },
    {
      "clip_ratio/high_max": 0.025069444673135877,
      "clip_ratio/high_mean": 0.012534722336567938,
      "clip_ratio/low_mean": 0.00493923609610647,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01747395854908973,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 635.0,
      "completions/max_terminated_length": 533.0,
      "completions/mean_length": 186.65625,
      "completions/mean_terminated_length": 167.90000915527344,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6537342593073845,
      "epoch": 0.00454,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2690731287002563,
      "kl": 0.32011164259165525,
      "learning_rate": 9.133338297794396e-06,
      "loss": -0.2146,
      "num_tokens": 9091129.0,
      "reward": 0.5118749737739563,
      "reward_std": 0.6341406106948853,
      "rewards/rollout_reward_func/mean": 0.5118749737739563,
      "rewards/rollout_reward_func/std": 0.6823083758354187,
      "sampling/importance_sampling_ratio/max": 1.6266982555389404,
      "sampling/importance_sampling_ratio/mean": 0.9005835056304932,
      "sampling/importance_sampling_ratio/min": 7.342837140189226e-14,
      "sampling/sampling_logp_difference/max": 21.235322952270508,
      "sampling/sampling_logp_difference/mean": 0.13871724903583527,
      "step": 227,
      "step_time": 12.635913769976469
    },
    {
      "clip_ratio/high_max": 0.005859375,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0029296875,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 654.0,
      "completions/max_terminated_length": 654.0,
      "completions/mean_length": 180.96875,
      "completions/mean_terminated_length": 168.6333465576172,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5552345216274261,
      "epoch": 0.00456,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9901663064956665,
      "kl": 1.0956237092614174,
      "learning_rate": 9.124611175610998e-06,
      "loss": -0.0924,
      "num_tokens": 9129306.0,
      "reward": 0.5159374475479126,
      "reward_std": 0.7227333188056946,
      "rewards/rollout_reward_func/mean": 0.5159374475479126,
      "rewards/rollout_reward_func/std": 0.7616698741912842,
      "sampling/importance_sampling_ratio/max": 2.222350835800171,
      "sampling/importance_sampling_ratio/mean": 1.0023398399353027,
      "sampling/importance_sampling_ratio/min": 0.2678067982196808,
      "sampling/sampling_logp_difference/max": 1.2902240753173828,
      "sampling/sampling_logp_difference/mean": 0.07331384718418121,
      "step": 228,
      "step_time": 12.142987796032685
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 548.0,
      "completions/mean_length": 235.03125,
      "completions/mean_terminated_length": 209.8148193359375,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.638156071305275,
      "epoch": 0.00458,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9192842245101929,
      "kl": 0.5457145627588034,
      "learning_rate": 9.11584613536295e-06,
      "loss": -0.2541,
      "num_tokens": 9169553.0,
      "reward": 0.6284779906272888,
      "reward_std": 0.47350433468818665,
      "rewards/rollout_reward_func/mean": 0.6284779906272888,
      "rewards/rollout_reward_func/std": 0.7348927855491638,
      "sampling/importance_sampling_ratio/max": 2.846877336502075,
      "sampling/importance_sampling_ratio/mean": 0.9310541749000549,
      "sampling/importance_sampling_ratio/min": 1.2237533209003004e-09,
      "sampling/sampling_logp_difference/max": 18.150867462158203,
      "sampling/sampling_logp_difference/mean": 0.15971501171588898,
      "step": 229,
      "step_time": 12.25962684499973
    },
    {
      "clip_ratio/high_max": 0.0516020271461457,
      "clip_ratio/high_mean": 0.02580101357307285,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02580101357307285,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 206.71875,
      "completions/mean_terminated_length": 204.58621215820312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5148060526698828,
      "epoch": 0.0046,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3529607057571411,
      "kl": 0.5790175963193178,
      "learning_rate": 9.107043292667056e-06,
      "loss": -0.0758,
      "num_tokens": 9208883.0,
      "reward": 0.8228529691696167,
      "reward_std": 0.48110172152519226,
      "rewards/rollout_reward_func/mean": 0.8228529691696167,
      "rewards/rollout_reward_func/std": 0.48840707540512085,
      "sampling/importance_sampling_ratio/max": 2.008890390396118,
      "sampling/importance_sampling_ratio/mean": 0.9528816938400269,
      "sampling/importance_sampling_ratio/min": 1.49951508238766e-19,
      "sampling/sampling_logp_difference/max": 21.083524703979492,
      "sampling/sampling_logp_difference/mean": 0.15625900030136108,
      "step": 230,
      "step_time": 12.263589166032034
    },
    {
      "clip_ratio/high_max": 0.02183159743435681,
      "clip_ratio/high_mean": 0.017165798810310662,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017165798810310662,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 603.0,
      "completions/mean_length": 181.21875,
      "completions/mean_terminated_length": 121.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5932736517861485,
      "epoch": 0.00462,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1088746786117554,
      "kl": 0.4526619641110301,
      "learning_rate": 9.098202763638763e-06,
      "loss": -0.104,
      "num_tokens": 9245928.0,
      "reward": 0.6556249856948853,
      "reward_std": 0.6350587010383606,
      "rewards/rollout_reward_func/mean": 0.6556249856948853,
      "rewards/rollout_reward_func/std": 0.7531393766403198,
      "sampling/importance_sampling_ratio/max": 1.7097012996673584,
      "sampling/importance_sampling_ratio/mean": 0.8087704181671143,
      "sampling/importance_sampling_ratio/min": 2.207147003321155e-18,
      "sampling/sampling_logp_difference/max": 23.915006637573242,
      "sampling/sampling_logp_difference/mean": 0.16882669925689697,
      "step": 231,
      "step_time": 12.450157222978305
    },
    {
      "clip_ratio/high_max": 0.003289473708719015,
      "clip_ratio/high_mean": 0.0016447368543595076,
      "clip_ratio/low_mean": 0.0013586956774815917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0030034325318410993,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 641.0,
      "completions/mean_length": 278.625,
      "completions/mean_terminated_length": 256.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.514517679810524,
      "epoch": 0.00464,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0401017665863037,
      "kl": 1.0989286839030683,
      "learning_rate": 9.089324664890625e-06,
      "loss": -0.1291,
      "num_tokens": 9288617.0,
      "reward": 0.45303866267204285,
      "reward_std": 0.675103485584259,
      "rewards/rollout_reward_func/mean": 0.45303866267204285,
      "rewards/rollout_reward_func/std": 0.7068684697151184,
      "sampling/importance_sampling_ratio/max": 2.2170166969299316,
      "sampling/importance_sampling_ratio/mean": 1.0120124816894531,
      "sampling/importance_sampling_ratio/min": 4.88054792466987e-17,
      "sampling/sampling_logp_difference/max": 23.929853439331055,
      "sampling/sampling_logp_difference/mean": 0.13970279693603516,
      "step": 232,
      "step_time": 12.672101854972425
    },
    {
      "clip_ratio/high_max": 0.0031250000465661287,
      "clip_ratio/high_mean": 0.0015625000232830644,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015625000232830644,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 864.0,
      "completions/max_terminated_length": 864.0,
      "completions/mean_length": 246.0,
      "completions/mean_terminated_length": 261.3333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49904471077024937,
      "epoch": 0.00466,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3100522756576538,
      "kl": 1.2493150299414992,
      "learning_rate": 9.080409113530767e-06,
      "loss": -0.1897,
      "num_tokens": 9329599.0,
      "reward": 0.43687736988067627,
      "reward_std": 0.7783619165420532,
      "rewards/rollout_reward_func/mean": 0.43687736988067627,
      "rewards/rollout_reward_func/std": 0.8356614112854004,
      "sampling/importance_sampling_ratio/max": 1.8836472034454346,
      "sampling/importance_sampling_ratio/mean": 0.9324465394020081,
      "sampling/importance_sampling_ratio/min": 0.10795243084430695,
      "sampling/sampling_logp_difference/max": 1.413036584854126,
      "sampling/sampling_logp_difference/mean": 0.06860889494419098,
      "step": 233,
      "step_time": 12.638017312972806
    },
    {
      "clip_ratio/high_max": 0.022656250279396772,
      "clip_ratio/high_mean": 0.011328125139698386,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011328125139698386,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 602.0,
      "completions/max_terminated_length": 602.0,
      "completions/mean_length": 221.40625,
      "completions/mean_terminated_length": 211.95652770996094,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5778834410011768,
      "epoch": 0.00468,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7576336860656738,
      "kl": 0.3519200775772333,
      "learning_rate": 9.071456227161335e-06,
      "loss": 0.0193,
      "num_tokens": 9369292.0,
      "reward": 0.7875404953956604,
      "reward_std": 0.4328577518463135,
      "rewards/rollout_reward_func/mean": 0.7875404953956604,
      "rewards/rollout_reward_func/std": 0.586717963218689,
      "sampling/importance_sampling_ratio/max": 2.58561110496521,
      "sampling/importance_sampling_ratio/mean": 0.8571667671203613,
      "sampling/importance_sampling_ratio/min": 1.6177653847338584e-28,
      "sampling/sampling_logp_difference/max": 26.901790618896484,
      "sampling/sampling_logp_difference/mean": 0.4520688056945801,
      "step": 234,
      "step_time": 12.647470426978543
    },
    {
      "clip_ratio/high_max": 0.0061011905781924725,
      "clip_ratio/high_mean": 0.0030505952890962362,
      "clip_ratio/low_mean": 0.00704656878951937,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010097163962200284,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 694.0,
      "completions/max_terminated_length": 694.0,
      "completions/mean_length": 252.5625,
      "completions/mean_terminated_length": 256.7857360839844,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.47048557363450527,
      "epoch": 0.0047,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.059199810028076,
      "kl": 3.0728377932682633,
      "learning_rate": 9.062466123876954e-06,
      "loss": -0.0155,
      "num_tokens": 9411142.0,
      "reward": 0.6431273818016052,
      "reward_std": 0.6474026441574097,
      "rewards/rollout_reward_func/mean": 0.6431273818016052,
      "rewards/rollout_reward_func/std": 0.6310160160064697,
      "sampling/importance_sampling_ratio/max": 2.878406286239624,
      "sampling/importance_sampling_ratio/mean": 1.0596339702606201,
      "sampling/importance_sampling_ratio/min": 1.1109439523370635e-21,
      "sampling/sampling_logp_difference/max": 22.87687110900879,
      "sampling/sampling_logp_difference/mean": 0.15200932323932648,
      "step": 235,
      "step_time": 12.968184354976984
    },
    {
      "clip_ratio/high_max": 0.018973214784637094,
      "clip_ratio/high_mean": 0.009486607392318547,
      "clip_ratio/low_mean": 0.007589285960420966,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017075893352739513,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 908.0,
      "completions/max_terminated_length": 603.0,
      "completions/mean_length": 227.21875,
      "completions/mean_terminated_length": 200.30435180664062,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4736304646357894,
      "epoch": 0.00472,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5354945659637451,
      "kl": 0.6210969481617212,
      "learning_rate": 9.053438922263165e-06,
      "loss": -0.1994,
      "num_tokens": 9452200.0,
      "reward": 0.4849112629890442,
      "reward_std": 0.8804377913475037,
      "rewards/rollout_reward_func/mean": 0.4849112629890442,
      "rewards/rollout_reward_func/std": 0.8652898073196411,
      "sampling/importance_sampling_ratio/max": 1.9575339555740356,
      "sampling/importance_sampling_ratio/mean": 0.8994759321212769,
      "sampling/importance_sampling_ratio/min": 0.20897142589092255,
      "sampling/sampling_logp_difference/max": 1.0903387069702148,
      "sampling/sampling_logp_difference/mean": 0.0645613893866539,
      "step": 236,
      "step_time": 13.256981692960835
    },
    {
      "clip_ratio/high_max": 0.016975976061075926,
      "clip_ratio/high_mean": 0.008487988030537963,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008487988030537963,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 661.0,
      "completions/max_terminated_length": 661.0,
      "completions/mean_length": 299.21875,
      "completions/mean_terminated_length": 279.8214416503906,
      "completions/min_length": 1.0,
      "completions/min_terminated_length": 1.0,
      "entropy": 0.5434614224359393,
      "epoch": 0.00474,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9048882722854614,
      "kl": 0.6899931253865361,
      "learning_rate": 9.044374741394859e-06,
      "loss": 0.2124,
      "num_tokens": 9494369.0,
      "reward": 0.7590625286102295,
      "reward_std": 0.4335324466228485,
      "rewards/rollout_reward_func/mean": 0.7590625286102295,
      "rewards/rollout_reward_func/std": 0.4262995719909668,
      "sampling/importance_sampling_ratio/max": 2.141603946685791,
      "sampling/importance_sampling_ratio/mean": 0.8735755681991577,
      "sampling/importance_sampling_ratio/min": 3.9391934674023674e-14,
      "sampling/sampling_logp_difference/max": 22.264251708984375,
      "sampling/sampling_logp_difference/mean": 0.21212875843048096,
      "step": 237,
      "step_time": 12.607058708992554
    },
    {
      "clip_ratio/high_max": 0.009765625,
      "clip_ratio/high_mean": 0.007486979477107525,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008463541977107525,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 833.0,
      "completions/max_terminated_length": 833.0,
      "completions/mean_length": 212.1875,
      "completions/mean_terminated_length": 167.58334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4837746676057577,
      "epoch": 0.00476,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6982638835906982,
      "kl": 0.8184778862632811,
      "learning_rate": 9.035273700834707e-06,
      "loss": -0.121,
      "num_tokens": 9532017.0,
      "reward": 0.6844376921653748,
      "reward_std": 0.46037203073501587,
      "rewards/rollout_reward_func/mean": 0.6844376921653748,
      "rewards/rollout_reward_func/std": 0.5967605113983154,
      "sampling/importance_sampling_ratio/max": 2.0748372077941895,
      "sampling/importance_sampling_ratio/mean": 0.9035058617591858,
      "sampling/importance_sampling_ratio/min": 0.08952101320028305,
      "sampling/sampling_logp_difference/max": 1.4996509552001953,
      "sampling/sampling_logp_difference/mean": 0.06739787757396698,
      "step": 238,
      "step_time": 13.430842643923825
    },
    {
      "clip_ratio/high_max": 0.016406250186264515,
      "clip_ratio/high_mean": 0.008203125093132257,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008203125093132257,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 1067.0,
      "completions/max_terminated_length": 1067.0,
      "completions/mean_length": 206.125,
      "completions/mean_terminated_length": 210.55172729492188,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.47768070735037327,
      "epoch": 0.00478,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.3239450454711914,
      "kl": 0.48567298008129,
      "learning_rate": 9.026135920631592e-06,
      "loss": 0.0996,
      "num_tokens": 9571074.0,
      "reward": 0.6021875143051147,
      "reward_std": 0.5909303426742554,
      "rewards/rollout_reward_func/mean": 0.6021875143051147,
      "rewards/rollout_reward_func/std": 0.6143117547035217,
      "sampling/importance_sampling_ratio/max": 2.3153738975524902,
      "sampling/importance_sampling_ratio/mean": 1.0691850185394287,
      "sampling/importance_sampling_ratio/min": 0.3224697411060333,
      "sampling/sampling_logp_difference/max": 1.019662857055664,
      "sampling/sampling_logp_difference/mean": 0.05736405402421951,
      "step": 239,
      "step_time": 13.310935819987208
    },
    {
      "clip_ratio/high_max": 0.0020833334419876337,
      "clip_ratio/high_mean": 0.0010416667209938169,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002994791720993817,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 642.0,
      "completions/max_terminated_length": 642.0,
      "completions/mean_length": 234.0,
      "completions/mean_terminated_length": 248.7857208251953,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5084124226123095,
      "epoch": 0.0048,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0526957511901855,
      "kl": 0.3209646735340357,
      "learning_rate": 9.016961521319006e-06,
      "loss": 0.0442,
      "num_tokens": 9610580.0,
      "reward": 0.8003124594688416,
      "reward_std": 0.49250590801239014,
      "rewards/rollout_reward_func/mean": 0.8003124594688416,
      "rewards/rollout_reward_func/std": 0.5648778080940247,
      "sampling/importance_sampling_ratio/max": 2.646749496459961,
      "sampling/importance_sampling_ratio/mean": 1.0988585948944092,
      "sampling/importance_sampling_ratio/min": 0.311273992061615,
      "sampling/sampling_logp_difference/max": 0.8150291442871094,
      "sampling/sampling_logp_difference/mean": 0.052210137248039246,
      "step": 240,
      "step_time": 12.713659904984524
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004464285913854837,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 1429.0,
      "completions/max_terminated_length": 1429.0,
      "completions/mean_length": 213.09375,
      "completions/mean_terminated_length": 214.51724243164062,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.48869029991328716,
      "epoch": 0.00482,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9064422845840454,
      "kl": 1.1255414932966232,
      "learning_rate": 9.00775062391348e-06,
      "loss": -0.0684,
      "num_tokens": 9649500.0,
      "reward": 0.6903529763221741,
      "reward_std": 0.5285418629646301,
      "rewards/rollout_reward_func/mean": 0.6903529763221741,
      "rewards/rollout_reward_func/std": 0.5449788570404053,
      "sampling/importance_sampling_ratio/max": 2.831763744354248,
      "sampling/importance_sampling_ratio/mean": 0.8919353485107422,
      "sampling/importance_sampling_ratio/min": 4.7615906526724574e-29,
      "sampling/sampling_logp_difference/max": 28.221912384033203,
      "sampling/sampling_logp_difference/mean": 0.4001326262950897,
      "step": 241,
      "step_time": 15.649598211006378
    },
    {
      "clip_ratio/high_max": 0.005425347248092294,
      "clip_ratio/high_mean": 0.002712673624046147,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003689236124046147,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 605.0,
      "completions/max_terminated_length": 605.0,
      "completions/mean_length": 223.6875,
      "completions/mean_terminated_length": 211.1538543701172,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5448360778391361,
      "epoch": 0.00484,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9326989650726318,
      "kl": 0.4039561850950122,
      "learning_rate": 8.998503349912977e-06,
      "loss": -0.1401,
      "num_tokens": 9690076.0,
      "reward": 0.7431249618530273,
      "reward_std": 0.4421554207801819,
      "rewards/rollout_reward_func/mean": 0.7431249618530273,
      "rewards/rollout_reward_func/std": 0.5218109488487244,
      "sampling/importance_sampling_ratio/max": 1.963469386100769,
      "sampling/importance_sampling_ratio/mean": 1.031839370727539,
      "sampling/importance_sampling_ratio/min": 1.6386401486545379e-21,
      "sampling/sampling_logp_difference/max": 23.90364646911621,
      "sampling/sampling_logp_difference/mean": 0.19282257556915283,
      "step": 242,
      "step_time": 12.719803333995515
    },
    {
      "clip_ratio/high_max": 0.0020833334419876337,
      "clip_ratio/high_mean": 0.0010416667209938169,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002018229220993817,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 597.0,
      "completions/mean_length": 273.125,
      "completions/mean_terminated_length": 267.7200012207031,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5055758226662874,
      "epoch": 0.00486,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.5965969562530518,
      "kl": 0.3517384179867804,
      "learning_rate": 8.989219821295294e-06,
      "loss": 0.1418,
      "num_tokens": 9732892.0,
      "reward": 0.7699999809265137,
      "reward_std": 0.4365108609199524,
      "rewards/rollout_reward_func/mean": 0.7699999809265137,
      "rewards/rollout_reward_func/std": 0.44405752420425415,
      "sampling/importance_sampling_ratio/max": 2.3721346855163574,
      "sampling/importance_sampling_ratio/mean": 0.9413117170333862,
      "sampling/importance_sampling_ratio/min": 4.1688638675740324e-21,
      "sampling/sampling_logp_difference/max": 23.811443328857422,
      "sampling/sampling_logp_difference/mean": 0.1974773406982422,
      "step": 243,
      "step_time": 12.674957512019319
    },
    {
      "clip_ratio/high_max": 0.006119791883975267,
      "clip_ratio/high_mean": 0.0030598959419876337,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0030598959419876337,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 662.0,
      "completions/max_terminated_length": 622.0,
      "completions/mean_length": 273.75,
      "completions/mean_terminated_length": 267.7200012207031,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5175339877605438,
      "epoch": 0.00488,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.475949287414551,
      "kl": 1.692062597721815,
      "learning_rate": 8.979900160516447e-06,
      "loss": 0.254,
      "num_tokens": 9774173.0,
      "reward": 0.8624999523162842,
      "reward_std": 0.30673182010650635,
      "rewards/rollout_reward_func/mean": 0.8624999523162842,
      "rewards/rollout_reward_func/std": 0.3653985857963562,
      "sampling/importance_sampling_ratio/max": 2.1815402507781982,
      "sampling/importance_sampling_ratio/mean": 1.046360731124878,
      "sampling/importance_sampling_ratio/min": 0.11716803908348083,
      "sampling/sampling_logp_difference/max": 1.9642579555511475,
      "sampling/sampling_logp_difference/mean": 0.05234575644135475,
      "step": 244,
      "step_time": 12.857161211999482
    },
    {
      "clip_ratio/high_max": 0.024215420242398977,
      "clip_ratio/high_mean": 0.013843821594491601,
      "clip_ratio/low_mean": 0.0031250000465661287,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01696882164105773,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1391.0,
      "completions/max_terminated_length": 1391.0,
      "completions/mean_length": 299.75,
      "completions/mean_terminated_length": 308.13336181640625,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5427849963307381,
      "epoch": 0.0049,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8233693242073059,
      "kl": 0.3684608908370137,
      "learning_rate": 8.970544490509064e-06,
      "loss": -0.0453,
      "num_tokens": 9817206.0,
      "reward": 0.7275428771972656,
      "reward_std": 0.6668952703475952,
      "rewards/rollout_reward_func/mean": 0.7275428771972656,
      "rewards/rollout_reward_func/std": 0.647696316242218,
      "sampling/importance_sampling_ratio/max": 1.4304170608520508,
      "sampling/importance_sampling_ratio/mean": 0.8172030448913574,
      "sampling/importance_sampling_ratio/min": 6.510752206871596e-15,
      "sampling/sampling_logp_difference/max": 21.787630081176758,
      "sampling/sampling_logp_difference/mean": 0.26038047671318054,
      "step": 245,
      "step_time": 15.767331483992166
    },
    {
      "clip_ratio/high_max": 0.011294157709926367,
      "clip_ratio/high_mean": 0.005647078854963183,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007600203854963183,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 862.0,
      "completions/max_terminated_length": 862.0,
      "completions/mean_length": 249.90625,
      "completions/mean_terminated_length": 250.83334350585938,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5012013856321573,
      "epoch": 0.00492,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.085538625717163,
      "kl": 0.4271015813574195,
      "learning_rate": 8.961152934680752e-06,
      "loss": -0.2772,
      "num_tokens": 9858677.0,
      "reward": 0.5435001850128174,
      "reward_std": 0.6955839395523071,
      "rewards/rollout_reward_func/mean": 0.5435001850128174,
      "rewards/rollout_reward_func/std": 0.6895714402198792,
      "sampling/importance_sampling_ratio/max": 1.7997865676879883,
      "sampling/importance_sampling_ratio/mean": 1.0371040105819702,
      "sampling/importance_sampling_ratio/min": 0.20560236275196075,
      "sampling/sampling_logp_difference/max": 1.3668644428253174,
      "sampling/sampling_logp_difference/mean": 0.05756765604019165,
      "step": 246,
      "step_time": 13.251813197988668
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 231.34375,
      "completions/mean_terminated_length": 245.92593383789062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5534438695758581,
      "epoch": 0.00494,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9257471561431885,
      "kl": 0.3588506458327174,
      "learning_rate": 8.951725616912489e-06,
      "loss": 0.3388,
      "num_tokens": 9897741.0,
      "reward": 0.6600000262260437,
      "reward_std": 0.4817848205566406,
      "rewards/rollout_reward_func/mean": 0.6600000262260437,
      "rewards/rollout_reward_func/std": 0.46851858496665955,
      "sampling/importance_sampling_ratio/max": 2.8060245513916016,
      "sampling/importance_sampling_ratio/mean": 0.9213325381278992,
      "sampling/importance_sampling_ratio/min": 2.5309009972798293e-19,
      "sampling/sampling_logp_difference/max": 22.942401885986328,
      "sampling/sampling_logp_difference/mean": 0.23978015780448914,
      "step": 247,
      "step_time": 12.115169413998956
    },
    {
      "clip_ratio/high_max": 0.010352334706112742,
      "clip_ratio/high_mean": 0.006152729969471693,
      "clip_ratio/low_mean": 0.017269736854359508,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234224668238312,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 656.0,
      "completions/mean_length": 208.8125,
      "completions/mean_terminated_length": 213.1428680419922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6141808833926916,
      "epoch": 0.00496,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.050034523010254,
      "kl": 0.545457218773663,
      "learning_rate": 8.942262661556962e-06,
      "loss": -0.0947,
      "num_tokens": 9937450.0,
      "reward": 0.8293750286102295,
      "reward_std": 0.40892869234085083,
      "rewards/rollout_reward_func/mean": 0.8293750286102295,
      "rewards/rollout_reward_func/std": 0.4739125669002533,
      "sampling/importance_sampling_ratio/max": 1.8348244428634644,
      "sampling/importance_sampling_ratio/mean": 0.8909742832183838,
      "sampling/importance_sampling_ratio/min": 1.5644990225563222e-25,
      "sampling/sampling_logp_difference/max": 26.441913604736328,
      "sampling/sampling_logp_difference/mean": 0.2952561378479004,
      "step": 248,
      "step_time": 12.748284137967858
    },
    {
      "clip_ratio/high_max": 0.013020833721384406,
      "clip_ratio/high_mean": 0.006510416860692203,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006510416860692203,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 1008.0,
      "completions/max_terminated_length": 594.0,
      "completions/mean_length": 203.90625,
      "completions/mean_terminated_length": 144.70370483398438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6434690356254578,
      "epoch": 0.00498,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.168568730354309,
      "kl": 0.2923278445377946,
      "learning_rate": 8.932764193436953e-06,
      "loss": 0.0184,
      "num_tokens": 9977130.0,
      "reward": 0.6550000309944153,
      "reward_std": 0.5902426242828369,
      "rewards/rollout_reward_func/mean": 0.6550000309944153,
      "rewards/rollout_reward_func/std": 0.6010367274284363,
      "sampling/importance_sampling_ratio/max": 2.0215466022491455,
      "sampling/importance_sampling_ratio/mean": 0.9124063849449158,
      "sampling/importance_sampling_ratio/min": 2.0714815892701246e-22,
      "sampling/sampling_logp_difference/max": 22.65732765197754,
      "sampling/sampling_logp_difference/mean": 0.2670382559299469,
      "step": 249,
      "step_time": 14.419875573003083
    },
    {
      "clip_ratio/high_max": 0.0041082974057644606,
      "clip_ratio/high_mean": 0.0020541487028822303,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00400727370288223,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 678.0,
      "completions/max_terminated_length": 678.0,
      "completions/mean_length": 358.96875,
      "completions/mean_terminated_length": 344.0833435058594,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5291668809950352,
      "epoch": 0.005,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0464811325073242,
      "kl": 0.46616613399237394,
      "learning_rate": 8.923230337843675e-06,
      "loss": -0.025,
      "num_tokens": 10022661.0,
      "reward": 0.5457021594047546,
      "reward_std": 0.536036491394043,
      "rewards/rollout_reward_func/mean": 0.5457021594047546,
      "rewards/rollout_reward_func/std": 0.6334956884384155,
      "sampling/importance_sampling_ratio/max": 2.3452725410461426,
      "sampling/importance_sampling_ratio/mean": 0.9818421602249146,
      "sampling/importance_sampling_ratio/min": 0.3010532259941101,
      "sampling/sampling_logp_difference/max": 1.0155611038208008,
      "sampling/sampling_logp_difference/mean": 0.055734239518642426,
      "step": 250,
      "step_time": 12.661999176998506
    },
    {
      "clip_ratio/high_max": 0.020833333488553762,
      "clip_ratio/high_mean": 0.011393229244276881,
      "clip_ratio/low_mean": 0.0013586956774815917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012751924921758473,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 777.0,
      "completions/max_terminated_length": 777.0,
      "completions/mean_length": 229.5625,
      "completions/mean_terminated_length": 222.1666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6609371341764927,
      "epoch": 0.00502,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5268062353134155,
      "kl": 1.129921443760395,
      "learning_rate": 8.913661220535132e-06,
      "loss": -0.1075,
      "num_tokens": 10062538.0,
      "reward": 0.6815624833106995,
      "reward_std": 0.5337181091308594,
      "rewards/rollout_reward_func/mean": 0.6815624833106995,
      "rewards/rollout_reward_func/std": 0.6681950092315674,
      "sampling/importance_sampling_ratio/max": 2.767707347869873,
      "sampling/importance_sampling_ratio/mean": 1.0333573818206787,
      "sampling/importance_sampling_ratio/min": 6.019419671813504e-22,
      "sampling/sampling_logp_difference/max": 23.59133529663086,
      "sampling/sampling_logp_difference/mean": 0.15626761317253113,
      "step": 251,
      "step_time": 13.15955935101374
    },
    {
      "clip_ratio/high_max": 0.012655772734433413,
      "clip_ratio/high_mean": 0.006327886367216706,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006327886367216706,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 675.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 222.125,
      "completions/mean_terminated_length": 219.70001220703125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5133199840784073,
      "epoch": 0.00504,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.167664051055908,
      "kl": 0.5926091698929667,
      "learning_rate": 8.904056967734447e-06,
      "loss": 0.1163,
      "num_tokens": 10103645.0,
      "reward": 0.45781248807907104,
      "reward_std": 0.704747200012207,
      "rewards/rollout_reward_func/mean": 0.45781248807907104,
      "rewards/rollout_reward_func/std": 0.7726858258247375,
      "sampling/importance_sampling_ratio/max": 2.5563297271728516,
      "sampling/importance_sampling_ratio/mean": 1.0807157754898071,
      "sampling/importance_sampling_ratio/min": 1.0257238388605659e-22,
      "sampling/sampling_logp_difference/max": 22.86585235595703,
      "sampling/sampling_logp_difference/mean": 0.1485586166381836,
      "step": 252,
      "step_time": 12.922824099034187
    },
    {
      "clip_ratio/high_max": 0.004629629664123058,
      "clip_ratio/high_mean": 0.002314814832061529,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006779100745916367,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 613.0,
      "completions/mean_length": 191.21875,
      "completions/mean_terminated_length": 172.83334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5493122581392527,
      "epoch": 0.00506,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0574729442596436,
      "kl": 0.8272303650155663,
      "learning_rate": 8.894417706128209e-06,
      "loss": -0.039,
      "num_tokens": 10141470.0,
      "reward": 0.5907021760940552,
      "reward_std": 0.6562352180480957,
      "rewards/rollout_reward_func/mean": 0.5907021760940552,
      "rewards/rollout_reward_func/std": 0.6854618191719055,
      "sampling/importance_sampling_ratio/max": 1.6704548597335815,
      "sampling/importance_sampling_ratio/mean": 0.8981623649597168,
      "sampling/importance_sampling_ratio/min": 0.23779045045375824,
      "sampling/sampling_logp_difference/max": 1.6960768699645996,
      "sampling/sampling_logp_difference/mean": 0.06276530027389526,
      "step": 253,
      "step_time": 12.054594979024841
    },
    {
      "clip_ratio/high_max": 0.012803819496184587,
      "clip_ratio/high_mean": 0.006401909748092294,
      "clip_ratio/low_mean": 0.011992187239229679,
      "clip_ratio/low_min": 0.0024999999441206455,
      "clip_ratio/region_mean": 0.018394096987321973,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 568.0,
      "completions/mean_length": 206.875,
      "completions/mean_terminated_length": 152.73077392578125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4950271788984537,
      "epoch": 0.00508,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.290506362915039,
      "kl": 1.1006019581109285,
      "learning_rate": 8.884743562864792e-06,
      "loss": -0.0774,
      "num_tokens": 10180003.0,
      "reward": 0.7021899223327637,
      "reward_std": 0.4655950367450714,
      "rewards/rollout_reward_func/mean": 0.7021899223327637,
      "rewards/rollout_reward_func/std": 0.6042341589927673,
      "sampling/importance_sampling_ratio/max": 2.342602014541626,
      "sampling/importance_sampling_ratio/mean": 1.1009737253189087,
      "sampling/importance_sampling_ratio/min": 0.08805643767118454,
      "sampling/sampling_logp_difference/max": 1.4292118549346924,
      "sampling/sampling_logp_difference/mean": 0.06061416491866112,
      "step": 254,
      "step_time": 12.565785899016191
    },
    {
      "clip_ratio/high_max": 0.007905506063252687,
      "clip_ratio/high_mean": 0.003952753031626344,
      "clip_ratio/low_mean": 0.005936880013905466,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00988963304553181,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1015.0,
      "completions/max_terminated_length": 1015.0,
      "completions/mean_length": 228.8125,
      "completions/mean_terminated_length": 244.9091033935547,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5615298375487328,
      "epoch": 0.0051,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3209527730941772,
      "kl": 1.0794194797053933,
      "learning_rate": 8.875034665552686e-06,
      "loss": -0.4027,
      "num_tokens": 10220503.0,
      "reward": 0.3097386360168457,
      "reward_std": 0.9181375503540039,
      "rewards/rollout_reward_func/mean": 0.3097386360168457,
      "rewards/rollout_reward_func/std": 0.8877979516983032,
      "sampling/importance_sampling_ratio/max": 2.245652675628662,
      "sampling/importance_sampling_ratio/mean": 0.9540248513221741,
      "sampling/importance_sampling_ratio/min": 0.12451189756393433,
      "sampling/sampling_logp_difference/max": 1.4617321491241455,
      "sampling/sampling_logp_difference/mean": 0.0746372640132904,
      "step": 255,
      "step_time": 13.746651851994102
    },
    {
      "clip_ratio/high_max": 0.0026041667442768812,
      "clip_ratio/high_mean": 0.0013020833721384406,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0013020833721384406,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 271.75,
      "completions/mean_terminated_length": 260.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5708245784044266,
      "epoch": 0.00512,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8069055676460266,
      "kl": 0.5920950509607792,
      "learning_rate": 8.865291142258813e-06,
      "loss": -0.1689,
      "num_tokens": 10261626.0,
      "reward": 0.4885406792163849,
      "reward_std": 0.8515567183494568,
      "rewards/rollout_reward_func/mean": 0.4885406792163849,
      "rewards/rollout_reward_func/std": 0.8336228132247925,
      "sampling/importance_sampling_ratio/max": 1.624332070350647,
      "sampling/importance_sampling_ratio/mean": 0.8166780471801758,
      "sampling/importance_sampling_ratio/min": 2.885497017105637e-13,
      "sampling/sampling_logp_difference/max": 22.55206298828125,
      "sampling/sampling_logp_difference/mean": 0.1180664598941803,
      "step": 256,
      "step_time": 12.312064158977591
    },
    {
      "clip_ratio/high_max": 0.0028409091755747795,
      "clip_ratio/high_mean": 0.0014204545877873898,
      "clip_ratio/low_mean": 0.0029296875,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00435014208778739,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 811.0,
      "completions/max_terminated_length": 811.0,
      "completions/mean_length": 288.0,
      "completions/mean_terminated_length": 281.4444580078125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5640455558896065,
      "epoch": 0.00514,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6392738819122314,
      "kl": 0.7293183049187064,
      "learning_rate": 8.855513121506826e-06,
      "loss": -0.1203,
      "num_tokens": 10305574.0,
      "reward": 0.453125,
      "reward_std": 0.6815417408943176,
      "rewards/rollout_reward_func/mean": 0.453125,
      "rewards/rollout_reward_func/std": 0.730891764163971,
      "sampling/importance_sampling_ratio/max": 1.678112506866455,
      "sampling/importance_sampling_ratio/mean": 0.8964781761169434,
      "sampling/importance_sampling_ratio/min": 3.111264128530305e-22,
      "sampling/sampling_logp_difference/max": 22.4278621673584,
      "sampling/sampling_logp_difference/mean": 0.1562006175518036,
      "step": 257,
      "step_time": 13.292072867974639
    },
    {
      "clip_ratio/high_max": 0.011067708488553762,
      "clip_ratio/high_mean": 0.005533854244276881,
      "clip_ratio/low_mean": 0.0049143144860863686,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01044816873036325,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 630.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 240.46875,
      "completions/mean_terminated_length": 247.33334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5336659699678421,
      "epoch": 0.00516,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.436086654663086,
      "kl": 0.6200018874369562,
      "learning_rate": 8.845700732275435e-06,
      "loss": -0.142,
      "num_tokens": 10346731.0,
      "reward": 0.377723753452301,
      "reward_std": 0.8817168474197388,
      "rewards/rollout_reward_func/mean": 0.377723753452301,
      "rewards/rollout_reward_func/std": 0.8673526048660278,
      "sampling/importance_sampling_ratio/max": 2.295440673828125,
      "sampling/importance_sampling_ratio/mean": 0.9543665647506714,
      "sampling/importance_sampling_ratio/min": 0.23825781047344208,
      "sampling/sampling_logp_difference/max": 1.0081114768981934,
      "sampling/sampling_logp_difference/mean": 0.05551508814096451,
      "step": 258,
      "step_time": 12.698149623000063
    },
    {
      "clip_ratio/high_max": 0.009114583488553762,
      "clip_ratio/high_mean": 0.004557291744276881,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02018229174427688,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 891.0,
      "completions/max_terminated_length": 891.0,
      "completions/mean_length": 281.96875,
      "completions/mean_terminated_length": 300.1481628417969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5405314713716507,
      "epoch": 0.00518,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9644437432289124,
      "kl": 1.4318017000332475,
      "learning_rate": 8.835854103996683e-06,
      "loss": -0.2119,
      "num_tokens": 10389046.0,
      "reward": 0.6257426738739014,
      "reward_std": 0.7057645916938782,
      "rewards/rollout_reward_func/mean": 0.6257426738739014,
      "rewards/rollout_reward_func/std": 0.6924726963043213,
      "sampling/importance_sampling_ratio/max": 1.8395767211914062,
      "sampling/importance_sampling_ratio/mean": 0.8937785625457764,
      "sampling/importance_sampling_ratio/min": 0.2512211799621582,
      "sampling/sampling_logp_difference/max": 1.340930461883545,
      "sampling/sampling_logp_difference/mean": 0.05504526197910309,
      "step": 259,
      "step_time": 13.371929887012811
    },
    {
      "clip_ratio/high_max": 0.025565476389601827,
      "clip_ratio/high_mean": 0.012782738194800913,
      "clip_ratio/low_mean": 0.0031250000465661287,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015907738241367042,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 670.0,
      "completions/max_terminated_length": 670.0,
      "completions/mean_length": 278.625,
      "completions/mean_terminated_length": 246.42308044433594,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5722781582735479,
      "epoch": 0.0052,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1958260536193848,
      "kl": 0.4839716963469982,
      "learning_rate": 8.825973366554254e-06,
      "loss": 0.0654,
      "num_tokens": 10432366.0,
      "reward": 0.6018750071525574,
      "reward_std": 0.48648905754089355,
      "rewards/rollout_reward_func/mean": 0.6018750071525574,
      "rewards/rollout_reward_func/std": 0.6177922487258911,
      "sampling/importance_sampling_ratio/max": 1.5975704193115234,
      "sampling/importance_sampling_ratio/mean": 0.7271268367767334,
      "sampling/importance_sampling_ratio/min": 1.7214414205358452e-25,
      "sampling/sampling_logp_difference/max": 25.105342864990234,
      "sampling/sampling_logp_difference/mean": 0.5097134709358215,
      "step": 260,
      "step_time": 12.550227359926794
    },
    {
      "clip_ratio/high_max": 0.0067471591755747795,
      "clip_ratio/high_mean": 0.0033735795877873898,
      "clip_ratio/low_mean": 0.0012499999720603228,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0046235795598477125,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 612.0,
      "completions/mean_length": 268.25,
      "completions/mean_terminated_length": 257.3599853515625,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5442182384431362,
      "epoch": 0.00522,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2828816175460815,
      "kl": 0.3458616966381669,
      "learning_rate": 8.816058650281758e-06,
      "loss": -0.0901,
      "num_tokens": 10474116.0,
      "reward": 0.6435146331787109,
      "reward_std": 0.564371645450592,
      "rewards/rollout_reward_func/mean": 0.6435146331787109,
      "rewards/rollout_reward_func/std": 0.5688491463661194,
      "sampling/importance_sampling_ratio/max": 2.0230302810668945,
      "sampling/importance_sampling_ratio/mean": 0.8262974619865417,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 28.231294631958008,
      "sampling/sampling_logp_difference/mean": 0.19613300263881683,
      "step": 261,
      "step_time": 12.983392989015556
    },
    {
      "clip_ratio/high_max": 0.039967758348211646,
      "clip_ratio/high_mean": 0.019983879174105823,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019983879174105823,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 223.875,
      "completions/mean_terminated_length": 234.63999938964844,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5416169241070747,
      "epoch": 0.00524,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.001020908355713,
      "kl": 0.3078559786081314,
      "learning_rate": 8.806110085960999e-06,
      "loss": 0.1877,
      "num_tokens": 10513740.0,
      "reward": 0.7890625,
      "reward_std": 0.36765778064727783,
      "rewards/rollout_reward_func/mean": 0.7890625,
      "rewards/rollout_reward_func/std": 0.4961617588996887,
      "sampling/importance_sampling_ratio/max": 2.0186269283294678,
      "sampling/importance_sampling_ratio/mean": 0.9157467484474182,
      "sampling/importance_sampling_ratio/min": 0.364488810300827,
      "sampling/sampling_logp_difference/max": 0.8611717224121094,
      "sampling/sampling_logp_difference/mean": 0.05427710711956024,
      "step": 262,
      "step_time": 12.6603681150591
    },
    {
      "clip_ratio/high_max": 0.0041082974057644606,
      "clip_ratio/high_mean": 0.0020541487028822303,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0020541487028822303,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 209.21875,
      "completions/mean_terminated_length": 216.92593383789062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5822854172438383,
      "epoch": 0.00526,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4004669189453125,
      "kl": 0.40430155396461487,
      "learning_rate": 8.79612780482027e-06,
      "loss": 0.0707,
      "num_tokens": 10553402.0,
      "reward": 0.6521874666213989,
      "reward_std": 0.6761115789413452,
      "rewards/rollout_reward_func/mean": 0.6521874666213989,
      "rewards/rollout_reward_func/std": 0.6600866317749023,
      "sampling/importance_sampling_ratio/max": 1.9852436780929565,
      "sampling/importance_sampling_ratio/mean": 0.9465670585632324,
      "sampling/importance_sampling_ratio/min": 0.10981152206659317,
      "sampling/sampling_logp_difference/max": 0.7001292705535889,
      "sampling/sampling_logp_difference/mean": 0.05766445770859718,
      "step": 263,
      "step_time": 13.065498438023496
    },
    {
      "clip_ratio/high_max": 0.006510416744276881,
      "clip_ratio/high_mean": 0.0032552083721384406,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005859375,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 572.0,
      "completions/mean_length": 165.5625,
      "completions/mean_terminated_length": 161.5416717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5954251307994127,
      "epoch": 0.00528,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7429633140563965,
      "kl": 0.42005133628845215,
      "learning_rate": 8.786111938532606e-06,
      "loss": -0.0828,
      "num_tokens": 10589986.0,
      "reward": 0.7228125333786011,
      "reward_std": 0.48502302169799805,
      "rewards/rollout_reward_func/mean": 0.7228125333786011,
      "rewards/rollout_reward_func/std": 0.5343988537788391,
      "sampling/importance_sampling_ratio/max": 2.099916934967041,
      "sampling/importance_sampling_ratio/mean": 1.0316898822784424,
      "sampling/importance_sampling_ratio/min": 0.5332803130149841,
      "sampling/sampling_logp_difference/max": 0.934959888458252,
      "sampling/sampling_logp_difference/mean": 0.05554133653640747,
      "step": 264,
      "step_time": 12.127957556032925
    },
    {
      "clip_ratio/high_max": 0.012673611287027597,
      "clip_ratio/high_mean": 0.0075868054991588,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0075868054991588,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 648.0,
      "completions/max_terminated_length": 622.0,
      "completions/mean_length": 272.28125,
      "completions/mean_terminated_length": 246.6428680419922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6194133386015892,
      "epoch": 0.0053,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0931178331375122,
      "kl": 0.38702596444636583,
      "learning_rate": 8.776062619214053e-06,
      "loss": -0.0888,
      "num_tokens": 10631295.0,
      "reward": 0.7540624737739563,
      "reward_std": 0.45535925030708313,
      "rewards/rollout_reward_func/mean": 0.7540624737739563,
      "rewards/rollout_reward_func/std": 0.5763671398162842,
      "sampling/importance_sampling_ratio/max": 1.4245022535324097,
      "sampling/importance_sampling_ratio/mean": 0.8473235368728638,
      "sampling/importance_sampling_ratio/min": 0.2802773416042328,
      "sampling/sampling_logp_difference/max": 1.1858439445495605,
      "sampling/sampling_logp_difference/mean": 0.06098029017448425,
      "step": 265,
      "step_time": 13.192183319988544
    },
    {
      "clip_ratio/high_max": 0.007170516299083829,
      "clip_ratio/high_mean": 0.0035852581495419145,
      "clip_ratio/low_mean": 0.016874999972060323,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020460258121602237,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 594.0,
      "completions/max_terminated_length": 567.0,
      "completions/mean_length": 179.375,
      "completions/mean_terminated_length": 126.13043975830078,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4933217689394951,
      "epoch": 0.00532,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9869648814201355,
      "kl": 1.2245624396018684,
      "learning_rate": 8.765979979421928e-06,
      "loss": -0.1172,
      "num_tokens": 10670168.0,
      "reward": 0.5262017846107483,
      "reward_std": 0.7280963659286499,
      "rewards/rollout_reward_func/mean": 0.5262017846107483,
      "rewards/rollout_reward_func/std": 0.7481173276901245,
      "sampling/importance_sampling_ratio/max": 1.6396535634994507,
      "sampling/importance_sampling_ratio/mean": 0.8665092587471008,
      "sampling/importance_sampling_ratio/min": 0.1976049393415451,
      "sampling/sampling_logp_difference/max": 1.162266492843628,
      "sampling/sampling_logp_difference/mean": 0.06339786946773529,
      "step": 266,
      "step_time": 12.375456522015156
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 242.9375,
      "completions/mean_terminated_length": 247.75001525878906,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.46095453947782516,
      "epoch": 0.00534,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8330289721488953,
      "kl": 0.48035445529967546,
      "learning_rate": 8.755864152153064e-06,
      "loss": -0.078,
      "num_tokens": 10711401.0,
      "reward": 0.7387499809265137,
      "reward_std": 0.524664580821991,
      "rewards/rollout_reward_func/mean": 0.7387499809265137,
      "rewards/rollout_reward_func/std": 0.5297579765319824,
      "sampling/importance_sampling_ratio/max": 2.2813665866851807,
      "sampling/importance_sampling_ratio/mean": 1.1036913394927979,
      "sampling/importance_sampling_ratio/min": 3.217266403507185e-25,
      "sampling/sampling_logp_difference/max": 22.305538177490234,
      "sampling/sampling_logp_difference/mean": 0.16871777176856995,
      "step": 267,
      "step_time": 12.157360893019359
    },
    {
      "clip_ratio/high_max": 0.013424859382212162,
      "clip_ratio/high_mean": 0.006712429691106081,
      "clip_ratio/low_mean": 0.0006510416860692203,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007363471377175301,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1152.0,
      "completions/max_terminated_length": 659.0,
      "completions/mean_length": 242.4375,
      "completions/mean_terminated_length": 217.33334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5609332341700792,
      "epoch": 0.00536,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1913412809371948,
      "kl": 0.4231171114370227,
      "learning_rate": 8.745715270842064e-06,
      "loss": -0.0038,
      "num_tokens": 10752095.0,
      "reward": 0.754687488079071,
      "reward_std": 0.5700393319129944,
      "rewards/rollout_reward_func/mean": 0.754687488079071,
      "rewards/rollout_reward_func/std": 0.575544536113739,
      "sampling/importance_sampling_ratio/max": 1.4563279151916504,
      "sampling/importance_sampling_ratio/mean": 0.9225751161575317,
      "sampling/importance_sampling_ratio/min": 3.2449910559859867e-15,
      "sampling/sampling_logp_difference/max": 18.129619598388672,
      "sampling/sampling_logp_difference/mean": 0.10878469794988632,
      "step": 268,
      "step_time": 14.310341471005813
    },
    {
      "clip_ratio/high_max": 0.009765625,
      "clip_ratio/high_mean": 0.0048828125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0048828125,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 610.0,
      "completions/max_terminated_length": 610.0,
      "completions/mean_length": 254.40625,
      "completions/mean_terminated_length": 249.1851806640625,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.519100209698081,
      "epoch": 0.00538,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1924206018447876,
      "kl": 0.39489909913390875,
      "learning_rate": 8.73553346935953e-06,
      "loss": -0.0114,
      "num_tokens": 10793654.0,
      "reward": 0.72718745470047,
      "reward_std": 0.5035135746002197,
      "rewards/rollout_reward_func/mean": 0.72718745470047,
      "rewards/rollout_reward_func/std": 0.5359118580818176,
      "sampling/importance_sampling_ratio/max": 2.1670854091644287,
      "sampling/importance_sampling_ratio/mean": 0.885656476020813,
      "sampling/importance_sampling_ratio/min": 6.131218200504678e-21,
      "sampling/sampling_logp_difference/max": 23.659204483032227,
      "sampling/sampling_logp_difference/mean": 0.24496234953403473,
      "step": 269,
      "step_time": 12.778201229986735
    },
    {
      "clip_ratio/high_max": 0.0036764706019312143,
      "clip_ratio/high_mean": 0.0018382353009656072,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004442402045242488,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 668.0,
      "completions/max_terminated_length": 668.0,
      "completions/mean_length": 212.25,
      "completions/mean_terminated_length": 210.40000915527344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4665683899074793,
      "epoch": 0.0054,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8091368675231934,
      "kl": 0.7170595228672028,
      "learning_rate": 8.725318882010307e-06,
      "loss": -0.0411,
      "num_tokens": 10833507.0,
      "reward": 0.5821875333786011,
      "reward_std": 0.5702049732208252,
      "rewards/rollout_reward_func/mean": 0.5821875333786011,
      "rewards/rollout_reward_func/std": 0.7242158055305481,
      "sampling/importance_sampling_ratio/max": 1.497339129447937,
      "sampling/importance_sampling_ratio/mean": 0.835294246673584,
      "sampling/importance_sampling_ratio/min": 1.5182569218226243e-14,
      "sampling/sampling_logp_difference/max": 20.72052764892578,
      "sampling/sampling_logp_difference/mean": 0.20571781694889069,
      "step": 270,
      "step_time": 12.333919421012979
    },
    {
      "clip_ratio/high_max": 0.01051794015802443,
      "clip_ratio/high_mean": 0.006235532579012215,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008188657579012215,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 576.0,
      "completions/max_terminated_length": 576.0,
      "completions/mean_length": 182.71875,
      "completions/mean_terminated_length": 193.95834350585938,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5774502046406269,
      "epoch": 0.00542,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.069632887840271,
      "kl": 0.4078567810356617,
      "learning_rate": 8.715071643531708e-06,
      "loss": -0.2345,
      "num_tokens": 10873078.0,
      "reward": 0.6725000143051147,
      "reward_std": 0.6612372398376465,
      "rewards/rollout_reward_func/mean": 0.6725000143051147,
      "rewards/rollout_reward_func/std": 0.665320634841919,
      "sampling/importance_sampling_ratio/max": 1.9237180948257446,
      "sampling/importance_sampling_ratio/mean": 0.9989001750946045,
      "sampling/importance_sampling_ratio/min": 3.0350064515469186e-24,
      "sampling/sampling_logp_difference/max": 27.39958953857422,
      "sampling/sampling_logp_difference/mean": 0.2891521453857422,
      "step": 271,
      "step_time": 12.564480729983188
    },
    {
      "clip_ratio/high_max": 0.010685350745916367,
      "clip_ratio/high_mean": 0.005342675372958183,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005342675372958183,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 272.625,
      "completions/mean_terminated_length": 261.3333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5548490174114704,
      "epoch": 0.00544,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0656490325927734,
      "kl": 0.5595456697046757,
      "learning_rate": 8.704791889091737e-06,
      "loss": -0.0042,
      "num_tokens": 10915494.0,
      "reward": 0.5784374475479126,
      "reward_std": 0.5408496260643005,
      "rewards/rollout_reward_func/mean": 0.5784374475479126,
      "rewards/rollout_reward_func/std": 0.5637896060943604,
      "sampling/importance_sampling_ratio/max": 1.9444421529769897,
      "sampling/importance_sampling_ratio/mean": 0.9131875038146973,
      "sampling/importance_sampling_ratio/min": 2.360117842772568e-22,
      "sampling/sampling_logp_difference/max": 19.929658889770508,
      "sampling/sampling_logp_difference/mean": 0.17713448405265808,
      "step": 272,
      "step_time": 13.057954037969466
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.017578125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01953125,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 218.25,
      "completions/mean_terminated_length": 226.35714721679688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5340163968503475,
      "epoch": 0.00546,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.190561056137085,
      "kl": 1.023565087467432,
      "learning_rate": 8.6944797542873e-06,
      "loss": 0.0244,
      "num_tokens": 10955721.0,
      "reward": 0.6949999332427979,
      "reward_std": 0.5816850662231445,
      "rewards/rollout_reward_func/mean": 0.6949999332427979,
      "rewards/rollout_reward_func/std": 0.5894557237625122,
      "sampling/importance_sampling_ratio/max": 1.7137608528137207,
      "sampling/importance_sampling_ratio/mean": 1.0312073230743408,
      "sampling/importance_sampling_ratio/min": 0.24780932068824768,
      "sampling/sampling_logp_difference/max": 0.9306693077087402,
      "sampling/sampling_logp_difference/mean": 0.049734748899936676,
      "step": 273,
      "step_time": 12.738474394005607
    },
    {
      "clip_ratio/high_max": 0.02329727611504495,
      "clip_ratio/high_mean": 0.011648638057522476,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012625200557522476,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 1048.0,
      "completions/max_terminated_length": 1048.0,
      "completions/mean_length": 250.34375,
      "completions/mean_terminated_length": 265.5555725097656,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5208232775330544,
      "epoch": 0.00548,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0655300617218018,
      "kl": 1.2283964240923524,
      "learning_rate": 8.68413537514243e-06,
      "loss": -0.0334,
      "num_tokens": 10996164.0,
      "reward": 0.6918749809265137,
      "reward_std": 0.511249303817749,
      "rewards/rollout_reward_func/mean": 0.6918749809265137,
      "rewards/rollout_reward_func/std": 0.6148245334625244,
      "sampling/importance_sampling_ratio/max": 1.764540672302246,
      "sampling/importance_sampling_ratio/mean": 0.9150140285491943,
      "sampling/importance_sampling_ratio/min": 1.0807363586648752e-26,
      "sampling/sampling_logp_difference/max": 26.949111938476562,
      "sampling/sampling_logp_difference/mean": 0.28716331720352173,
      "step": 274,
      "step_time": 13.262181975020212
    },
    {
      "clip_ratio/high_max": 0.01243832241743803,
      "clip_ratio/high_mean": 0.007707256358116865,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007707256358116865,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 624.0,
      "completions/max_terminated_length": 624.0,
      "completions/mean_length": 241.65625,
      "completions/mean_terminated_length": 225.84616088867188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5480573512613773,
      "epoch": 0.0055,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1609772443771362,
      "kl": 1.0592744080349803,
      "learning_rate": 8.673758888106481e-06,
      "loss": -0.2074,
      "num_tokens": 11036454.0,
      "reward": 0.6686612367630005,
      "reward_std": 0.6928995847702026,
      "rewards/rollout_reward_func/mean": 0.6686612367630005,
      "rewards/rollout_reward_func/std": 0.7332102656364441,
      "sampling/importance_sampling_ratio/max": 2.303600549697876,
      "sampling/importance_sampling_ratio/mean": 1.0303168296813965,
      "sampling/importance_sampling_ratio/min": 0.29827842116355896,
      "sampling/sampling_logp_difference/max": 0.9684064388275146,
      "sampling/sampling_logp_difference/mean": 0.06136919558048248,
      "step": 275,
      "step_time": 12.481000076964847
    },
    {
      "clip_ratio/high_max": 0.020295516354963183,
      "clip_ratio/high_mean": 0.010147758177481592,
      "clip_ratio/low_mean": 0.0028409091755747795,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012988667353056371,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 625.0,
      "completions/mean_length": 244.5625,
      "completions/mean_terminated_length": 202.8148193359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.596262913197279,
      "epoch": 0.00552,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3872603178024292,
      "kl": 0.334263673517853,
      "learning_rate": 8.663350430052328e-06,
      "loss": -0.0336,
      "num_tokens": 11077656.0,
      "reward": 0.635937511920929,
      "reward_std": 0.6610380411148071,
      "rewards/rollout_reward_func/mean": 0.635937511920929,
      "rewards/rollout_reward_func/std": 0.6527156829833984,
      "sampling/importance_sampling_ratio/max": 1.8096270561218262,
      "sampling/importance_sampling_ratio/mean": 0.9158169627189636,
      "sampling/importance_sampling_ratio/min": 1.9915565550518954e-20,
      "sampling/sampling_logp_difference/max": 23.92959976196289,
      "sampling/sampling_logp_difference/mean": 0.23112377524375916,
      "step": 276,
      "step_time": 13.150878515996737
    },
    {
      "clip_ratio/high_max": 0.006760817486792803,
      "clip_ratio/high_mean": 0.0033804087433964014,
      "clip_ratio/low_mean": 0.0052083334885537624,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008588742231950164,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 158.96875,
      "completions/mean_terminated_length": 147.6428680419922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5659680310636759,
      "epoch": 0.00554,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.100131869316101,
      "kl": 0.7795611056499183,
      "learning_rate": 8.652910138274573e-06,
      "loss": -0.1168,
      "num_tokens": 11114003.0,
      "reward": 0.6312500238418579,
      "reward_std": 0.7210004329681396,
      "rewards/rollout_reward_func/mean": 0.6312500238418579,
      "rewards/rollout_reward_func/std": 0.7109137177467346,
      "sampling/importance_sampling_ratio/max": 1.9644155502319336,
      "sampling/importance_sampling_ratio/mean": 0.9292389750480652,
      "sampling/importance_sampling_ratio/min": 6.264776627039575e-28,
      "sampling/sampling_logp_difference/max": 22.871749877929688,
      "sampling/sampling_logp_difference/mean": 0.31702089309692383,
      "step": 277,
      "step_time": 11.91772462100198
    },
    {
      "clip_ratio/high_max": 0.009331597248092294,
      "clip_ratio/high_mean": 0.004665798624046147,
      "clip_ratio/low_mean": 0.0028409091755747795,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007506707799620926,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 244.875,
      "completions/mean_terminated_length": 232.83999633789062,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.5419516079127789,
      "epoch": 0.00556,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4551398754119873,
      "kl": 0.5122738927602768,
      "learning_rate": 8.642438150487718e-06,
      "loss": -0.2642,
      "num_tokens": 11154308.0,
      "reward": 0.4708487391471863,
      "reward_std": 0.873871386051178,
      "rewards/rollout_reward_func/mean": 0.4708487391471863,
      "rewards/rollout_reward_func/std": 0.8449071645736694,
      "sampling/importance_sampling_ratio/max": 1.702509880065918,
      "sampling/importance_sampling_ratio/mean": 1.0328657627105713,
      "sampling/importance_sampling_ratio/min": 0.4034571945667267,
      "sampling/sampling_logp_difference/max": 0.8790719509124756,
      "sampling/sampling_logp_difference/mean": 0.05516907945275307,
      "step": 278,
      "step_time": 12.844105629003025
    },
    {
      "clip_ratio/high_max": 0.03244791668839753,
      "clip_ratio/high_mean": 0.016223958344198763,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018177083344198763,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 606.0,
      "completions/mean_length": 229.03125,
      "completions/mean_terminated_length": 201.27586364746094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.516918670386076,
      "epoch": 0.00558,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9456140995025635,
      "kl": 0.54081722907722,
      "learning_rate": 8.631934604824366e-06,
      "loss": -0.0668,
      "num_tokens": 11194459.0,
      "reward": 0.46660298109054565,
      "reward_std": 0.8039901256561279,
      "rewards/rollout_reward_func/mean": 0.46660298109054565,
      "rewards/rollout_reward_func/std": 0.8242729306221008,
      "sampling/importance_sampling_ratio/max": 1.7011460065841675,
      "sampling/importance_sampling_ratio/mean": 0.903037428855896,
      "sampling/importance_sampling_ratio/min": 0.2492627203464508,
      "sampling/sampling_logp_difference/max": 0.9857833385467529,
      "sampling/sampling_logp_difference/mean": 0.05905836820602417,
      "step": 279,
      "step_time": 12.382074195003952
    },
    {
      "clip_ratio/high_max": 0.02184606483206153,
      "clip_ratio/high_mean": 0.010923032416030765,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011899594916030765,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 682.0,
      "completions/max_terminated_length": 682.0,
      "completions/mean_length": 333.46875,
      "completions/mean_terminated_length": 327.4137878417969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5320689864456654,
      "epoch": 0.0056,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1132533550262451,
      "kl": 0.973154604434967,
      "learning_rate": 8.62139963983338e-06,
      "loss": -0.0952,
      "num_tokens": 11239652.0,
      "reward": 0.45272374153137207,
      "reward_std": 0.7288464307785034,
      "rewards/rollout_reward_func/mean": 0.45272374153137207,
      "rewards/rollout_reward_func/std": 0.7334537506103516,
      "sampling/importance_sampling_ratio/max": 2.014308452606201,
      "sampling/importance_sampling_ratio/mean": 0.8367502689361572,
      "sampling/importance_sampling_ratio/min": 2.465373608387389e-12,
      "sampling/sampling_logp_difference/max": 15.376739501953125,
      "sampling/sampling_logp_difference/mean": 0.1576831340789795,
      "step": 280,
      "step_time": 12.295460028995876
    },
    {
      "clip_ratio/high_max": 0.006393861956894398,
      "clip_ratio/high_mean": 0.003196930978447199,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003196930978447199,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 621.0,
      "completions/max_terminated_length": 621.0,
      "completions/mean_length": 250.53125,
      "completions/mean_terminated_length": 247.50001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5899146534502506,
      "epoch": 0.00562,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8447449803352356,
      "kl": 0.4785450678318739,
      "learning_rate": 8.610833394478073e-06,
      "loss": -0.1695,
      "num_tokens": 11281700.0,
      "reward": 0.6659780144691467,
      "reward_std": 0.6413812637329102,
      "rewards/rollout_reward_func/mean": 0.6659780144691467,
      "rewards/rollout_reward_func/std": 0.6745311617851257,
      "sampling/importance_sampling_ratio/max": 1.7575149536132812,
      "sampling/importance_sampling_ratio/mean": 0.8404484987258911,
      "sampling/importance_sampling_ratio/min": 3.1101904178029523e-18,
      "sampling/sampling_logp_difference/max": 26.98841094970703,
      "sampling/sampling_logp_difference/mean": 0.24903328716754913,
      "step": 281,
      "step_time": 12.777874022023752
    },
    {
      "clip_ratio/high_max": 0.005859375,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0029296875,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 594.0,
      "completions/max_terminated_length": 594.0,
      "completions/mean_length": 258.09375,
      "completions/mean_terminated_length": 250.4666748046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5029415879398584,
      "epoch": 0.00564,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9736205339431763,
      "kl": 0.4333905028179288,
      "learning_rate": 8.60023600813436e-06,
      "loss": -0.0955,
      "num_tokens": 11323087.0,
      "reward": 0.7712500095367432,
      "reward_std": 0.5019099712371826,
      "rewards/rollout_reward_func/mean": 0.7712500095367432,
      "rewards/rollout_reward_func/std": 0.5709909200668335,
      "sampling/importance_sampling_ratio/max": 2.0703377723693848,
      "sampling/importance_sampling_ratio/mean": 1.0094125270843506,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 21.629302978515625,
      "sampling/sampling_logp_difference/mean": 0.1337393969297409,
      "step": 282,
      "step_time": 12.50957973099139
    },
    {
      "clip_ratio/high_max": 0.004670516354963183,
      "clip_ratio/high_mean": 0.0023352581774815917,
      "clip_ratio/low_mean": 0.011366959195584059,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01370221737306565,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 688.0,
      "completions/max_terminated_length": 688.0,
      "completions/mean_length": 277.5,
      "completions/mean_terminated_length": 267.4444580078125,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.5634420625865459,
      "epoch": 0.00566,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.550695538520813,
      "kl": 0.39020966924726963,
      "learning_rate": 8.589607620588937e-06,
      "loss": -0.1239,
      "num_tokens": 11365574.0,
      "reward": 0.7999999523162842,
      "reward_std": 0.42354241013526917,
      "rewards/rollout_reward_func/mean": 0.7999999523162842,
      "rewards/rollout_reward_func/std": 0.49296990036964417,
      "sampling/importance_sampling_ratio/max": 2.0069665908813477,
      "sampling/importance_sampling_ratio/mean": 1.0503177642822266,
      "sampling/importance_sampling_ratio/min": 5.253793927684214e-12,
      "sampling/sampling_logp_difference/max": 20.84823989868164,
      "sampling/sampling_logp_difference/mean": 0.09940419346094131,
      "step": 283,
      "step_time": 12.879158257012023
    },
    {
      "clip_ratio/high_max": 0.0013020833721384406,
      "clip_ratio/high_mean": 0.0006510416860692203,
      "clip_ratio/low_mean": 0.01790364587213844,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01855468755820766,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 1269.0,
      "completions/max_terminated_length": 700.0,
      "completions/mean_length": 298.4375,
      "completions/mean_terminated_length": 216.87998962402344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5229798629879951,
      "epoch": 0.00568,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4341779947280884,
      "kl": 0.37283281050622463,
      "learning_rate": 8.578948372037416e-06,
      "loss": -0.0406,
      "num_tokens": 11407609.0,
      "reward": 0.6974999904632568,
      "reward_std": 0.5796388983726501,
      "rewards/rollout_reward_func/mean": 0.6974999904632568,
      "rewards/rollout_reward_func/std": 0.5909314751625061,
      "sampling/importance_sampling_ratio/max": 1.9875483512878418,
      "sampling/importance_sampling_ratio/mean": 1.0286742448806763,
      "sampling/importance_sampling_ratio/min": 0.3221196234226227,
      "sampling/sampling_logp_difference/max": 0.7662172317504883,
      "sampling/sampling_logp_difference/mean": 0.0451638363301754,
      "step": 284,
      "step_time": 15.033717533995514
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0013020833721384406,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022786458721384406,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 854.0,
      "completions/max_terminated_length": 854.0,
      "completions/mean_length": 299.84375,
      "completions/mean_terminated_length": 273.0,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5817802827805281,
      "epoch": 0.0057,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4754000902175903,
      "kl": 0.2602753662504256,
      "learning_rate": 8.568258403082492e-06,
      "loss": -0.1543,
      "num_tokens": 11451186.0,
      "reward": 0.6428124904632568,
      "reward_std": 0.5766918063163757,
      "rewards/rollout_reward_func/mean": 0.6428124904632568,
      "rewards/rollout_reward_func/std": 0.6210149526596069,
      "sampling/importance_sampling_ratio/max": 2.1277589797973633,
      "sampling/importance_sampling_ratio/mean": 1.0695533752441406,
      "sampling/importance_sampling_ratio/min": 1.52068047099316e-32,
      "sampling/sampling_logp_difference/max": 23.084009170532227,
      "sampling/sampling_logp_difference/mean": 0.22904697060585022,
      "step": 285,
      "step_time": 13.754947948022163
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.007406994234770536,
      "clip_ratio/low_min": 0.0013020833721384406,
      "clip_ratio/region_mean": 0.007406994234770536,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1107.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 283.0,
      "completions/mean_terminated_length": 231.25001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5636868048459291,
      "epoch": 0.00572,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2108744382858276,
      "kl": 0.7925708647817373,
      "learning_rate": 8.557537854732082e-06,
      "loss": 0.0746,
      "num_tokens": 11491958.0,
      "reward": 0.5307793021202087,
      "reward_std": 0.7161239385604858,
      "rewards/rollout_reward_func/mean": 0.5307793021202087,
      "rewards/rollout_reward_func/std": 0.7110646963119507,
      "sampling/importance_sampling_ratio/max": 1.8880395889282227,
      "sampling/importance_sampling_ratio/mean": 0.9086657762527466,
      "sampling/importance_sampling_ratio/min": 4.4036750755286606e-18,
      "sampling/sampling_logp_difference/max": 27.03826904296875,
      "sampling/sampling_logp_difference/mean": 0.23364484310150146,
      "step": 286,
      "step_time": 14.06892507300654
    },
    {
      "clip_ratio/high_max": 0.007436052430421114,
      "clip_ratio/high_mean": 0.004834097810089588,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004834097810089588,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 613.0,
      "completions/max_terminated_length": 613.0,
      "completions/mean_length": 146.25,
      "completions/mean_terminated_length": 152.73077392578125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5507006254047155,
      "epoch": 0.00574,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8293165564537048,
      "kl": 0.6477417880669236,
      "learning_rate": 8.546786868397465e-06,
      "loss": -0.1633,
      "num_tokens": 11528037.0,
      "reward": 0.6509374976158142,
      "reward_std": 0.7631633281707764,
      "rewards/rollout_reward_func/mean": 0.6509374976158142,
      "rewards/rollout_reward_func/std": 0.7280391454696655,
      "sampling/importance_sampling_ratio/max": 1.6377354860305786,
      "sampling/importance_sampling_ratio/mean": 0.8656424283981323,
      "sampling/importance_sampling_ratio/min": 2.3125890363418746e-22,
      "sampling/sampling_logp_difference/max": 20.914348602294922,
      "sampling/sampling_logp_difference/mean": 0.21854905784130096,
      "step": 287,
      "step_time": 12.294496908973088
    },
    {
      "clip_ratio/high_max": 0.021234031533822417,
      "clip_ratio/high_mean": 0.010617015766911209,
      "clip_ratio/low_mean": 0.0020833334419876337,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012700349208898842,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 278.71875,
      "completions/mean_terminated_length": 243.57693481445312,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5564366970211267,
      "epoch": 0.00576,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9643985033035278,
      "kl": 0.3522453736513853,
      "learning_rate": 8.536005585891422e-06,
      "loss": 0.0541,
      "num_tokens": 11570482.0,
      "reward": 0.5493749976158142,
      "reward_std": 0.6281640529632568,
      "rewards/rollout_reward_func/mean": 0.5493749976158142,
      "rewards/rollout_reward_func/std": 0.6813077330589294,
      "sampling/importance_sampling_ratio/max": 1.616706371307373,
      "sampling/importance_sampling_ratio/mean": 0.8954521417617798,
      "sampling/importance_sampling_ratio/min": 1.2701151257432544e-33,
      "sampling/sampling_logp_difference/max": 20.18747329711914,
      "sampling/sampling_logp_difference/mean": 0.3566548228263855,
      "step": 288,
      "step_time": 12.42214960801357
    },
    {
      "clip_ratio/high_max": 0.020996880251914263,
      "clip_ratio/high_mean": 0.010498440125957131,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012451565125957131,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 675.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 250.40625,
      "completions/mean_terminated_length": 251.16668701171875,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4688313016667962,
      "epoch": 0.00578,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.911766529083252,
      "kl": 0.6891351994127035,
      "learning_rate": 8.525194149426354e-06,
      "loss": 0.031,
      "num_tokens": 11611769.0,
      "reward": 0.609375,
      "reward_std": 0.7464461326599121,
      "rewards/rollout_reward_func/mean": 0.609375,
      "rewards/rollout_reward_func/std": 0.7165371179580688,
      "sampling/importance_sampling_ratio/max": 2.8157880306243896,
      "sampling/importance_sampling_ratio/mean": 1.0286338329315186,
      "sampling/importance_sampling_ratio/min": 1.511336122916873e-08,
      "sampling/sampling_logp_difference/max": 17.50009536743164,
      "sampling/sampling_logp_difference/mean": 0.08279306441545486,
      "step": 289,
      "step_time": 12.536049625006854
    },
    {
      "clip_ratio/high_max": 0.009982638992369175,
      "clip_ratio/high_mean": 0.0049913194961845875,
      "clip_ratio/low_mean": 0.0011160714784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006107390974648297,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 683.0,
      "completions/mean_length": 213.4375,
      "completions/mean_terminated_length": 233.862060546875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4839852638542652,
      "epoch": 0.0058,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.461670994758606,
      "kl": 0.5655760271474719,
      "learning_rate": 8.514352701612418e-06,
      "loss": 0.0895,
      "num_tokens": 11651493.0,
      "reward": 0.7262499928474426,
      "reward_std": 0.600041389465332,
      "rewards/rollout_reward_func/mean": 0.7262499928474426,
      "rewards/rollout_reward_func/std": 0.5908209085464478,
      "sampling/importance_sampling_ratio/max": 2.5294578075408936,
      "sampling/importance_sampling_ratio/mean": 0.9673727750778198,
      "sampling/importance_sampling_ratio/min": 2.3538802426021613e-20,
      "sampling/sampling_logp_difference/max": 19.312036514282227,
      "sampling/sampling_logp_difference/mean": 0.13013553619384766,
      "step": 290,
      "step_time": 12.797142498995527
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.004448784748092294,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006401909748092294,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 620.0,
      "completions/max_terminated_length": 617.0,
      "completions/mean_length": 169.25,
      "completions/mean_terminated_length": 167.6666717529297,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.485273415222764,
      "epoch": 0.00582,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7888396978378296,
      "kl": 0.7630397314205766,
      "learning_rate": 8.503481385455637e-06,
      "loss": 0.0687,
      "num_tokens": 11688515.0,
      "reward": 0.7278125286102295,
      "reward_std": 0.4295271933078766,
      "rewards/rollout_reward_func/mean": 0.7278125286102295,
      "rewards/rollout_reward_func/std": 0.5286912322044373,
      "sampling/importance_sampling_ratio/max": 1.5498046875,
      "sampling/importance_sampling_ratio/mean": 0.9917097091674805,
      "sampling/importance_sampling_ratio/min": 0.24115660786628723,
      "sampling/sampling_logp_difference/max": 0.7808971405029297,
      "sampling/sampling_logp_difference/mean": 0.04827158525586128,
      "step": 291,
      "step_time": 11.87558427802287
    },
    {
      "clip_ratio/high_max": 0.005425347248092294,
      "clip_ratio/high_mean": 0.002712673624046147,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003689236124046147,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 539.0,
      "completions/max_terminated_length": 531.0,
      "completions/mean_length": 228.53125,
      "completions/mean_terminated_length": 200.0800018310547,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5074332226067781,
      "epoch": 0.00584,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1244572401046753,
      "kl": 0.3679599119350314,
      "learning_rate": 8.492580344356023e-06,
      "loss": -0.1815,
      "num_tokens": 11729354.0,
      "reward": 0.8650000095367432,
      "reward_std": 0.39495742321014404,
      "rewards/rollout_reward_func/mean": 0.8650000095367432,
      "rewards/rollout_reward_func/std": 0.44845253229141235,
      "sampling/importance_sampling_ratio/max": 1.5861928462982178,
      "sampling/importance_sampling_ratio/mean": 1.0474636554718018,
      "sampling/importance_sampling_ratio/min": 0.5460078716278076,
      "sampling/sampling_logp_difference/max": 0.7365200519561768,
      "sampling/sampling_logp_difference/mean": 0.040405042469501495,
      "step": 292,
      "step_time": 12.952557066979352
    },
    {
      "clip_ratio/high_max": 0.020461309934034944,
      "clip_ratio/high_mean": 0.010230654967017472,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011207217467017472,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 922.0,
      "completions/max_terminated_length": 922.0,
      "completions/mean_length": 344.59375,
      "completions/mean_terminated_length": 356.7500305175781,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5577900279313326,
      "epoch": 0.00586,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3503241539001465,
      "kl": 1.6989887291565537,
      "learning_rate": 8.481649722105677e-06,
      "loss": -0.0423,
      "num_tokens": 11774948.0,
      "reward": 0.7646875381469727,
      "reward_std": 0.46988892555236816,
      "rewards/rollout_reward_func/mean": 0.7646875381469727,
      "rewards/rollout_reward_func/std": 0.5736076235771179,
      "sampling/importance_sampling_ratio/max": 1.8309789896011353,
      "sampling/importance_sampling_ratio/mean": 0.8864504098892212,
      "sampling/importance_sampling_ratio/min": 0.08218871057033539,
      "sampling/sampling_logp_difference/max": 1.3203423023223877,
      "sampling/sampling_logp_difference/mean": 0.056817445904016495,
      "step": 293,
      "step_time": 14.473504206980579
    },
    {
      "clip_ratio/high_max": 0.012369791744276881,
      "clip_ratio/high_mean": 0.006184895872138441,
      "clip_ratio/low_mean": 0.0006510416860692203,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006835937558207661,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1134.0,
      "completions/max_terminated_length": 688.0,
      "completions/mean_length": 261.5625,
      "completions/mean_terminated_length": 246.38462829589844,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5267591383308172,
      "epoch": 0.00588,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2456529140472412,
      "kl": 1.1880546445026994,
      "learning_rate": 8.470689662886895e-06,
      "loss": 0.0075,
      "num_tokens": 11817709.0,
      "reward": 0.703125,
      "reward_std": 0.5321820974349976,
      "rewards/rollout_reward_func/mean": 0.703125,
      "rewards/rollout_reward_func/std": 0.5389142632484436,
      "sampling/importance_sampling_ratio/max": 2.032693386077881,
      "sampling/importance_sampling_ratio/mean": 0.9442247152328491,
      "sampling/importance_sampling_ratio/min": 7.387902248706279e-22,
      "sampling/sampling_logp_difference/max": 22.169086456298828,
      "sampling/sampling_logp_difference/mean": 0.21211135387420654,
      "step": 294,
      "step_time": 13.96328596798412
    },
    {
      "clip_ratio/high_max": 0.004166666883975267,
      "clip_ratio/high_mean": 0.0020833334419876337,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0020833334419876337,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 1121.0,
      "completions/max_terminated_length": 1121.0,
      "completions/mean_length": 220.25,
      "completions/mean_terminated_length": 222.10345458984375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4604577701538801,
      "epoch": 0.0059,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9505118727684021,
      "kl": 0.4429945796728134,
      "learning_rate": 8.459700311270268e-06,
      "loss": 0.0373,
      "num_tokens": 11856466.0,
      "reward": 0.7287499904632568,
      "reward_std": 0.5220839381217957,
      "rewards/rollout_reward_func/mean": 0.7287499904632568,
      "rewards/rollout_reward_func/std": 0.5250422358512878,
      "sampling/importance_sampling_ratio/max": 1.8998442888259888,
      "sampling/importance_sampling_ratio/mean": 0.8834415674209595,
      "sampling/importance_sampling_ratio/min": 2.674500327100859e-19,
      "sampling/sampling_logp_difference/max": 22.5677433013916,
      "sampling/sampling_logp_difference/mean": 0.18694204092025757,
      "step": 295,
      "step_time": 13.752963833030662
    },
    {
      "clip_ratio/high_max": 0.012500000186264515,
      "clip_ratio/high_mean": 0.0062500000931322575,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0062500000931322575,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 608.0,
      "completions/max_terminated_length": 608.0,
      "completions/mean_length": 199.53125,
      "completions/mean_terminated_length": 192.35482788085938,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.49424443393945694,
      "epoch": 0.00592,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1772843599319458,
      "kl": 0.39571147598326206,
      "learning_rate": 8.44868181221277e-06,
      "loss": -0.2307,
      "num_tokens": 11895693.0,
      "reward": 0.7596874833106995,
      "reward_std": 0.4437393248081207,
      "rewards/rollout_reward_func/mean": 0.7596874833106995,
      "rewards/rollout_reward_func/std": 0.572242259979248,
      "sampling/importance_sampling_ratio/max": 2.0989577770233154,
      "sampling/importance_sampling_ratio/mean": 0.9883334636688232,
      "sampling/importance_sampling_ratio/min": 3.443121528293505e-21,
      "sampling/sampling_logp_difference/max": 27.186187744140625,
      "sampling/sampling_logp_difference/mean": 0.24693983793258667,
      "step": 296,
      "step_time": 12.729635896030231
    },
    {
      "clip_ratio/high_max": 0.029040751745924354,
      "clip_ratio/high_mean": 0.014520375872962177,
      "clip_ratio/low_mean": 0.0028409091755747795,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017361285048536956,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 633.0,
      "completions/max_terminated_length": 633.0,
      "completions/mean_length": 243.9375,
      "completions/mean_terminated_length": 237.90321350097656,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5392671348527074,
      "epoch": 0.00594,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5946135520935059,
      "kl": 0.35147016402333975,
      "learning_rate": 8.437634311055855e-06,
      "loss": 0.0342,
      "num_tokens": 11937385.0,
      "reward": 0.6168749928474426,
      "reward_std": 0.531964898109436,
      "rewards/rollout_reward_func/mean": 0.6168749928474426,
      "rewards/rollout_reward_func/std": 0.5537230968475342,
      "sampling/importance_sampling_ratio/max": 2.1955597400665283,
      "sampling/importance_sampling_ratio/mean": 0.9132822751998901,
      "sampling/importance_sampling_ratio/min": 1.9925900461414465e-20,
      "sampling/sampling_logp_difference/max": 23.483888626098633,
      "sampling/sampling_logp_difference/mean": 0.3714762330055237,
      "step": 297,
      "step_time": 12.462582045031013
    },
    {
      "clip_ratio/high_max": 0.006076388992369175,
      "clip_ratio/high_mean": 0.0030381944961845875,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0030381944961845875,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 168.625,
      "completions/mean_terminated_length": 140.85714721679688,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.43036549910902977,
      "epoch": 0.00596,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5387485027313232,
      "kl": 0.6057403041049838,
      "learning_rate": 8.42655795352353e-06,
      "loss": -0.1436,
      "num_tokens": 11974486.0,
      "reward": 0.7268749475479126,
      "reward_std": 0.43914729356765747,
      "rewards/rollout_reward_func/mean": 0.7268749475479126,
      "rewards/rollout_reward_func/std": 0.5884886980056763,
      "sampling/importance_sampling_ratio/max": 1.753420114517212,
      "sampling/importance_sampling_ratio/mean": 0.994858980178833,
      "sampling/importance_sampling_ratio/min": 0.13543827831745148,
      "sampling/sampling_logp_difference/max": 1.851804494857788,
      "sampling/sampling_logp_difference/mean": 0.04403349384665489,
      "step": 298,
      "step_time": 12.37503075299901
    },
    {
      "clip_ratio/high_max": 0.006623641354963183,
      "clip_ratio/high_mean": 0.0033118206774815917,
      "clip_ratio/low_mean": 0.0028409091755747795,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006152729853056371,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 678.0,
      "completions/max_terminated_length": 678.0,
      "completions/mean_length": 175.34375,
      "completions/mean_terminated_length": 159.48275756835938,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.42322571109980345,
      "epoch": 0.00598,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2773373126983643,
      "kl": 0.3985568732023239,
      "learning_rate": 8.415452885720438e-06,
      "loss": 0.0868,
      "num_tokens": 12012208.0,
      "reward": 0.6728124618530273,
      "reward_std": 0.6094969511032104,
      "rewards/rollout_reward_func/mean": 0.6728124618530273,
      "rewards/rollout_reward_func/std": 0.5946657061576843,
      "sampling/importance_sampling_ratio/max": 2.4114818572998047,
      "sampling/importance_sampling_ratio/mean": 0.9848952889442444,
      "sampling/importance_sampling_ratio/min": 0.3361913561820984,
      "sampling/sampling_logp_difference/max": 1.1043251752853394,
      "sampling/sampling_logp_difference/mean": 0.04464820399880409,
      "step": 299,
      "step_time": 12.451447289000498
    },
    {
      "clip_ratio/high_max": 0.009114583488553762,
      "clip_ratio/high_mean": 0.004557291744276881,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004557291744276881,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 650.0,
      "completions/mean_length": 277.875,
      "completions/mean_terminated_length": 268.8846130371094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4797269906848669,
      "epoch": 0.006,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6567587852478027,
      "kl": 0.4210485862568021,
      "learning_rate": 8.40431925412993e-06,
      "loss": 0.0149,
      "num_tokens": 12054090.0,
      "reward": 0.6418750286102295,
      "reward_std": 0.5393630862236023,
      "rewards/rollout_reward_func/mean": 0.6418750286102295,
      "rewards/rollout_reward_func/std": 0.5595703125,
      "sampling/importance_sampling_ratio/max": 2.2101855278015137,
      "sampling/importance_sampling_ratio/mean": 1.0303800106048584,
      "sampling/importance_sampling_ratio/min": 0.5338290333747864,
      "sampling/sampling_logp_difference/max": 0.6822850704193115,
      "sampling/sampling_logp_difference/mean": 0.037411026656627655,
      "step": 300,
      "step_time": 13.117839618003927
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 291.15625,
      "completions/mean_terminated_length": 283.8709716796875,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.41004679910838604,
      "epoch": 0.00602,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2725657224655151,
      "kl": 0.41305980179458857,
      "learning_rate": 8.393157205612137e-06,
      "loss": 0.0935,
      "num_tokens": 12097212.0,
      "reward": 0.7718749642372131,
      "reward_std": 0.3625251352787018,
      "rewards/rollout_reward_func/mean": 0.7718749642372131,
      "rewards/rollout_reward_func/std": 0.4305393099784851,
      "sampling/importance_sampling_ratio/max": 1.7940099239349365,
      "sampling/importance_sampling_ratio/mean": 0.9646265506744385,
      "sampling/importance_sampling_ratio/min": 0.32845696806907654,
      "sampling/sampling_logp_difference/max": 1.024425983428955,
      "sampling/sampling_logp_difference/mean": 0.041669879108667374,
      "step": 301,
      "step_time": 11.978105820002384
    },
    {
      "clip_ratio/high_max": 0.008446558145806193,
      "clip_ratio/high_mean": 0.004223279072903097,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004223279072903097,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 633.0,
      "completions/max_terminated_length": 633.0,
      "completions/mean_length": 262.09375,
      "completions/mean_terminated_length": 253.45159912109375,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.4401724115014076,
      "epoch": 0.00604,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3799006938934326,
      "kl": 0.9543331759050488,
      "learning_rate": 8.38196688740202e-06,
      "loss": -0.2064,
      "num_tokens": 12139322.0,
      "reward": 0.5787523984909058,
      "reward_std": 0.6598265171051025,
      "rewards/rollout_reward_func/mean": 0.5787523984909058,
      "rewards/rollout_reward_func/std": 0.6854074001312256,
      "sampling/importance_sampling_ratio/max": 2.697169780731201,
      "sampling/importance_sampling_ratio/mean": 0.9825540781021118,
      "sampling/importance_sampling_ratio/min": 9.760039705133632e-17,
      "sampling/sampling_logp_difference/max": 20.93475914001465,
      "sampling/sampling_logp_difference/mean": 0.14610826969146729,
      "step": 302,
      "step_time": 12.813516806971165
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0029296875,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 648.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 232.59375,
      "completions/mean_terminated_length": 220.07144165039062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.45259237941354513,
      "epoch": 0.00606,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3013170957565308,
      "kl": 1.7604645323008299,
      "learning_rate": 8.370748447107445e-06,
      "loss": -0.0746,
      "num_tokens": 12180623.0,
      "reward": 0.513124942779541,
      "reward_std": 0.5873017311096191,
      "rewards/rollout_reward_func/mean": 0.513124942779541,
      "rewards/rollout_reward_func/std": 0.6132432818412781,
      "sampling/importance_sampling_ratio/max": 1.6829348802566528,
      "sampling/importance_sampling_ratio/mean": 1.0074145793914795,
      "sampling/importance_sampling_ratio/min": 0.23248159885406494,
      "sampling/sampling_logp_difference/max": 1.0323596000671387,
      "sampling/sampling_logp_difference/mean": 0.050939545035362244,
      "step": 303,
      "step_time": 12.35252166405553
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 1112.0,
      "completions/max_terminated_length": 1112.0,
      "completions/mean_length": 261.53125,
      "completions/mean_terminated_length": 272.1851806640625,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4794614724814892,
      "epoch": 0.00608,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9820563197135925,
      "kl": 0.36218645982444286,
      "learning_rate": 8.359502032707219e-06,
      "loss": -0.1281,
      "num_tokens": 12223147.0,
      "reward": 0.6409374475479126,
      "reward_std": 0.6450225114822388,
      "rewards/rollout_reward_func/mean": 0.6409374475479126,
      "rewards/rollout_reward_func/std": 0.6703362464904785,
      "sampling/importance_sampling_ratio/max": 1.725956916809082,
      "sampling/importance_sampling_ratio/mean": 0.989924430847168,
      "sampling/importance_sampling_ratio/min": 0.3860521912574768,
      "sampling/sampling_logp_difference/max": 0.8404061794281006,
      "sampling/sampling_logp_difference/mean": 0.044010426849126816,
      "step": 304,
      "step_time": 14.089848739968147
    },
    {
      "clip_ratio/high_max": 0.01616071444004774,
      "clip_ratio/high_mean": 0.00808035722002387,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01003348222002387,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1060.0,
      "completions/max_terminated_length": 1060.0,
      "completions/mean_length": 236.9375,
      "completions/mean_terminated_length": 241.82144165039062,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.44224483147263527,
      "epoch": 0.0061,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1624045372009277,
      "kl": 0.3629315849393606,
      "learning_rate": 8.34822779254915e-06,
      "loss": -0.018,
      "num_tokens": 12263236.0,
      "reward": 0.8531249761581421,
      "reward_std": 0.32334309816360474,
      "rewards/rollout_reward_func/mean": 0.8531249761581421,
      "rewards/rollout_reward_func/std": 0.3674975037574768,
      "sampling/importance_sampling_ratio/max": 1.8425886631011963,
      "sampling/importance_sampling_ratio/mean": 0.9315986037254333,
      "sampling/importance_sampling_ratio/min": 6.739338897234526e-17,
      "sampling/sampling_logp_difference/max": 20.04157066345215,
      "sampling/sampling_logp_difference/mean": 0.11154128611087799,
      "step": 305,
      "step_time": 13.601251635991503
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 195.0,
      "completions/mean_terminated_length": 195.10345458984375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.39774864353239536,
      "epoch": 0.00612,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.759473204612732,
      "kl": 1.6839289767667651,
      "learning_rate": 8.336925875348093e-06,
      "loss": -0.0759,
      "num_tokens": 12302010.0,
      "reward": 0.6981250047683716,
      "reward_std": 0.4118298292160034,
      "rewards/rollout_reward_func/mean": 0.6981250047683716,
      "rewards/rollout_reward_func/std": 0.5400683283805847,
      "sampling/importance_sampling_ratio/max": 2.0755412578582764,
      "sampling/importance_sampling_ratio/mean": 1.017045497894287,
      "sampling/importance_sampling_ratio/min": 0.2873917520046234,
      "sampling/sampling_logp_difference/max": 1.223393440246582,
      "sampling/sampling_logp_difference/mean": 0.041064973920583725,
      "step": 306,
      "step_time": 12.357795086005353
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 210.8125,
      "completions/mean_terminated_length": 199.35482788085938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4549230933189392,
      "epoch": 0.00614,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.675602376461029,
      "kl": 0.5227059023454785,
      "learning_rate": 8.32559643018397e-06,
      "loss": 0.0289,
      "num_tokens": 12341682.0,
      "reward": 0.8006249666213989,
      "reward_std": 0.4737958610057831,
      "rewards/rollout_reward_func/mean": 0.8006249666213989,
      "rewards/rollout_reward_func/std": 0.4852530360221863,
      "sampling/importance_sampling_ratio/max": 1.9331941604614258,
      "sampling/importance_sampling_ratio/mean": 0.8052453994750977,
      "sampling/importance_sampling_ratio/min": 5.17212701892743e-22,
      "sampling/sampling_logp_difference/max": 29.31505584716797,
      "sampling/sampling_logp_difference/mean": 0.4101036489009857,
      "step": 307,
      "step_time": 12.646711532041081
    },
    {
      "clip_ratio/high_max": 0.007305195089429617,
      "clip_ratio/high_mean": 0.0036525975447148085,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036525975447148085,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 279.78125,
      "completions/mean_terminated_length": 271.17242431640625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.46677009761333466,
      "epoch": 0.00616,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.297461986541748,
      "kl": 0.9498905269429088,
      "learning_rate": 8.314239606499828e-06,
      "loss": 0.0332,
      "num_tokens": 12383747.0,
      "reward": 0.6731876730918884,
      "reward_std": 0.6228747367858887,
      "rewards/rollout_reward_func/mean": 0.6731876730918884,
      "rewards/rollout_reward_func/std": 0.6146537661552429,
      "sampling/importance_sampling_ratio/max": 2.1273674964904785,
      "sampling/importance_sampling_ratio/mean": 0.9714600443840027,
      "sampling/importance_sampling_ratio/min": 2.2245643234398635e-14,
      "sampling/sampling_logp_difference/max": 23.602783203125,
      "sampling/sampling_logp_difference/mean": 0.10591796785593033,
      "step": 308,
      "step_time": 12.368246132027707
    },
    {
      "clip_ratio/high_max": 0.023437500232830644,
      "clip_ratio/high_mean": 0.011718750116415322,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011718750116415322,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 626.0,
      "completions/max_terminated_length": 626.0,
      "completions/mean_length": 217.625,
      "completions/mean_terminated_length": 212.6333465576172,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4329261798411608,
      "epoch": 0.00618,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0076905488967896,
      "kl": 1.307756402529776,
      "learning_rate": 8.302855554099842e-06,
      "loss": -0.2595,
      "num_tokens": 12423566.0,
      "reward": 0.5746874809265137,
      "reward_std": 0.5835037231445312,
      "rewards/rollout_reward_func/mean": 0.5746874809265137,
      "rewards/rollout_reward_func/std": 0.6878108382225037,
      "sampling/importance_sampling_ratio/max": 1.5494205951690674,
      "sampling/importance_sampling_ratio/mean": 0.8924027681350708,
      "sampling/importance_sampling_ratio/min": 6.903549660819408e-07,
      "sampling/sampling_logp_difference/max": 3.49668025970459,
      "sampling/sampling_logp_difference/mean": 0.08327680081129074,
      "step": 309,
      "step_time": 12.413042257001507
    },
    {
      "clip_ratio/high_max": 0.007938507944345474,
      "clip_ratio/high_mean": 0.003969253972172737,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003969253972172737,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 612.0,
      "completions/max_terminated_length": 596.0,
      "completions/mean_length": 282.71875,
      "completions/mean_terminated_length": 291.67999267578125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.48805269226431847,
      "epoch": 0.0062,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1841163635253906,
      "kl": 0.3829494221135974,
      "learning_rate": 8.29144442314736e-06,
      "loss": -0.1551,
      "num_tokens": 12467049.0,
      "reward": 0.8662499189376831,
      "reward_std": 0.3794723153114319,
      "rewards/rollout_reward_func/mean": 0.8662499189376831,
      "rewards/rollout_reward_func/std": 0.4603346884250641,
      "sampling/importance_sampling_ratio/max": 2.1256747245788574,
      "sampling/importance_sampling_ratio/mean": 1.0479602813720703,
      "sampling/importance_sampling_ratio/min": 0.20649801194667816,
      "sampling/sampling_logp_difference/max": 0.7095634937286377,
      "sampling/sampling_logp_difference/mean": 0.04733724519610405,
      "step": 310,
      "step_time": 13.279909269011114
    },
    {
      "clip_ratio/high_max": 0.018880208721384406,
      "clip_ratio/high_mean": 0.009440104360692203,
      "clip_ratio/low_mean": 0.0016447368543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01108484121505171,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 624.0,
      "completions/max_terminated_length": 624.0,
      "completions/mean_length": 211.53125,
      "completions/mean_terminated_length": 209.29629516601562,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.48539761640131474,
      "epoch": 0.00622,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8885459899902344,
      "kl": 0.5460547590628266,
      "learning_rate": 8.280006364162915e-06,
      "loss": 0.0105,
      "num_tokens": 12507438.0,
      "reward": 0.6443749666213989,
      "reward_std": 0.6219362020492554,
      "rewards/rollout_reward_func/mean": 0.6443749666213989,
      "rewards/rollout_reward_func/std": 0.6103353500366211,
      "sampling/importance_sampling_ratio/max": 2.37459397315979,
      "sampling/importance_sampling_ratio/mean": 1.0112807750701904,
      "sampling/importance_sampling_ratio/min": 0.2874722182750702,
      "sampling/sampling_logp_difference/max": 0.9583673477172852,
      "sampling/sampling_logp_difference/mean": 0.05421842634677887,
      "step": 311,
      "step_time": 12.089739246002864
    },
    {
      "clip_ratio/high_max": 0.04166666674427688,
      "clip_ratio/high_mean": 0.02083333337213844,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022671568673104048,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 316.46875,
      "completions/mean_terminated_length": 336.66668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5021736170165241,
      "epoch": 0.00624,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0980348587036133,
      "kl": 0.40064464835450053,
      "learning_rate": 8.268541528022238e-06,
      "loss": -0.079,
      "num_tokens": 12550992.0,
      "reward": 0.7049999833106995,
      "reward_std": 0.5506595373153687,
      "rewards/rollout_reward_func/mean": 0.7049999833106995,
      "rewards/rollout_reward_func/std": 0.5355250239372253,
      "sampling/importance_sampling_ratio/max": 2.429114580154419,
      "sampling/importance_sampling_ratio/mean": 0.9549639821052551,
      "sampling/importance_sampling_ratio/min": 0.3262085020542145,
      "sampling/sampling_logp_difference/max": 0.8619797229766846,
      "sampling/sampling_logp_difference/mean": 0.04913249611854553,
      "step": 312,
      "step_time": 12.959745342013775
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 300.53125,
      "completions/mean_terminated_length": 313.1111145019531,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4075745716691017,
      "epoch": 0.00626,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.988919734954834,
      "kl": 0.3783861342817545,
      "learning_rate": 8.257050065954267e-06,
      "loss": -0.0466,
      "num_tokens": 12594355.0,
      "reward": 0.6209374666213989,
      "reward_std": 0.5192592144012451,
      "rewards/rollout_reward_func/mean": 0.6209374666213989,
      "rewards/rollout_reward_func/std": 0.6223595142364502,
      "sampling/importance_sampling_ratio/max": 2.2695531845092773,
      "sampling/importance_sampling_ratio/mean": 1.086808681488037,
      "sampling/importance_sampling_ratio/min": 0.29240745306015015,
      "sampling/sampling_logp_difference/max": 0.701533317565918,
      "sampling/sampling_logp_difference/mean": 0.04209626466035843,
      "step": 313,
      "step_time": 12.672739492962137
    },
    {
      "clip_ratio/high_max": 0.010977095691487193,
      "clip_ratio/high_mean": 0.0054885478457435966,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0054885478457435966,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 706.0,
      "completions/max_terminated_length": 706.0,
      "completions/mean_length": 182.625,
      "completions/mean_terminated_length": 164.0,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4719371823593974,
      "epoch": 0.00628,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5045700073242188,
      "kl": 0.5258753998205066,
      "learning_rate": 8.245532129539153e-06,
      "loss": -0.1357,
      "num_tokens": 12632368.0,
      "reward": 0.7206249833106995,
      "reward_std": 0.6073609590530396,
      "rewards/rollout_reward_func/mean": 0.7206249833106995,
      "rewards/rollout_reward_func/std": 0.68485426902771,
      "sampling/importance_sampling_ratio/max": 1.6636468172073364,
      "sampling/importance_sampling_ratio/mean": 0.9282735586166382,
      "sampling/importance_sampling_ratio/min": 1.710181537727893e-20,
      "sampling/sampling_logp_difference/max": 21.906564712524414,
      "sampling/sampling_logp_difference/mean": 0.13666628301143646,
      "step": 314,
      "step_time": 12.93943730101455
    },
    {
      "clip_ratio/high_max": 0.017968750093132257,
      "clip_ratio/high_mean": 0.008984375046566129,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008984375046566129,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 914.0,
      "completions/max_terminated_length": 914.0,
      "completions/mean_length": 328.0,
      "completions/mean_terminated_length": 330.85186767578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5200985418632627,
      "epoch": 0.0063,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2459323406219482,
      "kl": 0.4967865743674338,
      "learning_rate": 8.233987870706267e-06,
      "loss": 0.0417,
      "num_tokens": 12676705.0,
      "reward": 0.74022376537323,
      "reward_std": 0.48467469215393066,
      "rewards/rollout_reward_func/mean": 0.74022376537323,
      "rewards/rollout_reward_func/std": 0.6043890118598938,
      "sampling/importance_sampling_ratio/max": 1.884284257888794,
      "sampling/importance_sampling_ratio/mean": 0.8911020159721375,
      "sampling/importance_sampling_ratio/min": 2.1569959360312474e-12,
      "sampling/sampling_logp_difference/max": 19.190519332885742,
      "sampling/sampling_logp_difference/mean": 0.09494476020336151,
      "step": 315,
      "step_time": 13.56004562297312
    },
    {
      "clip_ratio/high_max": 0.009148848708719015,
      "clip_ratio/high_mean": 0.0045744243543595076,
      "clip_ratio/low_mean": 0.0032087054569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007783129811286926,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 284.28125,
      "completions/mean_terminated_length": 238.69232177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5479425247758627,
      "epoch": 0.00632,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.885845184326172,
      "kl": 0.4120043022558093,
      "learning_rate": 8.222417441732182e-06,
      "loss": 0.1544,
      "num_tokens": 12718761.0,
      "reward": 0.5784779787063599,
      "reward_std": 0.7187098860740662,
      "rewards/rollout_reward_func/mean": 0.5784779787063599,
      "rewards/rollout_reward_func/std": 0.7259992957115173,
      "sampling/importance_sampling_ratio/max": 1.5845986604690552,
      "sampling/importance_sampling_ratio/mean": 0.9385649561882019,
      "sampling/importance_sampling_ratio/min": 0.3212519884109497,
      "sampling/sampling_logp_difference/max": 0.8817768096923828,
      "sampling/sampling_logp_difference/mean": 0.04880259931087494,
      "step": 316,
      "step_time": 13.156197525997413
    },
    {
      "clip_ratio/high_max": 0.003969253972172737,
      "clip_ratio/high_mean": 0.0019846269860863686,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0019846269860863686,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 206.15625,
      "completions/mean_terminated_length": 173.86956787109375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4868337493389845,
      "epoch": 0.00634,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4489014148712158,
      "kl": 0.4647271204739809,
      "learning_rate": 8.210820995238682e-06,
      "loss": -0.1493,
      "num_tokens": 12758752.0,
      "reward": 0.6024999618530273,
      "reward_std": 0.5641997456550598,
      "rewards/rollout_reward_func/mean": 0.6024999618530273,
      "rewards/rollout_reward_func/std": 0.667044460773468,
      "sampling/importance_sampling_ratio/max": 2.280165433883667,
      "sampling/importance_sampling_ratio/mean": 1.1145942211151123,
      "sampling/importance_sampling_ratio/min": 0.32093214988708496,
      "sampling/sampling_logp_difference/max": 1.153285026550293,
      "sampling/sampling_logp_difference/mean": 0.05344346910715103,
      "step": 317,
      "step_time": 12.538303523993818
    },
    {
      "clip_ratio/high_max": 0.01226551248691976,
      "clip_ratio/high_mean": 0.00613275624345988,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00613275624345988,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 671.0,
      "completions/max_terminated_length": 671.0,
      "completions/mean_length": 248.5625,
      "completions/mean_terminated_length": 208.46429443359375,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.5276377536356449,
      "epoch": 0.00636,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4128156900405884,
      "kl": 0.5881783775985241,
      "learning_rate": 8.199198684190737e-06,
      "loss": -0.1195,
      "num_tokens": 12799089.0,
      "reward": 0.6916420459747314,
      "reward_std": 0.49899545311927795,
      "rewards/rollout_reward_func/mean": 0.6916420459747314,
      "rewards/rollout_reward_func/std": 0.6701527833938599,
      "sampling/importance_sampling_ratio/max": 2.083484172821045,
      "sampling/importance_sampling_ratio/mean": 0.9502729177474976,
      "sampling/importance_sampling_ratio/min": 0.40541666746139526,
      "sampling/sampling_logp_difference/max": 1.1783814430236816,
      "sampling/sampling_logp_difference/mean": 0.052470359951257706,
      "step": 318,
      "step_time": 12.314195166007266
    },
    {
      "clip_ratio/high_max": 0.008091517956927419,
      "clip_ratio/high_mean": 0.004045758978463709,
      "clip_ratio/low_mean": 0.0012499999720603228,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005295758950524032,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 663.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 329.0,
      "completions/mean_terminated_length": 287.7200012207031,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4777907347306609,
      "epoch": 0.00638,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2132855653762817,
      "kl": 0.7144772158935666,
      "learning_rate": 8.187550661894482e-06,
      "loss": 0.0237,
      "num_tokens": 12842698.0,
      "reward": 0.5512499809265137,
      "reward_std": 0.578349232673645,
      "rewards/rollout_reward_func/mean": 0.5512499809265137,
      "rewards/rollout_reward_func/std": 0.6356644630432129,
      "sampling/importance_sampling_ratio/max": 2.1389241218566895,
      "sampling/importance_sampling_ratio/mean": 0.950953483581543,
      "sampling/importance_sampling_ratio/min": 7.674524824307619e-23,
      "sampling/sampling_logp_difference/max": 21.5936336517334,
      "sampling/sampling_logp_difference/mean": 0.15326346457004547,
      "step": 319,
      "step_time": 13.086154939039261
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 630.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 254.46875,
      "completions/mean_terminated_length": 288.5357360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4512854628264904,
      "epoch": 0.0064,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2210794687271118,
      "kl": 0.4919296381995082,
      "learning_rate": 8.175877081995205e-06,
      "loss": -0.0467,
      "num_tokens": 12884110.0,
      "reward": 0.5174999833106995,
      "reward_std": 0.7437063455581665,
      "rewards/rollout_reward_func/mean": 0.5174999833106995,
      "rewards/rollout_reward_func/std": 0.7780081629753113,
      "sampling/importance_sampling_ratio/max": 1.5201157331466675,
      "sampling/importance_sampling_ratio/mean": 1.0476715564727783,
      "sampling/importance_sampling_ratio/min": 0.3093971312046051,
      "sampling/sampling_logp_difference/max": 1.2064003944396973,
      "sampling/sampling_logp_difference/mean": 0.042752400040626526,
      "step": 320,
      "step_time": 12.523057512036758
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 630.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 242.65625,
      "completions/mean_terminated_length": 224.11538696289062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4669450353831053,
      "epoch": 0.00642,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2182490825653076,
      "kl": 0.4643328273668885,
      "learning_rate": 8.16417809847532e-06,
      "loss": 0.0355,
      "num_tokens": 12923702.0,
      "reward": 0.7887904644012451,
      "reward_std": 0.41385993361473083,
      "rewards/rollout_reward_func/mean": 0.7887904644012451,
      "rewards/rollout_reward_func/std": 0.4850296080112457,
      "sampling/importance_sampling_ratio/max": 1.8062011003494263,
      "sampling/importance_sampling_ratio/mean": 1.0477699041366577,
      "sampling/importance_sampling_ratio/min": 0.3581528961658478,
      "sampling/sampling_logp_difference/max": 0.8533978462219238,
      "sampling/sampling_logp_difference/mean": 0.040005117654800415,
      "step": 321,
      "step_time": 12.160757052988629
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 638.0,
      "completions/max_terminated_length": 638.0,
      "completions/mean_length": 220.1875,
      "completions/mean_terminated_length": 228.42308044433594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5314071429893374,
      "epoch": 0.00644,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.779200553894043,
      "kl": 0.37780622066929936,
      "learning_rate": 8.152453865652327e-06,
      "loss": -0.0476,
      "num_tokens": 12962408.0,
      "reward": 0.6024999618530273,
      "reward_std": 0.7235965728759766,
      "rewards/rollout_reward_func/mean": 0.6024999618530273,
      "rewards/rollout_reward_func/std": 0.7043459415435791,
      "sampling/importance_sampling_ratio/max": 1.753944993019104,
      "sampling/importance_sampling_ratio/mean": 0.9928716421127319,
      "sampling/importance_sampling_ratio/min": 0.45911210775375366,
      "sampling/sampling_logp_difference/max": 0.8572278022766113,
      "sampling/sampling_logp_difference/mean": 0.04557346552610397,
      "step": 322,
      "step_time": 12.433551878028084
    },
    {
      "clip_ratio/high_max": 0.0069444444961845875,
      "clip_ratio/high_mean": 0.0034722222480922937,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0034722222480922937,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 648.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 220.4375,
      "completions/mean_terminated_length": 241.03448486328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.44986592046916485,
      "epoch": 0.00646,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7823982834815979,
      "kl": 0.3706073872745037,
      "learning_rate": 8.140704538176782e-06,
      "loss": -0.1752,
      "num_tokens": 13001870.0,
      "reward": 0.7893749475479126,
      "reward_std": 0.5096417665481567,
      "rewards/rollout_reward_func/mean": 0.7893749475479126,
      "rewards/rollout_reward_func/std": 0.6260626316070557,
      "sampling/importance_sampling_ratio/max": 1.8918007612228394,
      "sampling/importance_sampling_ratio/mean": 0.928809642791748,
      "sampling/importance_sampling_ratio/min": 9.868101533457097e-21,
      "sampling/sampling_logp_difference/max": 23.152816772460938,
      "sampling/sampling_logp_difference/mean": 0.23890507221221924,
      "step": 323,
      "step_time": 12.235214308006107
    },
    {
      "clip_ratio/high_max": 0.015690267784520984,
      "clip_ratio/high_mean": 0.007845133892260492,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008821696392260492,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 685.0,
      "completions/max_terminated_length": 685.0,
      "completions/mean_length": 256.625,
      "completions/mean_terminated_length": 277.96295166015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5487983021885157,
      "epoch": 0.00648,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1234714984893799,
      "kl": 0.3936740206554532,
      "learning_rate": 8.128930271030256e-06,
      "loss": -0.0139,
      "num_tokens": 13042428.0,
      "reward": 0.5696874856948853,
      "reward_std": 0.6264265179634094,
      "rewards/rollout_reward_func/mean": 0.5696874856948853,
      "rewards/rollout_reward_func/std": 0.6191433668136597,
      "sampling/importance_sampling_ratio/max": 1.760693073272705,
      "sampling/importance_sampling_ratio/mean": 0.9263312220573425,
      "sampling/importance_sampling_ratio/min": 3.5716294688653554e-22,
      "sampling/sampling_logp_difference/max": 27.121294021606445,
      "sampling/sampling_logp_difference/mean": 0.3610017001628876,
      "step": 324,
      "step_time": 12.82673529900785
    },
    {
      "clip_ratio/high_max": 0.022385817486792803,
      "clip_ratio/high_mean": 0.011192908743396401,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012169471243396401,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 797.0,
      "completions/max_terminated_length": 797.0,
      "completions/mean_length": 249.84375,
      "completions/mean_terminated_length": 225.7777862548828,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4246586859226227,
      "epoch": 0.0065,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0657110214233398,
      "kl": 0.4817621661350131,
      "learning_rate": 8.117131219523293e-06,
      "loss": 0.0993,
      "num_tokens": 13083278.0,
      "reward": 0.6341251730918884,
      "reward_std": 0.6255955696105957,
      "rewards/rollout_reward_func/mean": 0.6341251730918884,
      "rewards/rollout_reward_func/std": 0.670932948589325,
      "sampling/importance_sampling_ratio/max": 2.329777240753174,
      "sampling/importance_sampling_ratio/mean": 0.9788886308670044,
      "sampling/importance_sampling_ratio/min": 0.3121226131916046,
      "sampling/sampling_logp_difference/max": 0.8605947494506836,
      "sampling/sampling_logp_difference/mean": 0.0505315437912941,
      "step": 325,
      "step_time": 12.485990647022845
    },
    {
      "clip_ratio/high_max": 0.012500000186264515,
      "clip_ratio/high_mean": 0.0062500000931322575,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0062500000931322575,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 1465.0,
      "completions/max_terminated_length": 1465.0,
      "completions/mean_length": 249.78125,
      "completions/mean_terminated_length": 228.88888549804688,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.3968965457752347,
      "epoch": 0.00652,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7287344336509705,
      "kl": 0.852862142957747,
      "learning_rate": 8.10530753929336e-06,
      "loss": -0.0289,
      "num_tokens": 13123831.0,
      "reward": 0.5946797728538513,
      "reward_std": 0.7397448420524597,
      "rewards/rollout_reward_func/mean": 0.5946797728538513,
      "rewards/rollout_reward_func/std": 0.7509238719940186,
      "sampling/importance_sampling_ratio/max": 1.5245009660720825,
      "sampling/importance_sampling_ratio/mean": 0.8765004873275757,
      "sampling/importance_sampling_ratio/min": 5.14707966282293e-19,
      "sampling/sampling_logp_difference/max": 26.05772590637207,
      "sampling/sampling_logp_difference/mean": 0.22768273949623108,
      "step": 326,
      "step_time": 16.17781132698292
    },
    {
      "clip_ratio/high_max": 0.006770833628252149,
      "clip_ratio/high_mean": 0.0033854168141260743,
      "clip_ratio/low_mean": 0.0012499999720603228,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004635416786186397,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 274.28125,
      "completions/mean_terminated_length": 270.96295166015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4589244741946459,
      "epoch": 0.00654,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.229943871498108,
      "kl": 0.315511760301888,
      "learning_rate": 8.093459386302788e-06,
      "loss": -0.0493,
      "num_tokens": 13166820.0,
      "reward": 0.6114737391471863,
      "reward_std": 0.6898340582847595,
      "rewards/rollout_reward_func/mean": 0.6114737391471863,
      "rewards/rollout_reward_func/std": 0.6639074087142944,
      "sampling/importance_sampling_ratio/max": 1.9025462865829468,
      "sampling/importance_sampling_ratio/mean": 1.019953727722168,
      "sampling/importance_sampling_ratio/min": 0.43675196170806885,
      "sampling/sampling_logp_difference/max": 0.7609531879425049,
      "sampling/sampling_logp_difference/mean": 0.041474536061286926,
      "step": 327,
      "step_time": 12.809381528990343
    },
    {
      "clip_ratio/high_max": 0.04226190596818924,
      "clip_ratio/high_mean": 0.02113095298409462,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02113095298409462,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 1195.0,
      "completions/max_terminated_length": 1195.0,
      "completions/mean_length": 358.8125,
      "completions/mean_terminated_length": 386.8888854980469,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.42317097820341587,
      "epoch": 0.00656,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0074546337127686,
      "kl": 0.6459536598995328,
      "learning_rate": 8.081586916836728e-06,
      "loss": -0.1398,
      "num_tokens": 13211512.0,
      "reward": 0.5057021379470825,
      "reward_std": 0.7011765241622925,
      "rewards/rollout_reward_func/mean": 0.5057021379470825,
      "rewards/rollout_reward_func/std": 0.7330716848373413,
      "sampling/importance_sampling_ratio/max": 1.7379062175750732,
      "sampling/importance_sampling_ratio/mean": 0.9455592632293701,
      "sampling/importance_sampling_ratio/min": 0.2641665041446686,
      "sampling/sampling_logp_difference/max": 1.4810419082641602,
      "sampling/sampling_logp_difference/mean": 0.04713539779186249,
      "step": 328,
      "step_time": 14.610014393969323
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0087890625,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 242.34375,
      "completions/mean_terminated_length": 208.2413787841797,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4207946192473173,
      "epoch": 0.00658,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1630207300186157,
      "kl": 0.44007748924195766,
      "learning_rate": 8.069690287501078e-06,
      "loss": 0.1132,
      "num_tokens": 13251469.0,
      "reward": 0.762499988079071,
      "reward_std": 0.502318263053894,
      "rewards/rollout_reward_func/mean": 0.762499988079071,
      "rewards/rollout_reward_func/std": 0.5258970260620117,
      "sampling/importance_sampling_ratio/max": 1.912102460861206,
      "sampling/importance_sampling_ratio/mean": 1.0678372383117676,
      "sampling/importance_sampling_ratio/min": 4.1480432977358817e-10,
      "sampling/sampling_logp_difference/max": 21.48827362060547,
      "sampling/sampling_logp_difference/mean": 0.08698268234729767,
      "step": 329,
      "step_time": 12.56753756699618
    },
    {
      "clip_ratio/high_max": 0.03751860139891505,
      "clip_ratio/high_mean": 0.018759300699457526,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018759300699457526,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 634.0,
      "completions/max_terminated_length": 634.0,
      "completions/mean_length": 189.90625,
      "completions/mean_terminated_length": 189.74073791503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.47680601850152016,
      "epoch": 0.0066,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.289692997932434,
      "kl": 0.40974822640419006,
      "learning_rate": 8.05776965522042e-06,
      "loss": -0.1547,
      "num_tokens": 13290517.0,
      "reward": 0.5406655073165894,
      "reward_std": 0.7139931321144104,
      "rewards/rollout_reward_func/mean": 0.5406655073165894,
      "rewards/rollout_reward_func/std": 0.769454300403595,
      "sampling/importance_sampling_ratio/max": 1.8910157680511475,
      "sampling/importance_sampling_ratio/mean": 0.9979549050331116,
      "sampling/importance_sampling_ratio/min": 3.916849011159747e-23,
      "sampling/sampling_logp_difference/max": 23.99772834777832,
      "sampling/sampling_logp_difference/mean": 0.22136516869068146,
      "step": 330,
      "step_time": 12.57919114001561
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 647.0,
      "completions/mean_length": 230.8125,
      "completions/mean_terminated_length": 229.1290283203125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.38341217767447233,
      "epoch": 0.00662,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7635312676429749,
      "kl": 0.7050189580768347,
      "learning_rate": 8.045825177235952e-06,
      "loss": -0.0899,
      "num_tokens": 13330766.0,
      "reward": 0.6703149080276489,
      "reward_std": 0.6167160272598267,
      "rewards/rollout_reward_func/mean": 0.6703149080276489,
      "rewards/rollout_reward_func/std": 0.6146390438079834,
      "sampling/importance_sampling_ratio/max": 1.8133646249771118,
      "sampling/importance_sampling_ratio/mean": 1.0295659303665161,
      "sampling/importance_sampling_ratio/min": 0.2541066110134125,
      "sampling/sampling_logp_difference/max": 0.9520206451416016,
      "sampling/sampling_logp_difference/mean": 0.038452841341495514,
      "step": 331,
      "step_time": 12.283926708987565
    },
    {
      "clip_ratio/high_max": 0.078125,
      "clip_ratio/high_mean": 0.04531249962747097,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.06093749962747097,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 824.0,
      "completions/max_terminated_length": 824.0,
      "completions/mean_length": 187.03125,
      "completions/mean_terminated_length": 192.40000915527344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.37285716040059924,
      "epoch": 0.00664,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.834619402885437,
      "kl": 1.0165564129129052,
      "learning_rate": 8.033857011103411e-06,
      "loss": 0.0678,
      "num_tokens": 13368140.0,
      "reward": 0.6893749833106995,
      "reward_std": 0.601416826248169,
      "rewards/rollout_reward_func/mean": 0.6893749833106995,
      "rewards/rollout_reward_func/std": 0.6394702196121216,
      "sampling/importance_sampling_ratio/max": 1.6978381872177124,
      "sampling/importance_sampling_ratio/mean": 0.8242970705032349,
      "sampling/importance_sampling_ratio/min": 1.4455785637509866e-18,
      "sampling/sampling_logp_difference/max": 26.97461700439453,
      "sampling/sampling_logp_difference/mean": 0.2503422498703003,
      "step": 332,
      "step_time": 12.001920457027154
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.007393973413854837,
      "clip_ratio/low_min": 0.001953125,
      "clip_ratio/region_mean": 0.009347098413854837,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 264.875,
      "completions/mean_terminated_length": 246.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3999841441400349,
      "epoch": 0.00666,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1382099390029907,
      "kl": 0.4344940120354295,
      "learning_rate": 8.021865314691006e-06,
      "loss": 0.011,
      "num_tokens": 13409624.0,
      "reward": 0.6725000143051147,
      "reward_std": 0.6535829305648804,
      "rewards/rollout_reward_func/mean": 0.6725000143051147,
      "rewards/rollout_reward_func/std": 0.6658634543418884,
      "sampling/importance_sampling_ratio/max": 2.0061657428741455,
      "sampling/importance_sampling_ratio/mean": 0.9700638055801392,
      "sampling/importance_sampling_ratio/min": 0.2835310697555542,
      "sampling/sampling_logp_difference/max": 1.289806604385376,
      "sampling/sampling_logp_difference/mean": 0.04202311113476753,
      "step": 333,
      "step_time": 12.866312921018107
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0029296875,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0029296875,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 661.0,
      "completions/max_terminated_length": 661.0,
      "completions/mean_length": 224.75,
      "completions/mean_terminated_length": 223.27999877929688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4809950962662697,
      "epoch": 0.00668,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.1846678256988525,
      "kl": 1.7700256649404764,
      "learning_rate": 8.009850246177311e-06,
      "loss": -0.0413,
      "num_tokens": 13449928.0,
      "reward": 0.6868749856948853,
      "reward_std": 0.5709550380706787,
      "rewards/rollout_reward_func/mean": 0.6868749856948853,
      "rewards/rollout_reward_func/std": 0.6659528017044067,
      "sampling/importance_sampling_ratio/max": 2.602832555770874,
      "sampling/importance_sampling_ratio/mean": 1.055869221687317,
      "sampling/importance_sampling_ratio/min": 0.36023372411727905,
      "sampling/sampling_logp_difference/max": 1.1968307495117188,
      "sampling/sampling_logp_difference/mean": 0.04637528955936432,
      "step": 334,
      "step_time": 12.647785951994592
    },
    {
      "clip_ratio/high_max": 0.0044531249441206455,
      "clip_ratio/high_mean": 0.0022265624720603228,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022265624720603228,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 608.0,
      "completions/max_terminated_length": 608.0,
      "completions/mean_length": 217.125,
      "completions/mean_terminated_length": 194.25001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.35296088363975286,
      "epoch": 0.0067,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8164500594139099,
      "kl": 0.4999607661738992,
      "learning_rate": 7.997811964049207e-06,
      "loss": 0.0587,
      "num_tokens": 13487811.0,
      "reward": 0.7956249713897705,
      "reward_std": 0.4721258878707886,
      "rewards/rollout_reward_func/mean": 0.7956249713897705,
      "rewards/rollout_reward_func/std": 0.48809659481048584,
      "sampling/importance_sampling_ratio/max": 1.664133071899414,
      "sampling/importance_sampling_ratio/mean": 1.003755807876587,
      "sampling/importance_sampling_ratio/min": 0.24740450084209442,
      "sampling/sampling_logp_difference/max": 1.6397833824157715,
      "sampling/sampling_logp_difference/mean": 0.04113231599330902,
      "step": 335,
      "step_time": 11.788996178016532
    },
    {
      "clip_ratio/high_max": 0.0028409091755747795,
      "clip_ratio/high_mean": 0.0014204545877873898,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0033735795877873898,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 248.65625,
      "completions/mean_terminated_length": 266.77777099609375,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4224699940532446,
      "epoch": 0.00672,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1539974212646484,
      "kl": 0.63172865845263,
      "learning_rate": 7.98575062709977e-06,
      "loss": -0.0296,
      "num_tokens": 13529693.0,
      "reward": 0.579125165939331,
      "reward_std": 0.6435240507125854,
      "rewards/rollout_reward_func/mean": 0.579125165939331,
      "rewards/rollout_reward_func/std": 0.6268348097801208,
      "sampling/importance_sampling_ratio/max": 1.5829296112060547,
      "sampling/importance_sampling_ratio/mean": 0.9177603721618652,
      "sampling/importance_sampling_ratio/min": 7.654144935832113e-16,
      "sampling/sampling_logp_difference/max": 24.656509399414062,
      "sampling/sampling_logp_difference/mean": 0.12785205245018005,
      "step": 336,
      "step_time": 12.631401132020983
    },
    {
      "clip_ratio/high_max": 0.011850660433992743,
      "clip_ratio/high_mean": 0.005925330216996372,
      "clip_ratio/low_mean": 0.0017361111240461469,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007661441341042519,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 597.0,
      "completions/max_terminated_length": 597.0,
      "completions/mean_length": 213.5,
      "completions/mean_terminated_length": 216.51724243164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.44248829409480095,
      "epoch": 0.00674,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9985675811767578,
      "kl": 0.4073210060596466,
      "learning_rate": 7.97366639442619e-06,
      "loss": 0.0379,
      "num_tokens": 13569718.0,
      "reward": 0.7962499856948853,
      "reward_std": 0.4159829020500183,
      "rewards/rollout_reward_func/mean": 0.7962499856948853,
      "rewards/rollout_reward_func/std": 0.5461285710334778,
      "sampling/importance_sampling_ratio/max": 2.4672460556030273,
      "sampling/importance_sampling_ratio/mean": 0.9877119660377502,
      "sampling/importance_sampling_ratio/min": 0.35907575488090515,
      "sampling/sampling_logp_difference/max": 1.0240545272827148,
      "sampling/sampling_logp_difference/mean": 0.04284299537539482,
      "step": 337,
      "step_time": 12.613344003955717
    },
    {
      "clip_ratio/high_max": 0.006265664240345359,
      "clip_ratio/high_mean": 0.0031328321201726794,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031328321201726794,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 648.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 189.75,
      "completions/mean_terminated_length": 201.44827270507812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4333979170769453,
      "epoch": 0.00676,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3492748737335205,
      "kl": 1.6020841281861067,
      "learning_rate": 7.961559425427666e-06,
      "loss": 0.009,
      "num_tokens": 13607990.0,
      "reward": 0.7365624904632568,
      "reward_std": 0.48182302713394165,
      "rewards/rollout_reward_func/mean": 0.7365624904632568,
      "rewards/rollout_reward_func/std": 0.5767909288406372,
      "sampling/importance_sampling_ratio/max": 1.4665862321853638,
      "sampling/importance_sampling_ratio/mean": 0.9187883138656616,
      "sampling/importance_sampling_ratio/min": 0.384683758020401,
      "sampling/sampling_logp_difference/max": 0.9990079402923584,
      "sampling/sampling_logp_difference/mean": 0.049473680555820465,
      "step": 338,
      "step_time": 12.465361922018928
    },
    {
      "clip_ratio/high_max": 0.008928571827709675,
      "clip_ratio/high_mean": 0.004464285913854837,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004464285913854837,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 284.3125,
      "completions/mean_terminated_length": 269.629638671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.39021115843206644,
      "epoch": 0.00678,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3103227615356445,
      "kl": 0.4098603343591094,
      "learning_rate": 7.949429879803298e-06,
      "loss": 0.1199,
      "num_tokens": 13650149.0,
      "reward": 0.6400405168533325,
      "reward_std": 0.5493500828742981,
      "rewards/rollout_reward_func/mean": 0.6400405168533325,
      "rewards/rollout_reward_func/std": 0.5623735785484314,
      "sampling/importance_sampling_ratio/max": 2.161395788192749,
      "sampling/importance_sampling_ratio/mean": 0.9307050704956055,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9891633987426758,
      "sampling/sampling_logp_difference/mean": 0.04385314881801605,
      "step": 339,
      "step_time": 12.331484877999173
    },
    {
      "clip_ratio/high_max": 0.011904762359336019,
      "clip_ratio/high_mean": 0.005952381179668009,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005952381179668009,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 621.0,
      "completions/max_terminated_length": 621.0,
      "completions/mean_length": 184.46875,
      "completions/mean_terminated_length": 189.06668090820312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4008747860789299,
      "epoch": 0.0068,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0462101697921753,
      "kl": 0.42089592292904854,
      "learning_rate": 7.937277917549998e-06,
      "loss": -0.072,
      "num_tokens": 13687883.0,
      "reward": 0.796875,
      "reward_std": 0.40688562393188477,
      "rewards/rollout_reward_func/mean": 0.796875,
      "rewards/rollout_reward_func/std": 0.4926323890686035,
      "sampling/importance_sampling_ratio/max": 1.68885338306427,
      "sampling/importance_sampling_ratio/mean": 1.050094723701477,
      "sampling/importance_sampling_ratio/min": 0.4163772761821747,
      "sampling/sampling_logp_difference/max": 0.7219972610473633,
      "sampling/sampling_logp_difference/mean": 0.04138198122382164,
      "step": 340,
      "step_time": 12.427231567984563
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0027173913549631834,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0027173913549631834,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 234.59375,
      "completions/mean_terminated_length": 215.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.38718167319893837,
      "epoch": 0.00682,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.436326265335083,
      "kl": 0.4245688868686557,
      "learning_rate": 7.925103698960361e-06,
      "loss": 0.0595,
      "num_tokens": 13728743.0,
      "reward": 0.6743749380111694,
      "reward_std": 0.5654648542404175,
      "rewards/rollout_reward_func/mean": 0.6743749380111694,
      "rewards/rollout_reward_func/std": 0.611470639705658,
      "sampling/importance_sampling_ratio/max": 2.2270402908325195,
      "sampling/importance_sampling_ratio/mean": 1.1330486536026,
      "sampling/importance_sampling_ratio/min": 0.5006272196769714,
      "sampling/sampling_logp_difference/max": 0.7731819152832031,
      "sampling/sampling_logp_difference/mean": 0.04057689011096954,
      "step": 341,
      "step_time": 12.430020276020514
    },
    {
      "clip_ratio/high_max": 0.008576766354963183,
      "clip_ratio/high_mean": 0.004288383177481592,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004288383177481592,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 643.0,
      "completions/max_terminated_length": 643.0,
      "completions/mean_length": 264.0625,
      "completions/mean_terminated_length": 260.1600036621094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.499120581895113,
      "epoch": 0.00684,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.28640079498291,
      "kl": 1.0365355145186186,
      "learning_rate": 7.91290738462056e-06,
      "loss": 0.0953,
      "num_tokens": 13770157.0,
      "reward": 0.7635146379470825,
      "reward_std": 0.4949694871902466,
      "rewards/rollout_reward_func/mean": 0.7635146379470825,
      "rewards/rollout_reward_func/std": 0.6032620668411255,
      "sampling/importance_sampling_ratio/max": 2.8582334518432617,
      "sampling/importance_sampling_ratio/mean": 0.9626871347427368,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.1844213008880615,
      "sampling/sampling_logp_difference/mean": 0.046933628618717194,
      "step": 342,
      "step_time": 12.63418330096465
    },
    {
      "clip_ratio/high_max": 0.009478208841755986,
      "clip_ratio/high_mean": 0.004739104420877993,
      "clip_ratio/low_mean": 0.007085585268214345,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011824689689092338,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 982.0,
      "completions/max_terminated_length": 982.0,
      "completions/mean_length": 232.90625,
      "completions/mean_terminated_length": 224.37037658691406,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.42683227453380823,
      "epoch": 0.00686,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9055384993553162,
      "kl": 0.5033328784629703,
      "learning_rate": 7.900689135408226e-06,
      "loss": -0.0182,
      "num_tokens": 13810486.0,
      "reward": 0.6396874785423279,
      "reward_std": 0.5999326109886169,
      "rewards/rollout_reward_func/mean": 0.6396874785423279,
      "rewards/rollout_reward_func/std": 0.6144154667854309,
      "sampling/importance_sampling_ratio/max": 1.9258301258087158,
      "sampling/importance_sampling_ratio/mean": 0.9238013625144958,
      "sampling/importance_sampling_ratio/min": 0.3712351620197296,
      "sampling/sampling_logp_difference/max": 1.1390552520751953,
      "sampling/sampling_logp_difference/mean": 0.048786092549562454,
      "step": 343,
      "step_time": 13.773013457976049
    },
    {
      "clip_ratio/high_max": 0.015017361380159855,
      "clip_ratio/high_mean": 0.0075086806900799274,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0075086806900799274,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 811.0,
      "completions/max_terminated_length": 811.0,
      "completions/mean_length": 206.5,
      "completions/mean_terminated_length": 188.40740966796875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49065452069044113,
      "epoch": 0.00688,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2135192155838013,
      "kl": 1.1918784938752651,
      "learning_rate": 7.888449112490325e-06,
      "loss": 0.0753,
      "num_tokens": 13848967.0,
      "reward": 0.5953751802444458,
      "reward_std": 0.6895500421524048,
      "rewards/rollout_reward_func/mean": 0.5953751802444458,
      "rewards/rollout_reward_func/std": 0.7249426245689392,
      "sampling/importance_sampling_ratio/max": 1.6378940343856812,
      "sampling/importance_sampling_ratio/mean": 0.782747745513916,
      "sampling/importance_sampling_ratio/min": 3.0971440882788528e-25,
      "sampling/sampling_logp_difference/max": 27.804580688476562,
      "sampling/sampling_logp_difference/mean": 0.5017714500427246,
      "step": 344,
      "step_time": 12.675316943001235
    },
    {
      "clip_ratio/high_max": 0.007582720601931214,
      "clip_ratio/high_mean": 0.003791360300965607,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003791360300965607,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 1168.0,
      "completions/max_terminated_length": 603.0,
      "completions/mean_length": 269.03125,
      "completions/mean_terminated_length": 248.0399932861328,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.47586480155587196,
      "epoch": 0.0069,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.992414653301239,
      "kl": 0.388470109552145,
      "learning_rate": 7.876187477321033e-06,
      "loss": -0.116,
      "num_tokens": 13890723.0,
      "reward": 0.6472501754760742,
      "reward_std": 0.6583719849586487,
      "rewards/rollout_reward_func/mean": 0.6472501754760742,
      "rewards/rollout_reward_func/std": 0.7205114364624023,
      "sampling/importance_sampling_ratio/max": 1.6967158317565918,
      "sampling/importance_sampling_ratio/mean": 0.9424106478691101,
      "sampling/importance_sampling_ratio/min": 1.8397262830414354e-27,
      "sampling/sampling_logp_difference/max": 22.820775985717773,
      "sampling/sampling_logp_difference/mean": 0.15699833631515503,
      "step": 345,
      "step_time": 15.064826934991288
    },
    {
      "clip_ratio/high_max": 0.021875000093132257,
      "clip_ratio/high_mean": 0.010937500046566129,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010937500046566129,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 673.0,
      "completions/max_terminated_length": 673.0,
      "completions/mean_length": 219.1875,
      "completions/mean_terminated_length": 225.629638671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4296400686725974,
      "epoch": 0.00692,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1687067747116089,
      "kl": 0.3691533487290144,
      "learning_rate": 7.86390439163961e-06,
      "loss": -0.0031,
      "num_tokens": 13929410.0,
      "reward": 0.7825000286102295,
      "reward_std": 0.40286940336227417,
      "rewards/rollout_reward_func/mean": 0.7825000286102295,
      "rewards/rollout_reward_func/std": 0.4786742329597473,
      "sampling/importance_sampling_ratio/max": 1.6700724363327026,
      "sampling/importance_sampling_ratio/mean": 0.9754288196563721,
      "sampling/importance_sampling_ratio/min": 0.34172993898391724,
      "sampling/sampling_logp_difference/max": 0.8624591827392578,
      "sampling/sampling_logp_difference/mean": 0.049009770154953,
      "step": 346,
      "step_time": 12.02023401101178
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0029296875,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 603.0,
      "completions/max_terminated_length": 603.0,
      "completions/mean_length": 184.90625,
      "completions/mean_terminated_length": 144.9199981689453,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.462111858651042,
      "epoch": 0.00694,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9598464369773865,
      "kl": 0.6746675446629524,
      "learning_rate": 7.851600017468257e-06,
      "loss": -0.1109,
      "num_tokens": 13967816.0,
      "reward": 0.6893749833106995,
      "reward_std": 0.6081529259681702,
      "rewards/rollout_reward_func/mean": 0.6893749833106995,
      "rewards/rollout_reward_func/std": 0.6619420647621155,
      "sampling/importance_sampling_ratio/max": 1.360652208328247,
      "sampling/importance_sampling_ratio/mean": 0.9168429374694824,
      "sampling/importance_sampling_ratio/min": 0.3783349394798279,
      "sampling/sampling_logp_difference/max": 1.0623421669006348,
      "sampling/sampling_logp_difference/mean": 0.04752412438392639,
      "step": 347,
      "step_time": 12.195823556001415
    },
    {
      "clip_ratio/high_max": 0.02218864532187581,
      "clip_ratio/high_mean": 0.011094322660937905,
      "clip_ratio/low_mean": 0.0010080644860863686,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012102387147024274,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 955.0,
      "completions/max_terminated_length": 955.0,
      "completions/mean_length": 249.96875,
      "completions/mean_terminated_length": 229.2413787841797,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.42559339851140976,
      "epoch": 0.00696,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3843011856079102,
      "kl": 1.207936018705368,
      "learning_rate": 7.839274517109987e-06,
      "loss": -0.0782,
      "num_tokens": 14009066.0,
      "reward": 0.603853166103363,
      "reward_std": 0.7394682168960571,
      "rewards/rollout_reward_func/mean": 0.603853166103363,
      "rewards/rollout_reward_func/std": 0.728130042552948,
      "sampling/importance_sampling_ratio/max": 1.701692819595337,
      "sampling/importance_sampling_ratio/mean": 0.9964526295661926,
      "sampling/importance_sampling_ratio/min": 8.242213415998765e-21,
      "sampling/sampling_logp_difference/max": 22.207117080688477,
      "sampling/sampling_logp_difference/mean": 0.16786544024944305,
      "step": 348,
      "step_time": 13.815054122998845
    },
    {
      "clip_ratio/high_max": 0.02118055592291057,
      "clip_ratio/high_mean": 0.010590277961455286,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010590277961455286,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 597.0,
      "completions/mean_length": 237.78125,
      "completions/mean_terminated_length": 221.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.36938377749174833,
      "epoch": 0.00698,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.031865119934082,
      "kl": 0.7143490361049771,
      "learning_rate": 7.826928053146482e-06,
      "loss": -0.0408,
      "num_tokens": 14049411.0,
      "reward": 0.5978149175643921,
      "reward_std": 0.7573053240776062,
      "rewards/rollout_reward_func/mean": 0.5978149175643921,
      "rewards/rollout_reward_func/std": 0.7260693907737732,
      "sampling/importance_sampling_ratio/max": 2.7485971450805664,
      "sampling/importance_sampling_ratio/mean": 1.0070356130599976,
      "sampling/importance_sampling_ratio/min": 0.34114402532577515,
      "sampling/sampling_logp_difference/max": 1.0744590759277344,
      "sampling/sampling_logp_difference/mean": 0.05239509791135788,
      "step": 349,
      "step_time": 11.934143028003746
    },
    {
      "clip_ratio/high_max": 0.006321839289739728,
      "clip_ratio/high_mean": 0.003160919644869864,
      "clip_ratio/low_mean": 0.009300595615059137,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012461515259929001,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1197.0,
      "completions/max_terminated_length": 1197.0,
      "completions/mean_length": 270.96875,
      "completions/mean_terminated_length": 278.1000061035156,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4173172600567341,
      "epoch": 0.007,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2149289846420288,
      "kl": 0.5516554834321141,
      "learning_rate": 7.814560788435947e-06,
      "loss": 0.0028,
      "num_tokens": 14091059.0,
      "reward": 0.7328124642372131,
      "reward_std": 0.5046379566192627,
      "rewards/rollout_reward_func/mean": 0.7328124642372131,
      "rewards/rollout_reward_func/std": 0.5217664241790771,
      "sampling/importance_sampling_ratio/max": 2.206382989883423,
      "sampling/importance_sampling_ratio/mean": 0.9141413569450378,
      "sampling/importance_sampling_ratio/min": 9.263244933490234e-24,
      "sampling/sampling_logp_difference/max": 24.368104934692383,
      "sampling/sampling_logp_difference/mean": 0.2695417106151581,
      "step": 350,
      "step_time": 15.220541527029127
    },
    {
      "clip_ratio/high_max": 0.013020833488553762,
      "clip_ratio/high_mean": 0.008072916651144624,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008072916651144624,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 806.0,
      "completions/max_terminated_length": 806.0,
      "completions/mean_length": 340.78125,
      "completions/mean_terminated_length": 331.75,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.35465124249458313,
      "epoch": 0.00702,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7856358289718628,
      "kl": 0.8795061763375998,
      "learning_rate": 7.802172886110968e-06,
      "loss": -0.0645,
      "num_tokens": 14135660.0,
      "reward": 0.6431249976158142,
      "reward_std": 0.6000845432281494,
      "rewards/rollout_reward_func/mean": 0.6431249976158142,
      "rewards/rollout_reward_func/std": 0.6300150156021118,
      "sampling/importance_sampling_ratio/max": 1.5912212133407593,
      "sampling/importance_sampling_ratio/mean": 0.9915253520011902,
      "sampling/importance_sampling_ratio/min": 0.26171237230300903,
      "sampling/sampling_logp_difference/max": 1.3416414260864258,
      "sampling/sampling_logp_difference/mean": 0.03982101008296013,
      "step": 351,
      "step_time": 13.165235118023702
    },
    {
      "clip_ratio/high_max": 0.00866977241821587,
      "clip_ratio/high_mean": 0.004334886209107935,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004334886209107935,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 865.0,
      "completions/max_terminated_length": 865.0,
      "completions/mean_length": 260.0,
      "completions/mean_terminated_length": 276.82757568359375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4537485782057047,
      "epoch": 0.00704,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.132468819618225,
      "kl": 0.4788108058273792,
      "learning_rate": 7.789764509576347e-06,
      "loss": -0.1217,
      "num_tokens": 14176609.0,
      "reward": 0.6866251826286316,
      "reward_std": 0.6265203952789307,
      "rewards/rollout_reward_func/mean": 0.6866251826286316,
      "rewards/rollout_reward_func/std": 0.6578096151351929,
      "sampling/importance_sampling_ratio/max": 1.9956578016281128,
      "sampling/importance_sampling_ratio/mean": 1.0040113925933838,
      "sampling/importance_sampling_ratio/min": 0.2470885068178177,
      "sampling/sampling_logp_difference/max": 0.9075932502746582,
      "sampling/sampling_logp_difference/mean": 0.047556355595588684,
      "step": 352,
      "step_time": 12.922245385008864
    },
    {
      "clip_ratio/high_max": 0.013599088182672858,
      "clip_ratio/high_mean": 0.006799544091336429,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006799544091336429,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 679.0,
      "completions/max_terminated_length": 679.0,
      "completions/mean_length": 216.9375,
      "completions/mean_terminated_length": 207.55172729492188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4555352684110403,
      "epoch": 0.00706,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1115132570266724,
      "kl": 1.1982744429260492,
      "learning_rate": 7.77733582250696e-06,
      "loss": -0.0492,
      "num_tokens": 14215306.0,
      "reward": 0.53709876537323,
      "reward_std": 0.7230541110038757,
      "rewards/rollout_reward_func/mean": 0.53709876537323,
      "rewards/rollout_reward_func/std": 0.7234609127044678,
      "sampling/importance_sampling_ratio/max": 1.4962944984436035,
      "sampling/importance_sampling_ratio/mean": 0.9731826782226562,
      "sampling/importance_sampling_ratio/min": 0.21222460269927979,
      "sampling/sampling_logp_difference/max": 1.199864387512207,
      "sampling/sampling_logp_difference/mean": 0.052086204290390015,
      "step": 353,
      "step_time": 11.86112357400998
    },
    {
      "clip_ratio/high_max": 0.005908613558858633,
      "clip_ratio/high_mean": 0.003930869163013995,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019555869163013995,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 659.0,
      "completions/max_terminated_length": 659.0,
      "completions/mean_length": 244.03125,
      "completions/mean_terminated_length": 183.8000030517578,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4946094714105129,
      "epoch": 0.00708,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8759406805038452,
      "kl": 0.49277015682309866,
      "learning_rate": 7.764886988845588e-06,
      "loss": -0.1598,
      "num_tokens": 14255025.0,
      "reward": 0.7356249690055847,
      "reward_std": 0.5660844445228577,
      "rewards/rollout_reward_func/mean": 0.7356249690055847,
      "rewards/rollout_reward_func/std": 0.5789363980293274,
      "sampling/importance_sampling_ratio/max": 1.9586894512176514,
      "sampling/importance_sampling_ratio/mean": 0.9691637754440308,
      "sampling/importance_sampling_ratio/min": 0.20972533524036407,
      "sampling/sampling_logp_difference/max": 0.831082820892334,
      "sampling/sampling_logp_difference/mean": 0.046312496066093445,
      "step": 354,
      "step_time": 12.674376346025383
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 196.875,
      "completions/mean_terminated_length": 199.72413635253906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4040968772023916,
      "epoch": 0.0071,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7716999053955078,
      "kl": 0.9174633119255304,
      "learning_rate": 7.752418172800769e-06,
      "loss": -0.0599,
      "num_tokens": 14292355.0,
      "reward": 0.69334876537323,
      "reward_std": 0.6161481142044067,
      "rewards/rollout_reward_func/mean": 0.69334876537323,
      "rewards/rollout_reward_func/std": 0.6027310490608215,
      "sampling/importance_sampling_ratio/max": 1.7526978254318237,
      "sampling/importance_sampling_ratio/mean": 1.036548376083374,
      "sampling/importance_sampling_ratio/min": 0.29840272665023804,
      "sampling/sampling_logp_difference/max": 1.0200145244598389,
      "sampling/sampling_logp_difference/mean": 0.039310023188591,
      "step": 355,
      "step_time": 12.448885800025892
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00390625,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 675.0,
      "completions/max_terminated_length": 649.0,
      "completions/mean_length": 275.71875,
      "completions/mean_terminated_length": 272.3571472167969,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.3865357469767332,
      "epoch": 0.00712,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2281129360198975,
      "kl": 0.3698003883473575,
      "learning_rate": 7.73992953884461e-06,
      "loss": 0.0885,
      "num_tokens": 14334181.0,
      "reward": 0.7381249666213989,
      "reward_std": 0.45779529213905334,
      "rewards/rollout_reward_func/mean": 0.7381249666213989,
      "rewards/rollout_reward_func/std": 0.4590975344181061,
      "sampling/importance_sampling_ratio/max": 1.6169449090957642,
      "sampling/importance_sampling_ratio/mean": 0.9921033382415771,
      "sampling/importance_sampling_ratio/min": 7.242733968126913e-09,
      "sampling/sampling_logp_difference/max": 18.652286529541016,
      "sampling/sampling_logp_difference/mean": 0.07690809667110443,
      "step": 356,
      "step_time": 12.040051070987829
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 833.0,
      "completions/max_terminated_length": 833.0,
      "completions/mean_length": 265.15625,
      "completions/mean_terminated_length": 257.3333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4103917758911848,
      "epoch": 0.00714,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7632020711898804,
      "kl": 1.791604084894061,
      "learning_rate": 7.727421251710637e-06,
      "loss": -0.1199,
      "num_tokens": 14375525.0,
      "reward": 0.723437488079071,
      "reward_std": 0.48793917894363403,
      "rewards/rollout_reward_func/mean": 0.723437488079071,
      "rewards/rollout_reward_func/std": 0.6085657477378845,
      "sampling/importance_sampling_ratio/max": 2.971259117126465,
      "sampling/importance_sampling_ratio/mean": 1.0601176023483276,
      "sampling/importance_sampling_ratio/min": 0.34790676832199097,
      "sampling/sampling_logp_difference/max": 0.984868049621582,
      "sampling/sampling_logp_difference/mean": 0.038191355764865875,
      "step": 357,
      "step_time": 13.022821636986919
    },
    {
      "clip_ratio/high_max": 0.006189613603055477,
      "clip_ratio/high_mean": 0.0030948068015277386,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01871980680152774,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 943.0,
      "completions/max_terminated_length": 943.0,
      "completions/mean_length": 268.8125,
      "completions/mean_terminated_length": 291.5555725097656,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5019124429672956,
      "epoch": 0.00716,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3271747827529907,
      "kl": 0.6239168960601091,
      "learning_rate": 7.714893476391613e-06,
      "loss": -0.0236,
      "num_tokens": 14416904.0,
      "reward": 0.7956249713897705,
      "reward_std": 0.48908334970474243,
      "rewards/rollout_reward_func/mean": 0.7956249713897705,
      "rewards/rollout_reward_func/std": 0.5694589018821716,
      "sampling/importance_sampling_ratio/max": 1.6795037984848022,
      "sampling/importance_sampling_ratio/mean": 0.9160472750663757,
      "sampling/importance_sampling_ratio/min": 1.322612830380028e-16,
      "sampling/sampling_logp_difference/max": 23.763662338256836,
      "sampling/sampling_logp_difference/mean": 0.2773606777191162,
      "step": 358,
      "step_time": 13.532374576010625
    },
    {
      "clip_ratio/high_max": 0.014465161133557558,
      "clip_ratio/high_mean": 0.007232580566778779,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008209143066778779,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 676.0,
      "completions/max_terminated_length": 676.0,
      "completions/mean_length": 245.25,
      "completions/mean_terminated_length": 236.55555725097656,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5293060094118118,
      "epoch": 0.00718,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.060997486114502,
      "kl": 0.31977387983351946,
      "learning_rate": 7.702346378137364e-06,
      "loss": 0.101,
      "num_tokens": 14458407.0,
      "reward": 0.6037499904632568,
      "reward_std": 0.598892331123352,
      "rewards/rollout_reward_func/mean": 0.6037499904632568,
      "rewards/rollout_reward_func/std": 0.6042097210884094,
      "sampling/importance_sampling_ratio/max": 1.6559523344039917,
      "sampling/importance_sampling_ratio/mean": 0.8294237852096558,
      "sampling/importance_sampling_ratio/min": 1.5250913977399587e-15,
      "sampling/sampling_logp_difference/max": 19.37491798400879,
      "sampling/sampling_logp_difference/mean": 0.26361173391342163,
      "step": 359,
      "step_time": 12.259329491993412
    },
    {
      "clip_ratio/high_max": 0.0024999999441206455,
      "clip_ratio/high_mean": 0.0012499999720603228,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0012499999720603228,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 636.0,
      "completions/max_terminated_length": 636.0,
      "completions/mean_length": 282.0,
      "completions/mean_terminated_length": 271.0384826660156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5489872246980667,
      "epoch": 0.0072,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7709935307502747,
      "kl": 0.47503549978137016,
      "learning_rate": 7.689780122452598e-06,
      "loss": -0.1035,
      "num_tokens": 14501406.0,
      "reward": 0.59375,
      "reward_std": 0.6471434235572815,
      "rewards/rollout_reward_func/mean": 0.59375,
      "rewards/rollout_reward_func/std": 0.6589764952659607,
      "sampling/importance_sampling_ratio/max": 1.4858474731445312,
      "sampling/importance_sampling_ratio/mean": 0.8694159984588623,
      "sampling/importance_sampling_ratio/min": 2.400099619780582e-17,
      "sampling/sampling_logp_difference/max": 24.890625,
      "sampling/sampling_logp_difference/mean": 0.3394428491592407,
      "step": 360,
      "step_time": 12.711577293026494
    },
    {
      "clip_ratio/high_max": 0.0022321429569274187,
      "clip_ratio/high_mean": 0.0011160714784637094,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0011160714784637094,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 684.0,
      "completions/max_terminated_length": 684.0,
      "completions/mean_length": 256.84375,
      "completions/mean_terminated_length": 237.5357208251953,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.44760362058877945,
      "epoch": 0.00722,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9594613909721375,
      "kl": 0.6528082061558962,
      "learning_rate": 7.67719487509472e-06,
      "loss": -0.1232,
      "num_tokens": 14542521.0,
      "reward": 0.6094154715538025,
      "reward_std": 0.6648638248443604,
      "rewards/rollout_reward_func/mean": 0.6094154715538025,
      "rewards/rollout_reward_func/std": 0.6809405088424683,
      "sampling/importance_sampling_ratio/max": 1.4476238489151,
      "sampling/importance_sampling_ratio/mean": 1.0044753551483154,
      "sampling/importance_sampling_ratio/min": 0.3083702325820923,
      "sampling/sampling_logp_difference/max": 0.9209287166595459,
      "sampling/sampling_logp_difference/mean": 0.04412587732076645,
      "step": 361,
      "step_time": 12.687762369023403
    },
    {
      "clip_ratio/high_max": 0.0047940341755747795,
      "clip_ratio/high_mean": 0.0023970170877873898,
      "clip_ratio/low_mean": 0.0012499999720603228,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036470170598477125,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 694.0,
      "completions/max_terminated_length": 694.0,
      "completions/mean_length": 257.125,
      "completions/mean_terminated_length": 248.51724243164062,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.46913002617657185,
      "epoch": 0.00724,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0590113401412964,
      "kl": 0.8701184326782823,
      "learning_rate": 7.664590802071653e-06,
      "loss": -0.096,
      "num_tokens": 14583506.0,
      "reward": 0.5356655120849609,
      "reward_std": 0.7393569946289062,
      "rewards/rollout_reward_func/mean": 0.5356655120849609,
      "rewards/rollout_reward_func/std": 0.7349483966827393,
      "sampling/importance_sampling_ratio/max": 1.9656212329864502,
      "sampling/importance_sampling_ratio/mean": 0.9667774438858032,
      "sampling/importance_sampling_ratio/min": 1.707714503140778e-08,
      "sampling/sampling_logp_difference/max": 17.254985809326172,
      "sampling/sampling_logp_difference/mean": 0.09915365278720856,
      "step": 362,
      "step_time": 12.631624494024436
    },
    {
      "clip_ratio/high_max": 0.005859375,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.017578125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0205078125,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 675.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 244.96875,
      "completions/mean_terminated_length": 253.59259033203125,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5022276844829321,
      "epoch": 0.00726,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9369052648544312,
      "kl": 0.4194176667369902,
      "learning_rate": 7.651968069639637e-06,
      "loss": -0.0087,
      "num_tokens": 14623258.0,
      "reward": 0.6071874499320984,
      "reward_std": 0.6113590598106384,
      "rewards/rollout_reward_func/mean": 0.6071874499320984,
      "rewards/rollout_reward_func/std": 0.6227215528488159,
      "sampling/importance_sampling_ratio/max": 1.5918017625808716,
      "sampling/importance_sampling_ratio/mean": 0.9976466298103333,
      "sampling/importance_sampling_ratio/min": 1.4379147941970315e-29,
      "sampling/sampling_logp_difference/max": 22.75001335144043,
      "sampling/sampling_logp_difference/mean": 0.2995716631412506,
      "step": 363,
      "step_time": 12.008855329011567
    },
    {
      "clip_ratio/high_max": 0.019582106499001384,
      "clip_ratio/high_mean": 0.011149748810566962,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011149748810566962,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 602.0,
      "completions/max_terminated_length": 600.0,
      "completions/mean_length": 273.28125,
      "completions/mean_terminated_length": 252.8928680419922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5088741630315781,
      "epoch": 0.00728,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3489614725112915,
      "kl": 1.0506445281207561,
      "learning_rate": 7.63932684430105e-06,
      "loss": -0.0635,
      "num_tokens": 14665448.0,
      "reward": 0.692812442779541,
      "reward_std": 0.5722436904907227,
      "rewards/rollout_reward_func/mean": 0.692812442779541,
      "rewards/rollout_reward_func/std": 0.6119362711906433,
      "sampling/importance_sampling_ratio/max": 1.8909056186676025,
      "sampling/importance_sampling_ratio/mean": 0.9046364426612854,
      "sampling/importance_sampling_ratio/min": 2.4548545823481786e-15,
      "sampling/sampling_logp_difference/max": 20.23928451538086,
      "sampling/sampling_logp_difference/mean": 0.10413730144500732,
      "step": 364,
      "step_time": 13.009468934993492
    },
    {
      "clip_ratio/high_max": 0.007795516401529312,
      "clip_ratio/high_mean": 0.005385853466577828,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005385853466577828,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 896.0,
      "completions/max_terminated_length": 896.0,
      "completions/mean_length": 261.40625,
      "completions/mean_terminated_length": 255.64515686035156,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.48320689611136913,
      "epoch": 0.0073,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8774775266647339,
      "kl": 0.35556258354336023,
      "learning_rate": 7.626667292802197e-06,
      "loss": -0.125,
      "num_tokens": 14707328.0,
      "reward": 0.864062488079071,
      "reward_std": 0.3272578716278076,
      "rewards/rollout_reward_func/mean": 0.864062488079071,
      "rewards/rollout_reward_func/std": 0.3703516721725464,
      "sampling/importance_sampling_ratio/max": 1.4199655055999756,
      "sampling/importance_sampling_ratio/mean": 0.9463016986846924,
      "sampling/importance_sampling_ratio/min": 8.273845642728103e-20,
      "sampling/sampling_logp_difference/max": 22.9354190826416,
      "sampling/sampling_logp_difference/mean": 0.21277663111686707,
      "step": 365,
      "step_time": 13.190776912975707
    },
    {
      "clip_ratio/high_max": 0.0028409091755747795,
      "clip_ratio/high_mean": 0.0014204545877873898,
      "clip_ratio/low_mean": 0.0024038462433964014,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003824300831183791,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 604.0,
      "completions/max_terminated_length": 586.0,
      "completions/mean_length": 186.125,
      "completions/mean_terminated_length": 160.73333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.45814782567322254,
      "epoch": 0.00732,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.653632640838623,
      "kl": 0.3400844456627965,
      "learning_rate": 7.613989582131121e-06,
      "loss": -0.2097,
      "num_tokens": 14745755.0,
      "reward": 0.7006250023841858,
      "reward_std": 0.5126058459281921,
      "rewards/rollout_reward_func/mean": 0.7006250023841858,
      "rewards/rollout_reward_func/std": 0.640755295753479,
      "sampling/importance_sampling_ratio/max": 1.710739254951477,
      "sampling/importance_sampling_ratio/mean": 1.0370957851409912,
      "sampling/importance_sampling_ratio/min": 0.4091634750366211,
      "sampling/sampling_logp_difference/max": 0.8904809951782227,
      "sampling/sampling_logp_difference/mean": 0.03725098446011543,
      "step": 366,
      "step_time": 11.991100423008902
    },
    {
      "clip_ratio/high_max": 0.0061827958561480045,
      "clip_ratio/high_mean": 0.0030913979280740023,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005044522928074002,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 671.0,
      "completions/max_terminated_length": 629.0,
      "completions/mean_length": 204.03125,
      "completions/mean_terminated_length": 209.3636474609375,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.48440470546483994,
      "epoch": 0.00734,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.267211675643921,
      "kl": 0.4642068548128009,
      "learning_rate": 7.6012938795153966e-06,
      "loss": -0.0723,
      "num_tokens": 14784246.0,
      "reward": 0.7337499856948853,
      "reward_std": 0.396491676568985,
      "rewards/rollout_reward_func/mean": 0.7337499856948853,
      "rewards/rollout_reward_func/std": 0.4522755742073059,
      "sampling/importance_sampling_ratio/max": 1.534969449043274,
      "sampling/importance_sampling_ratio/mean": 1.0131254196166992,
      "sampling/importance_sampling_ratio/min": 3.2077644984696008e-09,
      "sampling/sampling_logp_difference/max": 19.538394927978516,
      "sampling/sampling_logp_difference/mean": 0.08514777570962906,
      "step": 367,
      "step_time": 12.469789085007505
    },
    {
      "clip_ratio/high_max": 0.003289473708719015,
      "clip_ratio/high_mean": 0.0016447368543595076,
      "clip_ratio/low_mean": 0.0052083334885537624,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00685307034291327,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 590.0,
      "completions/max_terminated_length": 590.0,
      "completions/mean_length": 194.0625,
      "completions/mean_terminated_length": 187.60000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.44519379641860723,
      "epoch": 0.00736,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8735387325286865,
      "kl": 0.9855811484158039,
      "learning_rate": 7.588580352419923e-06,
      "loss": 0.0365,
      "num_tokens": 14822251.0,
      "reward": 0.7637499570846558,
      "reward_std": 0.4558309316635132,
      "rewards/rollout_reward_func/mean": 0.7637499570846558,
      "rewards/rollout_reward_func/std": 0.44482505321502686,
      "sampling/importance_sampling_ratio/max": 1.779655933380127,
      "sampling/importance_sampling_ratio/mean": 0.9418030977249146,
      "sampling/importance_sampling_ratio/min": 0.28660768270492554,
      "sampling/sampling_logp_difference/max": 0.721369743347168,
      "sampling/sampling_logp_difference/mean": 0.04550208896398544,
      "step": 368,
      "step_time": 12.287779759033583
    },
    {
      "clip_ratio/high_max": 0.006313131423667073,
      "clip_ratio/high_mean": 0.0031565657118335366,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031565657118335366,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 634.0,
      "completions/mean_length": 172.09375,
      "completions/mean_terminated_length": 143.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.47579587809741497,
      "epoch": 0.00738,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.1172444820404053,
      "kl": 0.3155873464420438,
      "learning_rate": 7.575849168544717e-06,
      "loss": 0.305,
      "num_tokens": 14859544.0,
      "reward": 0.7659374475479126,
      "reward_std": 0.4911676049232483,
      "rewards/rollout_reward_func/mean": 0.7659374475479126,
      "rewards/rollout_reward_func/std": 0.5657815337181091,
      "sampling/importance_sampling_ratio/max": 2.6277029514312744,
      "sampling/importance_sampling_ratio/mean": 1.0896047353744507,
      "sampling/importance_sampling_ratio/min": 0.32836586236953735,
      "sampling/sampling_logp_difference/max": 0.7228055000305176,
      "sampling/sampling_logp_difference/mean": 0.042736344039440155,
      "step": 369,
      "step_time": 12.479936898031156
    },
    {
      "clip_ratio/high_max": 0.0044531249441206455,
      "clip_ratio/high_mean": 0.0032031249720603228,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0032031249720603228,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 593.0,
      "completions/max_terminated_length": 593.0,
      "completions/mean_length": 191.40625,
      "completions/mean_terminated_length": 215.32144165039062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5154035221785307,
      "epoch": 0.0074,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0868840217590332,
      "kl": 0.3920170087367296,
      "learning_rate": 7.5631004958227e-06,
      "loss": -0.018,
      "num_tokens": 14897646.0,
      "reward": 0.7328529953956604,
      "reward_std": 0.5588387846946716,
      "rewards/rollout_reward_func/mean": 0.7328529953956604,
      "rewards/rollout_reward_func/std": 0.6506181955337524,
      "sampling/importance_sampling_ratio/max": 1.4281370639801025,
      "sampling/importance_sampling_ratio/mean": 0.8846172094345093,
      "sampling/importance_sampling_ratio/min": 7.710136667974851e-26,
      "sampling/sampling_logp_difference/max": 26.135292053222656,
      "sampling/sampling_logp_difference/mean": 0.2815588414669037,
      "step": 370,
      "step_time": 12.250673126036418
    },
    {
      "clip_ratio/high_max": 0.004774305620230734,
      "clip_ratio/high_mean": 0.002387152810115367,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004340277810115367,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 963.0,
      "completions/max_terminated_length": 662.0,
      "completions/mean_length": 297.5625,
      "completions/mean_terminated_length": 264.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5588977336883545,
      "epoch": 0.00742,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.544822335243225,
      "kl": 0.4806318562477827,
      "learning_rate": 7.550334502417483e-06,
      "loss": 0.0701,
      "num_tokens": 14939928.0,
      "reward": 0.6221874952316284,
      "reward_std": 0.5580174922943115,
      "rewards/rollout_reward_func/mean": 0.6221874952316284,
      "rewards/rollout_reward_func/std": 0.5586026906967163,
      "sampling/importance_sampling_ratio/max": 2.3466227054595947,
      "sampling/importance_sampling_ratio/mean": 0.9831658601760864,
      "sampling/importance_sampling_ratio/min": 0.1899835765361786,
      "sampling/sampling_logp_difference/max": 0.8822603225708008,
      "sampling/sampling_logp_difference/mean": 0.04375050216913223,
      "step": 371,
      "step_time": 13.913260708985035
    },
    {
      "clip_ratio/high_max": 0.007378472248092294,
      "clip_ratio/high_mean": 0.003689236124046147,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003689236124046147,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 928.0,
      "completions/max_terminated_length": 928.0,
      "completions/mean_length": 338.53125,
      "completions/mean_terminated_length": 350.4827575683594,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.452408941462636,
      "epoch": 0.00744,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1116180419921875,
      "kl": 0.43764386512339115,
      "learning_rate": 7.537551356721149e-06,
      "loss": -0.1183,
      "num_tokens": 14984216.0,
      "reward": 0.61350017786026,
      "reward_std": 0.5464091300964355,
      "rewards/rollout_reward_func/mean": 0.61350017786026,
      "rewards/rollout_reward_func/std": 0.5669005513191223,
      "sampling/importance_sampling_ratio/max": 1.8505840301513672,
      "sampling/importance_sampling_ratio/mean": 0.9337740540504456,
      "sampling/importance_sampling_ratio/min": 1.1596875548635604e-24,
      "sampling/sampling_logp_difference/max": 24.0561580657959,
      "sampling/sampling_logp_difference/mean": 0.15208086371421814,
      "step": 372,
      "step_time": 13.688058400977752
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0013586956774815917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0023352581774815917,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 600.0,
      "completions/max_terminated_length": 600.0,
      "completions/mean_length": 165.21875,
      "completions/mean_terminated_length": 170.0800018310547,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5623857509344816,
      "epoch": 0.00746,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.205772042274475,
      "kl": 0.35661681834608316,
      "learning_rate": 7.5247512273520325e-06,
      "loss": -0.0977,
      "num_tokens": 15021924.0,
      "reward": 0.6295987367630005,
      "reward_std": 0.7098708152770996,
      "rewards/rollout_reward_func/mean": 0.6295987367630005,
      "rewards/rollout_reward_func/std": 0.7221973538398743,
      "sampling/importance_sampling_ratio/max": 2.259594440460205,
      "sampling/importance_sampling_ratio/mean": 0.966063380241394,
      "sampling/importance_sampling_ratio/min": 1.5765197442863324e-14,
      "sampling/sampling_logp_difference/max": 6.074400424957275,
      "sampling/sampling_logp_difference/mean": 0.15298855304718018,
      "step": 373,
      "step_time": 12.216572126970277
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 568.0,
      "completions/max_terminated_length": 568.0,
      "completions/mean_length": 186.15625,
      "completions/mean_terminated_length": 203.20689392089844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4663079739548266,
      "epoch": 0.00748,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.217982292175293,
      "kl": 0.4879214935936034,
      "learning_rate": 7.511934283152491e-06,
      "loss": -0.0269,
      "num_tokens": 15060544.0,
      "reward": 0.6303752064704895,
      "reward_std": 0.5601131319999695,
      "rewards/rollout_reward_func/mean": 0.6303752064704895,
      "rewards/rollout_reward_func/std": 0.613673210144043,
      "sampling/importance_sampling_ratio/max": 1.5803147554397583,
      "sampling/importance_sampling_ratio/mean": 0.9515833854675293,
      "sampling/importance_sampling_ratio/min": 8.315430803855711e-18,
      "sampling/sampling_logp_difference/max": 14.631623268127441,
      "sampling/sampling_logp_difference/mean": 0.16983173787593842,
      "step": 374,
      "step_time": 12.11424735300534
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 589.0,
      "completions/max_terminated_length": 589.0,
      "completions/mean_length": 141.75,
      "completions/mean_terminated_length": 143.92857360839844,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.458244726061821,
      "epoch": 0.0075,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2168546915054321,
      "kl": 0.3665638081729412,
      "learning_rate": 7.499100693186684e-06,
      "loss": -0.0134,
      "num_tokens": 15094812.0,
      "reward": 0.7574999928474426,
      "reward_std": 0.43453484773635864,
      "rewards/rollout_reward_func/mean": 0.7574999928474426,
      "rewards/rollout_reward_func/std": 0.44381773471832275,
      "sampling/importance_sampling_ratio/max": 1.9687235355377197,
      "sampling/importance_sampling_ratio/mean": 1.0812792778015137,
      "sampling/importance_sampling_ratio/min": 0.40252766013145447,
      "sampling/sampling_logp_difference/max": 0.9024887084960938,
      "sampling/sampling_logp_difference/mean": 0.031928349286317825,
      "step": 375,
      "step_time": 12.109336367022479
    },
    {
      "clip_ratio/high_max": 0.005629595601931214,
      "clip_ratio/high_mean": 0.002814797800965607,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002814797800965607,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 259.28125,
      "completions/mean_terminated_length": 275.4137878417969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4580981247127056,
      "epoch": 0.00752,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9696118831634521,
      "kl": 0.3642066279426217,
      "learning_rate": 7.486250626738338e-06,
      "loss": -0.0209,
      "num_tokens": 15136602.0,
      "reward": 0.5815625190734863,
      "reward_std": 0.5193930864334106,
      "rewards/rollout_reward_func/mean": 0.5815625190734863,
      "rewards/rollout_reward_func/std": 0.5631942749023438,
      "sampling/importance_sampling_ratio/max": 1.5593174695968628,
      "sampling/importance_sampling_ratio/mean": 1.0432511568069458,
      "sampling/importance_sampling_ratio/min": 0.5217245817184448,
      "sampling/sampling_logp_difference/max": 0.5974421501159668,
      "sampling/sampling_logp_difference/mean": 0.03562925010919571,
      "step": 376,
      "step_time": 12.473952981992625
    },
    {
      "clip_ratio/high_max": 0.024975896114483476,
      "clip_ratio/high_mean": 0.012487948057241738,
      "clip_ratio/low_mean": 0.0011574074160307646,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013645355473272502,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 946.0,
      "completions/max_terminated_length": 946.0,
      "completions/mean_length": 276.96875,
      "completions/mean_terminated_length": 275.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.45508138090372086,
      "epoch": 0.00754,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.32590913772583,
      "kl": 0.3990713618695736,
      "learning_rate": 7.473384253308518e-06,
      "loss": -0.0466,
      "num_tokens": 15178603.0,
      "reward": 0.8531249761581421,
      "reward_std": 0.3855856657028198,
      "rewards/rollout_reward_func/mean": 0.8531249761581421,
      "rewards/rollout_reward_func/std": 0.46674877405166626,
      "sampling/importance_sampling_ratio/max": 1.6212964057922363,
      "sampling/importance_sampling_ratio/mean": 0.974773108959198,
      "sampling/importance_sampling_ratio/min": 0.38474398851394653,
      "sampling/sampling_logp_difference/max": 0.8518047332763672,
      "sampling/sampling_logp_difference/mean": 0.0426722876727581,
      "step": 377,
      "step_time": 13.033275432055234
    },
    {
      "clip_ratio/high_max": 0.011488970601931214,
      "clip_ratio/high_mean": 0.005744485300965607,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005744485300965607,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 276.96875,
      "completions/mean_terminated_length": 285.5517272949219,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.43840767443180084,
      "epoch": 0.00756,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4792958498001099,
      "kl": 0.4196188226342201,
      "learning_rate": 7.460501742613385e-06,
      "loss": 0.0464,
      "num_tokens": 15220831.0,
      "reward": 0.7087499499320984,
      "reward_std": 0.48077869415283203,
      "rewards/rollout_reward_func/mean": 0.7087499499320984,
      "rewards/rollout_reward_func/std": 0.46305957436561584,
      "sampling/importance_sampling_ratio/max": 2.1579372882843018,
      "sampling/importance_sampling_ratio/mean": 0.9694286584854126,
      "sampling/importance_sampling_ratio/min": 7.218431981857889e-17,
      "sampling/sampling_logp_difference/max": 24.14595603942871,
      "sampling/sampling_logp_difference/mean": 0.13567806780338287,
      "step": 378,
      "step_time": 13.047298217978096
    },
    {
      "clip_ratio/high_max": 0.0028409091755747795,
      "clip_ratio/high_mean": 0.0014204545877873898,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0014204545877873898,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 649.0,
      "completions/max_terminated_length": 649.0,
      "completions/mean_length": 290.3125,
      "completions/mean_terminated_length": 280.13336181640625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4488293379545212,
      "epoch": 0.00758,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1744931936264038,
      "kl": 0.40379732474684715,
      "learning_rate": 7.447603264581964e-06,
      "loss": -0.172,
      "num_tokens": 15263565.0,
      "reward": 0.8087499737739563,
      "reward_std": 0.4926704168319702,
      "rewards/rollout_reward_func/mean": 0.8087499737739563,
      "rewards/rollout_reward_func/std": 0.4847596287727356,
      "sampling/importance_sampling_ratio/max": 2.217747211456299,
      "sampling/importance_sampling_ratio/mean": 1.12052583694458,
      "sampling/importance_sampling_ratio/min": 0.555884599685669,
      "sampling/sampling_logp_difference/max": 0.4691884517669678,
      "sampling/sampling_logp_difference/mean": 0.03755958378314972,
      "step": 379,
      "step_time": 12.794058800049243
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 641.0,
      "completions/max_terminated_length": 641.0,
      "completions/mean_length": 257.78125,
      "completions/mean_terminated_length": 268.4827575683594,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4541700556874275,
      "epoch": 0.0076,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7979265451431274,
      "kl": 0.6114187818020582,
      "learning_rate": 7.4346889893539e-06,
      "loss": -0.1512,
      "num_tokens": 15305227.0,
      "reward": 0.6396874785423279,
      "reward_std": 0.6374855041503906,
      "rewards/rollout_reward_func/mean": 0.6396874785423279,
      "rewards/rollout_reward_func/std": 0.6220075488090515,
      "sampling/importance_sampling_ratio/max": 1.4584590196609497,
      "sampling/importance_sampling_ratio/mean": 0.9969382286071777,
      "sampling/importance_sampling_ratio/min": 0.36085188388824463,
      "sampling/sampling_logp_difference/max": 1.0252876281738281,
      "sampling/sampling_logp_difference/mean": 0.034039247781038284,
      "step": 380,
      "step_time": 12.114868600023328
    },
    {
      "clip_ratio/high_max": 0.004464285913854837,
      "clip_ratio/high_mean": 0.0022321429569274187,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022321429569274187,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 679.0,
      "completions/mean_length": 263.25,
      "completions/mean_terminated_length": 232.239990234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.511651748791337,
      "epoch": 0.00762,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9996926188468933,
      "kl": 0.4494618084281683,
      "learning_rate": 7.421759087277214e-06,
      "loss": -0.0381,
      "num_tokens": 15347213.0,
      "reward": 0.6003124713897705,
      "reward_std": 0.5520510077476501,
      "rewards/rollout_reward_func/mean": 0.6003124713897705,
      "rewards/rollout_reward_func/std": 0.5721858739852905,
      "sampling/importance_sampling_ratio/max": 1.910505771636963,
      "sampling/importance_sampling_ratio/mean": 0.9940623044967651,
      "sampling/importance_sampling_ratio/min": 0.37333357334136963,
      "sampling/sampling_logp_difference/max": 0.7109620571136475,
      "sampling/sampling_logp_difference/mean": 0.0375174842774868,
      "step": 381,
      "step_time": 13.218429986023693
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 342.15625,
      "completions/mean_terminated_length": 333.8214416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.46911236830055714,
      "epoch": 0.00764,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1620715856552124,
      "kl": 0.35352775640785694,
      "learning_rate": 7.408813728906053e-06,
      "loss": 0.002,
      "num_tokens": 15392229.0,
      "reward": 0.6474999785423279,
      "reward_std": 0.5783788561820984,
      "rewards/rollout_reward_func/mean": 0.6474999785423279,
      "rewards/rollout_reward_func/std": 0.6128252148628235,
      "sampling/importance_sampling_ratio/max": 1.3418684005737305,
      "sampling/importance_sampling_ratio/mean": 0.9875934720039368,
      "sampling/importance_sampling_ratio/min": 0.5874407887458801,
      "sampling/sampling_logp_difference/max": 0.5895133018493652,
      "sampling/sampling_logp_difference/mean": 0.033135250210762024,
      "step": 382,
      "step_time": 12.866381364961853
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0033735795877873898,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0033735795877873898,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 234.0,
      "completions/mean_terminated_length": 241.96429443359375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.3992745755240321,
      "epoch": 0.00766,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9404740333557129,
      "kl": 0.7445312151685357,
      "learning_rate": 7.395853084998448e-06,
      "loss": -0.0274,
      "num_tokens": 15432756.0,
      "reward": 0.6087499856948853,
      "reward_std": 0.6298362612724304,
      "rewards/rollout_reward_func/mean": 0.6087499856948853,
      "rewards/rollout_reward_func/std": 0.6207786202430725,
      "sampling/importance_sampling_ratio/max": 1.5984609127044678,
      "sampling/importance_sampling_ratio/mean": 0.9597001075744629,
      "sampling/importance_sampling_ratio/min": 0.3458899259567261,
      "sampling/sampling_logp_difference/max": 1.02583646774292,
      "sampling/sampling_logp_difference/mean": 0.038074590265750885,
      "step": 383,
      "step_time": 12.665259872024762
    },
    {
      "clip_ratio/high_max": 0.008928571827709675,
      "clip_ratio/high_mean": 0.005115327425301075,
      "clip_ratio/low_mean": 0.0013586956774815917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006474023102782667,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1044.0,
      "completions/max_terminated_length": 611.0,
      "completions/mean_length": 350.59375,
      "completions/mean_terminated_length": 315.71429443359375,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.446381576359272,
      "epoch": 0.00768,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.892016887664795,
      "kl": 0.335650528781116,
      "learning_rate": 7.382877326514051e-06,
      "loss": 0.2412,
      "num_tokens": 15477443.0,
      "reward": 0.7290624976158142,
      "reward_std": 0.47243374586105347,
      "rewards/rollout_reward_func/mean": 0.7290624976158142,
      "rewards/rollout_reward_func/std": 0.4529285132884979,
      "sampling/importance_sampling_ratio/max": 1.926625370979309,
      "sampling/importance_sampling_ratio/mean": 0.9521855711936951,
      "sampling/importance_sampling_ratio/min": 0.44416317343711853,
      "sampling/sampling_logp_difference/max": 0.594273567199707,
      "sampling/sampling_logp_difference/mean": 0.036277443170547485,
      "step": 384,
      "step_time": 13.56420097699447
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.0013020833721384406,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005208333372138441,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1207.0,
      "completions/max_terminated_length": 1207.0,
      "completions/mean_length": 258.90625,
      "completions/mean_terminated_length": 253.00001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.43645303789526224,
      "epoch": 0.0077,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2049452066421509,
      "kl": 0.627929562702775,
      "learning_rate": 7.369886624611889e-06,
      "loss": 0.0568,
      "num_tokens": 15518361.0,
      "reward": 0.5106250047683716,
      "reward_std": 0.6246298551559448,
      "rewards/rollout_reward_func/mean": 0.5106250047683716,
      "rewards/rollout_reward_func/std": 0.6297205090522766,
      "sampling/importance_sampling_ratio/max": 1.3827533721923828,
      "sampling/importance_sampling_ratio/mean": 0.9256187677383423,
      "sampling/importance_sampling_ratio/min": 5.982285068826968e-16,
      "sampling/sampling_logp_difference/max": 25.870542526245117,
      "sampling/sampling_logp_difference/mean": 0.12322919815778732,
      "step": 385,
      "step_time": 15.415862803987693
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 625.0,
      "completions/max_terminated_length": 625.0,
      "completions/mean_length": 135.3125,
      "completions/mean_terminated_length": 142.73333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.420874061062932,
      "epoch": 0.00772,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7589897513389587,
      "kl": 0.42239530198276043,
      "learning_rate": 7.356881150648103e-06,
      "loss": 0.0404,
      "num_tokens": 15554098.0,
      "reward": 0.637499988079071,
      "reward_std": 0.538061261177063,
      "rewards/rollout_reward_func/mean": 0.637499988079071,
      "rewards/rollout_reward_func/std": 0.6017768979072571,
      "sampling/importance_sampling_ratio/max": 1.7613073587417603,
      "sampling/importance_sampling_ratio/mean": 1.0263701677322388,
      "sampling/importance_sampling_ratio/min": 6.451435045440749e-17,
      "sampling/sampling_logp_difference/max": 23.266977310180664,
      "sampling/sampling_logp_difference/mean": 0.1446499079465866,
      "step": 386,
      "step_time": 11.981576790029067
    },
    {
      "clip_ratio/high_max": 0.024388469755649567,
      "clip_ratio/high_mean": 0.012194234877824783,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012194234877824783,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 623.0,
      "completions/max_terminated_length": 623.0,
      "completions/mean_length": 246.75,
      "completions/mean_terminated_length": 228.27586364746094,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.45125361531972885,
      "epoch": 0.00774,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8127318024635315,
      "kl": 1.371409809216857,
      "learning_rate": 7.343861076173684e-06,
      "loss": -0.1154,
      "num_tokens": 15594523.0,
      "reward": 0.7256249785423279,
      "reward_std": 0.4244236946105957,
      "rewards/rollout_reward_func/mean": 0.7256249785423279,
      "rewards/rollout_reward_func/std": 0.5362229943275452,
      "sampling/importance_sampling_ratio/max": 1.5153484344482422,
      "sampling/importance_sampling_ratio/mean": 0.8907012939453125,
      "sampling/importance_sampling_ratio/min": 0.18472136557102203,
      "sampling/sampling_logp_difference/max": 1.5515446662902832,
      "sampling/sampling_logp_difference/mean": 0.039618249982595444,
      "step": 387,
      "step_time": 12.077938772999914
    },
    {
      "clip_ratio/high_max": 0.010745614301413298,
      "clip_ratio/high_mean": 0.005372807150706649,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007325932150706649,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 896.0,
      "completions/max_terminated_length": 896.0,
      "completions/mean_length": 314.125,
      "completions/mean_terminated_length": 328.6206970214844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3963037058711052,
      "epoch": 0.00776,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2115365266799927,
      "kl": 0.45833425503224134,
      "learning_rate": 7.330826572932217e-06,
      "loss": 0.0046,
      "num_tokens": 15639045.0,
      "reward": 0.6728124618530273,
      "reward_std": 0.4882604777812958,
      "rewards/rollout_reward_func/mean": 0.6728124618530273,
      "rewards/rollout_reward_func/std": 0.48039117455482483,
      "sampling/importance_sampling_ratio/max": 1.7929414510726929,
      "sampling/importance_sampling_ratio/mean": 1.0442496538162231,
      "sampling/importance_sampling_ratio/min": 0.5027572512626648,
      "sampling/sampling_logp_difference/max": 0.6912245750427246,
      "sampling/sampling_logp_difference/mean": 0.03165550157427788,
      "step": 388,
      "step_time": 13.449577767009032
    },
    {
      "clip_ratio/high_max": 0.034926470601931214,
      "clip_ratio/high_mean": 0.017463235300965607,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017463235300965607,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 687.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 252.34375,
      "completions/mean_terminated_length": 247.88462829589844,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.3924107886850834,
      "epoch": 0.00778,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.135040283203125,
      "kl": 0.40493219159543514,
      "learning_rate": 7.3177778128576124e-06,
      "loss": 0.0042,
      "num_tokens": 15681545.0,
      "reward": 0.8387500047683716,
      "reward_std": 0.40238845348358154,
      "rewards/rollout_reward_func/mean": 0.8387500047683716,
      "rewards/rollout_reward_func/std": 0.4717828035354614,
      "sampling/importance_sampling_ratio/max": 2.1132187843322754,
      "sampling/importance_sampling_ratio/mean": 1.0587393045425415,
      "sampling/importance_sampling_ratio/min": 0.4790618121623993,
      "sampling/sampling_logp_difference/max": 0.8169937133789062,
      "sampling/sampling_logp_difference/mean": 0.03559418022632599,
      "step": 389,
      "step_time": 12.505216621953878
    },
    {
      "clip_ratio/high_max": 0.010416666977107525,
      "clip_ratio/high_mean": 0.0052083334885537624,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0096726194024086,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 896.0,
      "completions/max_terminated_length": 896.0,
      "completions/mean_length": 232.625,
      "completions/mean_terminated_length": 239.61289978027344,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.4271478792652488,
      "epoch": 0.0078,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7495598793029785,
      "kl": 0.4172661043703556,
      "learning_rate": 7.304714968071834e-06,
      "loss": -0.0628,
      "num_tokens": 15721000.0,
      "reward": 0.6765624284744263,
      "reward_std": 0.6612355709075928,
      "rewards/rollout_reward_func/mean": 0.6765624284744263,
      "rewards/rollout_reward_func/std": 0.6427993178367615,
      "sampling/importance_sampling_ratio/max": 1.4148222208023071,
      "sampling/importance_sampling_ratio/mean": 0.9615904688835144,
      "sampling/importance_sampling_ratio/min": 0.4087088108062744,
      "sampling/sampling_logp_difference/max": 0.765251636505127,
      "sampling/sampling_logp_difference/mean": 0.03456985205411911,
      "step": 390,
      "step_time": 12.40958768899145
    },
    {
      "clip_ratio/high_max": 0.019594253972172737,
      "clip_ratio/high_mean": 0.009797126986086369,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009797126986086369,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 602.0,
      "completions/max_terminated_length": 602.0,
      "completions/mean_length": 259.8125,
      "completions/mean_terminated_length": 253.25001525878906,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.45262424647808075,
      "epoch": 0.00782,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5474547147750854,
      "kl": 0.6542339418083429,
      "learning_rate": 7.291638210882638e-06,
      "loss": 0.0788,
      "num_tokens": 15762256.0,
      "reward": 0.7046874761581421,
      "reward_std": 0.5892276763916016,
      "rewards/rollout_reward_func/mean": 0.7046874761581421,
      "rewards/rollout_reward_func/std": 0.6071880459785461,
      "sampling/importance_sampling_ratio/max": 1.7597482204437256,
      "sampling/importance_sampling_ratio/mean": 0.9509975910186768,
      "sampling/importance_sampling_ratio/min": 0.36514005064964294,
      "sampling/sampling_logp_difference/max": 0.5914068222045898,
      "sampling/sampling_logp_difference/mean": 0.037613317370414734,
      "step": 391,
      "step_time": 13.265386505023343
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.008138020988553762,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009114583488553762,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 748.0,
      "completions/max_terminated_length": 748.0,
      "completions/mean_length": 238.21875,
      "completions/mean_terminated_length": 227.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.43071379885077477,
      "epoch": 0.00784,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9452321529388428,
      "kl": 0.7696073446422815,
      "learning_rate": 7.278547713781288e-06,
      "loss": -0.1856,
      "num_tokens": 15802619.0,
      "reward": 0.7528125047683716,
      "reward_std": 0.42848435044288635,
      "rewards/rollout_reward_func/mean": 0.7528125047683716,
      "rewards/rollout_reward_func/std": 0.5402999520301819,
      "sampling/importance_sampling_ratio/max": 1.4930086135864258,
      "sampling/importance_sampling_ratio/mean": 1.0072450637817383,
      "sampling/importance_sampling_ratio/min": 0.176529198884964,
      "sampling/sampling_logp_difference/max": 0.7793912887573242,
      "sampling/sampling_logp_difference/mean": 0.04149286448955536,
      "step": 392,
      "step_time": 12.60155326698441
    },
    {
      "clip_ratio/high_max": 0.014732143143191934,
      "clip_ratio/high_mean": 0.007366071571595967,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007366071571595967,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 148.0,
      "completions/mean_terminated_length": 134.56668090820312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.44926171377301216,
      "epoch": 0.00786,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2371666431427002,
      "kl": 0.488713382743299,
      "learning_rate": 7.2654436494402955e-06,
      "loss": 0.0519,
      "num_tokens": 15839006.0,
      "reward": 0.7956249713897705,
      "reward_std": 0.47901681065559387,
      "rewards/rollout_reward_func/mean": 0.7956249713897705,
      "rewards/rollout_reward_func/std": 0.4931199848651886,
      "sampling/importance_sampling_ratio/max": 1.7224191427230835,
      "sampling/importance_sampling_ratio/mean": 0.9983428120613098,
      "sampling/importance_sampling_ratio/min": 5.992060358433874e-16,
      "sampling/sampling_logp_difference/max": 26.381216049194336,
      "sampling/sampling_logp_difference/mean": 0.12321116030216217,
      "step": 393,
      "step_time": 11.829922721008188
    },
    {
      "clip_ratio/high_max": 0.006884058238938451,
      "clip_ratio/high_mean": 0.0034420291194692254,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0034420291194692254,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 648.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 190.5,
      "completions/mean_terminated_length": 176.90321350097656,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.400854354724288,
      "epoch": 0.00788,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8924341797828674,
      "kl": 0.27446773182600737,
      "learning_rate": 7.252326190711121e-06,
      "loss": -0.0071,
      "num_tokens": 15876663.0,
      "reward": 0.890625,
      "reward_std": 0.29287928342819214,
      "rewards/rollout_reward_func/mean": 0.890625,
      "rewards/rollout_reward_func/std": 0.3312531113624573,
      "sampling/importance_sampling_ratio/max": 1.3652430772781372,
      "sampling/importance_sampling_ratio/mean": 1.0494877099990845,
      "sampling/importance_sampling_ratio/min": 0.60752272605896,
      "sampling/sampling_logp_difference/max": 0.40881872177124023,
      "sampling/sampling_logp_difference/mean": 0.02851809561252594,
      "step": 394,
      "step_time": 12.229001778003294
    },
    {
      "clip_ratio/high_max": 0.007843137485906482,
      "clip_ratio/high_mean": 0.003921568742953241,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006525735487230122,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 267.4375,
      "completions/mean_terminated_length": 255.03334045410156,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.42716559395194054,
      "epoch": 0.0079,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.801725387573242,
      "kl": 0.32933750841766596,
      "learning_rate": 7.239195510621918e-06,
      "loss": 0.1597,
      "num_tokens": 15918328.0,
      "reward": 0.7690624594688416,
      "reward_std": 0.4928715229034424,
      "rewards/rollout_reward_func/mean": 0.7690624594688416,
      "rewards/rollout_reward_func/std": 0.49474239349365234,
      "sampling/importance_sampling_ratio/max": 2.590547561645508,
      "sampling/importance_sampling_ratio/mean": 1.033278465270996,
      "sampling/importance_sampling_ratio/min": 0.3035348653793335,
      "sampling/sampling_logp_difference/max": 0.5834352970123291,
      "sampling/sampling_logp_difference/mean": 0.03558478504419327,
      "step": 395,
      "step_time": 12.510987414992996
    },
    {
      "clip_ratio/high_max": 0.0022321429569274187,
      "clip_ratio/high_mean": 0.0011160714784637094,
      "clip_ratio/low_mean": 0.0069390530698001385,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008055124548263848,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 643.0,
      "completions/max_terminated_length": 619.0,
      "completions/mean_length": 240.15625,
      "completions/mean_terminated_length": 217.0800018310547,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.44594016671180725,
      "epoch": 0.00792,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.167606234550476,
      "kl": 0.40770382899791,
      "learning_rate": 7.22605178237523e-06,
      "loss": -0.0118,
      "num_tokens": 15958904.0,
      "reward": 0.7093749642372131,
      "reward_std": 0.4470524191856384,
      "rewards/rollout_reward_func/mean": 0.7093749642372131,
      "rewards/rollout_reward_func/std": 0.5358859896659851,
      "sampling/importance_sampling_ratio/max": 1.4749870300292969,
      "sampling/importance_sampling_ratio/mean": 0.9736949801445007,
      "sampling/importance_sampling_ratio/min": 0.48128852248191833,
      "sampling/sampling_logp_difference/max": 0.7207903861999512,
      "sampling/sampling_logp_difference/mean": 0.03490980714559555,
      "step": 396,
      "step_time": 12.689915078008198
    },
    {
      "clip_ratio/high_max": 0.021484375,
      "clip_ratio/high_mean": 0.0107421875,
      "clip_ratio/low_mean": 0.0013586956774815917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012100883177481592,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 641.0,
      "completions/mean_length": 267.03125,
      "completions/mean_terminated_length": 260.77777099609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5300067700445652,
      "epoch": 0.00794,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7792929410934448,
      "kl": 0.5968853635713458,
      "learning_rate": 7.212895179345718e-06,
      "loss": 0.176,
      "num_tokens": 16000124.0,
      "reward": 0.825624942779541,
      "reward_std": 0.344956636428833,
      "rewards/rollout_reward_func/mean": 0.825624942779541,
      "rewards/rollout_reward_func/std": 0.40026551485061646,
      "sampling/importance_sampling_ratio/max": 1.881333351135254,
      "sampling/importance_sampling_ratio/mean": 1.0282840728759766,
      "sampling/importance_sampling_ratio/min": 1.548647552919198e-22,
      "sampling/sampling_logp_difference/max": 24.496259689331055,
      "sampling/sampling_logp_difference/mean": 0.13841719925403595,
      "step": 397,
      "step_time": 12.942030093021458
    },
    {
      "clip_ratio/high_max": 0.005425347248092294,
      "clip_ratio/high_mean": 0.002712673624046147,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010525173624046147,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 660.0,
      "completions/mean_length": 323.96875,
      "completions/mean_terminated_length": 329.0,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.5536190327256918,
      "epoch": 0.00796,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2974711656570435,
      "kl": 0.8176724831573665,
      "learning_rate": 7.199725875077871e-06,
      "loss": -0.0334,
      "num_tokens": 16044378.0,
      "reward": 0.765625,
      "reward_std": 0.4511410892009735,
      "rewards/rollout_reward_func/mean": 0.765625,
      "rewards/rollout_reward_func/std": 0.5238931775093079,
      "sampling/importance_sampling_ratio/max": 1.4241585731506348,
      "sampling/importance_sampling_ratio/mean": 0.9765754342079163,
      "sampling/importance_sampling_ratio/min": 0.48595699667930603,
      "sampling/sampling_logp_difference/max": 0.9554305076599121,
      "sampling/sampling_logp_difference/mean": 0.033830948173999786,
      "step": 398,
      "step_time": 12.817042907990981
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 937.0,
      "completions/max_terminated_length": 937.0,
      "completions/mean_length": 223.53125,
      "completions/mean_terminated_length": 237.36668395996094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5471077244728804,
      "epoch": 0.00798,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9816545844078064,
      "kl": 1.1190813202410936,
      "learning_rate": 7.186544043283715e-06,
      "loss": -0.1532,
      "num_tokens": 16084007.0,
      "reward": 0.5128124952316284,
      "reward_std": 0.7689528465270996,
      "rewards/rollout_reward_func/mean": 0.5128124952316284,
      "rewards/rollout_reward_func/std": 0.7362644672393799,
      "sampling/importance_sampling_ratio/max": 1.5645288228988647,
      "sampling/importance_sampling_ratio/mean": 0.9549540281295776,
      "sampling/importance_sampling_ratio/min": 1.2431400276313644e-14,
      "sampling/sampling_logp_difference/max": 21.6195011138916,
      "sampling/sampling_logp_difference/mean": 0.10769607871770859,
      "step": 399,
      "step_time": 14.150940994994016
    },
    {
      "clip_ratio/high_max": 0.007945075863972306,
      "clip_ratio/high_mean": 0.003972537931986153,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003972537931986153,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 656.0,
      "completions/mean_length": 242.9375,
      "completions/mean_terminated_length": 248.71429443359375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.6367805749177933,
      "epoch": 0.008,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0082658529281616,
      "kl": 0.7533454969525337,
      "learning_rate": 7.173349857840521e-06,
      "loss": -0.0152,
      "num_tokens": 16124824.0,
      "reward": 0.7300000190734863,
      "reward_std": 0.510733962059021,
      "rewards/rollout_reward_func/mean": 0.7300000190734863,
      "rewards/rollout_reward_func/std": 0.5216413140296936,
      "sampling/importance_sampling_ratio/max": 1.5563322305679321,
      "sampling/importance_sampling_ratio/mean": 0.7742918729782104,
      "sampling/importance_sampling_ratio/min": 4.8700447293588064e-15,
      "sampling/sampling_logp_difference/max": 23.272043228149414,
      "sampling/sampling_logp_difference/mean": 0.26736775040626526,
      "step": 400,
      "step_time": 13.026480829998036
    },
    {
      "clip_ratio/high_max": 0.014516129158437252,
      "clip_ratio/high_mean": 0.007258064579218626,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007258064579218626,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 660.0,
      "completions/mean_length": 244.59375,
      "completions/mean_terminated_length": 257.6785888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5132735762745142,
      "epoch": 0.00802,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7777087092399597,
      "kl": 0.6020323345437646,
      "learning_rate": 7.160143492788516e-06,
      "loss": -0.1317,
      "num_tokens": 16166004.0,
      "reward": 0.6081876754760742,
      "reward_std": 0.5682283043861389,
      "rewards/rollout_reward_func/mean": 0.6081876754760742,
      "rewards/rollout_reward_func/std": 0.6157501935958862,
      "sampling/importance_sampling_ratio/max": 2.3025896549224854,
      "sampling/importance_sampling_ratio/mean": 0.8662018179893494,
      "sampling/importance_sampling_ratio/min": 3.1194227874148097e-18,
      "sampling/sampling_logp_difference/max": 22.734416961669922,
      "sampling/sampling_logp_difference/mean": 0.21409714221954346,
      "step": 401,
      "step_time": 11.971376782021252
    },
    {
      "clip_ratio/high_max": 0.012500000186264515,
      "clip_ratio/high_mean": 0.0062500000931322575,
      "clip_ratio/low_mean": 0.0024038462433964014,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008653846336528659,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 659.0,
      "completions/max_terminated_length": 659.0,
      "completions/mean_length": 164.65625,
      "completions/mean_terminated_length": 169.4516143798828,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.48819055408239365,
      "epoch": 0.00804,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.102341651916504,
      "kl": 0.45569333527237177,
      "learning_rate": 7.146925122328581e-06,
      "loss": -0.0923,
      "num_tokens": 16203227.0,
      "reward": 0.6753125190734863,
      "reward_std": 0.7388067245483398,
      "rewards/rollout_reward_func/mean": 0.6753125190734863,
      "rewards/rollout_reward_func/std": 0.7042222023010254,
      "sampling/importance_sampling_ratio/max": 1.8704233169555664,
      "sampling/importance_sampling_ratio/mean": 1.0143401622772217,
      "sampling/importance_sampling_ratio/min": 0.46171876788139343,
      "sampling/sampling_logp_difference/max": 0.6459083557128906,
      "sampling/sampling_logp_difference/mean": 0.03796801716089249,
      "step": 402,
      "step_time": 12.366625673967064
    },
    {
      "clip_ratio/high_max": 0.007195723708719015,
      "clip_ratio/high_mean": 0.0035978618543595076,
      "clip_ratio/low_mean": 0.002464657765813172,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006062519620172679,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 916.0,
      "completions/max_terminated_length": 916.0,
      "completions/mean_length": 294.375,
      "completions/mean_terminated_length": 288.96551513671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5234600380063057,
      "epoch": 0.00806,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1870527267456055,
      "kl": 0.5335472952574492,
      "learning_rate": 7.133694920819958e-06,
      "loss": -0.0279,
      "num_tokens": 16246218.0,
      "reward": 0.6297279596328735,
      "reward_std": 0.5651939511299133,
      "rewards/rollout_reward_func/mean": 0.6297279596328735,
      "rewards/rollout_reward_func/std": 0.5626041293144226,
      "sampling/importance_sampling_ratio/max": 1.5646034479141235,
      "sampling/importance_sampling_ratio/mean": 0.9887101650238037,
      "sampling/importance_sampling_ratio/min": 0.3149046003818512,
      "sampling/sampling_logp_difference/max": 1.2121906280517578,
      "sampling/sampling_logp_difference/mean": 0.03833268582820892,
      "step": 403,
      "step_time": 13.520092982013011
    },
    {
      "clip_ratio/high_max": 0.011498918058350682,
      "clip_ratio/high_mean": 0.005749459029175341,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006726021529175341,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 262.96875,
      "completions/mean_terminated_length": 251.48147583007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.586882246658206,
      "epoch": 0.00808,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2693653106689453,
      "kl": 1.96343326382339,
      "learning_rate": 7.12045306277795e-06,
      "loss": -0.0601,
      "num_tokens": 16287898.0,
      "reward": 0.6039737462997437,
      "reward_std": 0.6194442510604858,
      "rewards/rollout_reward_func/mean": 0.6039737462997437,
      "rewards/rollout_reward_func/std": 0.6880065202713013,
      "sampling/importance_sampling_ratio/max": 1.6717033386230469,
      "sampling/importance_sampling_ratio/mean": 0.9185527563095093,
      "sampling/importance_sampling_ratio/min": 1.1400779360967452e-21,
      "sampling/sampling_logp_difference/max": 20.004440307617188,
      "sampling/sampling_logp_difference/mean": 0.15250247716903687,
      "step": 404,
      "step_time": 12.09793692996027
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 605.0,
      "completions/max_terminated_length": 605.0,
      "completions/mean_length": 205.78125,
      "completions/mean_terminated_length": 178.44827270507812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5462193861603737,
      "epoch": 0.0081,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.311851978302002,
      "kl": 1.2973613925278187,
      "learning_rate": 7.107199722871614e-06,
      "loss": 0.0508,
      "num_tokens": 16326918.0,
      "reward": 0.6509398818016052,
      "reward_std": 0.695960283279419,
      "rewards/rollout_reward_func/mean": 0.6509398818016052,
      "rewards/rollout_reward_func/std": 0.7238918542861938,
      "sampling/importance_sampling_ratio/max": 1.8714520931243896,
      "sampling/importance_sampling_ratio/mean": 0.9905357360839844,
      "sampling/importance_sampling_ratio/min": 0.38724595308303833,
      "sampling/sampling_logp_difference/max": 0.9115378856658936,
      "sampling/sampling_logp_difference/mean": 0.04472662881016731,
      "step": 405,
      "step_time": 12.46203777899791
    },
    {
      "clip_ratio/high_max": 0.011362390592694283,
      "clip_ratio/high_mean": 0.005681195296347141,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005681195296347141,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 288.1875,
      "completions/mean_terminated_length": 271.5862121582031,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5351509265601635,
      "epoch": 0.00812,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1110976934432983,
      "kl": 0.48420488368719816,
      "learning_rate": 7.093935075921465e-06,
      "loss": -0.0513,
      "num_tokens": 16369680.0,
      "reward": 0.5784374475479126,
      "reward_std": 0.6944984793663025,
      "rewards/rollout_reward_func/mean": 0.5784374475479126,
      "rewards/rollout_reward_func/std": 0.7383901476860046,
      "sampling/importance_sampling_ratio/max": 1.483978509902954,
      "sampling/importance_sampling_ratio/mean": 0.8825851678848267,
      "sampling/importance_sampling_ratio/min": 4.6120081975686844e-08,
      "sampling/sampling_logp_difference/max": 6.211292266845703,
      "sampling/sampling_logp_difference/mean": 0.07691603899002075,
      "step": 406,
      "step_time": 13.093106386033469
    },
    {
      "clip_ratio/high_max": 0.00854223920032382,
      "clip_ratio/high_mean": 0.00427111960016191,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00427111960016191,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 901.0,
      "completions/max_terminated_length": 901.0,
      "completions/mean_length": 281.9375,
      "completions/mean_terminated_length": 282.923095703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.560358164831996,
      "epoch": 0.00814,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3614755868911743,
      "kl": 0.2746063335798681,
      "learning_rate": 7.0806592968971624e-06,
      "loss": -0.0561,
      "num_tokens": 16411783.0,
      "reward": 0.8865624666213989,
      "reward_std": 0.3042775094509125,
      "rewards/rollout_reward_func/mean": 0.8865624666213989,
      "rewards/rollout_reward_func/std": 0.339445561170578,
      "sampling/importance_sampling_ratio/max": 1.6081476211547852,
      "sampling/importance_sampling_ratio/mean": 0.9619937539100647,
      "sampling/importance_sampling_ratio/min": 0.3076362907886505,
      "sampling/sampling_logp_difference/max": 0.7250819206237793,
      "sampling/sampling_logp_difference/mean": 0.0352339893579483,
      "step": 407,
      "step_time": 13.173170108988415
    },
    {
      "clip_ratio/high_max": 0.005859375,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.00390625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0068359375,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 635.0,
      "completions/max_terminated_length": 635.0,
      "completions/mean_length": 179.4375,
      "completions/mean_terminated_length": 202.7857208251953,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.552070552483201,
      "epoch": 0.00816,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4207940101623535,
      "kl": 0.39555522333830595,
      "learning_rate": 7.067372560915205e-06,
      "loss": -0.047,
      "num_tokens": 16449407.0,
      "reward": 0.6362500190734863,
      "reward_std": 0.6253925561904907,
      "rewards/rollout_reward_func/mean": 0.6362500190734863,
      "rewards/rollout_reward_func/std": 0.7060555815696716,
      "sampling/importance_sampling_ratio/max": 1.5459027290344238,
      "sampling/importance_sampling_ratio/mean": 1.0292199850082397,
      "sampling/importance_sampling_ratio/min": 0.5425012707710266,
      "sampling/sampling_logp_difference/max": 0.6122705936431885,
      "sampling/sampling_logp_difference/mean": 0.0346788614988327,
      "step": 408,
      "step_time": 12.457205063023139
    },
    {
      "clip_ratio/high_max": 0.006623641354963183,
      "clip_ratio/high_mean": 0.0033118206774815917,
      "clip_ratio/low_mean": 0.00390625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007218070677481592,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 647.0,
      "completions/mean_length": 309.84375,
      "completions/mean_terminated_length": 288.137939453125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.507036866620183,
      "epoch": 0.00818,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8216016888618469,
      "kl": 0.38674365007318556,
      "learning_rate": 7.054075043236623e-06,
      "loss": -0.2477,
      "num_tokens": 16492353.0,
      "reward": 0.7024112343788147,
      "reward_std": 0.5034451484680176,
      "rewards/rollout_reward_func/mean": 0.7024112343788147,
      "rewards/rollout_reward_func/std": 0.6527315974235535,
      "sampling/importance_sampling_ratio/max": 1.6015545129776,
      "sampling/importance_sampling_ratio/mean": 0.9558461308479309,
      "sampling/importance_sampling_ratio/min": 0.3548782765865326,
      "sampling/sampling_logp_difference/max": 0.8082339763641357,
      "sampling/sampling_logp_difference/mean": 0.04025811329483986,
      "step": 409,
      "step_time": 12.480635540996445
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 638.0,
      "completions/max_terminated_length": 638.0,
      "completions/mean_length": 238.03125,
      "completions/mean_terminated_length": 239.7586212158203,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5270455405116081,
      "epoch": 0.0082,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3342574834823608,
      "kl": 0.2987032267265022,
      "learning_rate": 7.040766919264664e-06,
      "loss": -0.1464,
      "num_tokens": 16532847.0,
      "reward": 0.692187488079071,
      "reward_std": 0.523415207862854,
      "rewards/rollout_reward_func/mean": 0.692187488079071,
      "rewards/rollout_reward_func/std": 0.5990569591522217,
      "sampling/importance_sampling_ratio/max": 2.6161715984344482,
      "sampling/importance_sampling_ratio/mean": 0.9943006634712219,
      "sampling/importance_sampling_ratio/min": 0.45232364535331726,
      "sampling/sampling_logp_difference/max": 0.7557568550109863,
      "sampling/sampling_logp_difference/mean": 0.0416591539978981,
      "step": 410,
      "step_time": 11.987561600035406
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.009457236854359508,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011410361854359508,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 279.75,
      "completions/mean_terminated_length": 269.7241516113281,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5624054800719023,
      "epoch": 0.00822,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1601942777633667,
      "kl": 0.769990558270365,
      "learning_rate": 7.027448364542479e-06,
      "loss": -0.0408,
      "num_tokens": 16574902.0,
      "reward": 0.6050771474838257,
      "reward_std": 0.6471419334411621,
      "rewards/rollout_reward_func/mean": 0.6050771474838257,
      "rewards/rollout_reward_func/std": 0.681192934513092,
      "sampling/importance_sampling_ratio/max": 1.7873121500015259,
      "sampling/importance_sampling_ratio/mean": 0.9790860414505005,
      "sampling/importance_sampling_ratio/min": 0.26270610094070435,
      "sampling/sampling_logp_difference/max": 1.2439324855804443,
      "sampling/sampling_logp_difference/mean": 0.040431953966617584,
      "step": 411,
      "step_time": 13.384075150010176
    },
    {
      "clip_ratio/high_max": 0.023722166195511818,
      "clip_ratio/high_mean": 0.011861083097755909,
      "clip_ratio/low_mean": 0.017361111124046147,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.029222194221802056,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 663.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 290.90625,
      "completions/mean_terminated_length": 299.774169921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.45213753543794155,
      "epoch": 0.00824,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.933573603630066,
      "kl": 1.7775249760597944,
      "learning_rate": 7.01411955475081e-06,
      "loss": 0.048,
      "num_tokens": 16617840.0,
      "reward": 0.7986612319946289,
      "reward_std": 0.422925740480423,
      "rewards/rollout_reward_func/mean": 0.7986612319946289,
      "rewards/rollout_reward_func/std": 0.569767415523529,
      "sampling/importance_sampling_ratio/max": 1.693803310394287,
      "sampling/importance_sampling_ratio/mean": 1.0451971292495728,
      "sampling/importance_sampling_ratio/min": 0.3449190855026245,
      "sampling/sampling_logp_difference/max": 1.092379093170166,
      "sampling/sampling_logp_difference/mean": 0.04219285398721695,
      "step": 412,
      "step_time": 12.206955886998912
    },
    {
      "clip_ratio/high_max": 0.01774259889498353,
      "clip_ratio/high_mean": 0.008871299447491765,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008871299447491765,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 961.0,
      "completions/max_terminated_length": 961.0,
      "completions/mean_length": 198.875,
      "completions/mean_terminated_length": 183.10000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4748345799744129,
      "epoch": 0.00826,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8704678416252136,
      "kl": 1.274025003425777,
      "learning_rate": 7.000780665705665e-06,
      "loss": -0.1045,
      "num_tokens": 16656040.0,
      "reward": 0.6646875143051147,
      "reward_std": 0.6421021819114685,
      "rewards/rollout_reward_func/mean": 0.6646875143051147,
      "rewards/rollout_reward_func/std": 0.6583801507949829,
      "sampling/importance_sampling_ratio/max": 1.6753170490264893,
      "sampling/importance_sampling_ratio/mean": 0.9865659475326538,
      "sampling/importance_sampling_ratio/min": 0.2955247461795807,
      "sampling/sampling_logp_difference/max": 0.8916049003601074,
      "sampling/sampling_logp_difference/mean": 0.04260970652103424,
      "step": 413,
      "step_time": 12.859573085966986
    },
    {
      "clip_ratio/high_max": 0.0036764706019312143,
      "clip_ratio/high_mean": 0.0018382353009656072,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009650735300965607,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 1191.0,
      "completions/max_terminated_length": 1191.0,
      "completions/mean_length": 319.5,
      "completions/mean_terminated_length": 311.8387145996094,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.49765278585255146,
      "epoch": 0.00828,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8181155920028687,
      "kl": 0.4839303530752659,
      "learning_rate": 6.987431873356011e-06,
      "loss": -0.0819,
      "num_tokens": 16699065.0,
      "reward": 0.5228148698806763,
      "reward_std": 0.5337269306182861,
      "rewards/rollout_reward_func/mean": 0.5228148698806763,
      "rewards/rollout_reward_func/std": 0.677609384059906,
      "sampling/importance_sampling_ratio/max": 1.7812013626098633,
      "sampling/importance_sampling_ratio/mean": 0.9582885503768921,
      "sampling/importance_sampling_ratio/min": 0.2975458800792694,
      "sampling/sampling_logp_difference/max": 0.7044844627380371,
      "sampling/sampling_logp_difference/mean": 0.0402049794793129,
      "step": 414,
      "step_time": 14.93548355497478
    },
    {
      "clip_ratio/high_max": 0.018342391354963183,
      "clip_ratio/high_mean": 0.009171195677481592,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009171195677481592,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 618.0,
      "completions/max_terminated_length": 586.0,
      "completions/mean_length": 275.5625,
      "completions/mean_terminated_length": 272.8000183105469,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.39584807213395834,
      "epoch": 0.0083,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2432128190994263,
      "kl": 0.31749406456947327,
      "learning_rate": 6.97407335378144e-06,
      "loss": 0.1216,
      "num_tokens": 16740483.0,
      "reward": 0.7156249284744263,
      "reward_std": 0.500478208065033,
      "rewards/rollout_reward_func/mean": 0.7156249284744263,
      "rewards/rollout_reward_func/std": 0.6005530953407288,
      "sampling/importance_sampling_ratio/max": 1.7361488342285156,
      "sampling/importance_sampling_ratio/mean": 0.9993247389793396,
      "sampling/importance_sampling_ratio/min": 1.9763310779574716e-11,
      "sampling/sampling_logp_difference/max": 13.689481735229492,
      "sampling/sampling_logp_difference/mean": 0.09150288254022598,
      "step": 415,
      "step_time": 11.81139427202288
    },
    {
      "clip_ratio/high_max": 0.006265664240345359,
      "clip_ratio/high_mean": 0.0031328321201726794,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031328321201726794,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 276.125,
      "completions/mean_terminated_length": 276.125,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.49322522431612015,
      "epoch": 0.00832,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8898942470550537,
      "kl": 0.5124372313730419,
      "learning_rate": 6.960705283189857e-06,
      "loss": -0.013,
      "num_tokens": 16782946.0,
      "reward": 0.6768749952316284,
      "reward_std": 0.6146007776260376,
      "rewards/rollout_reward_func/mean": 0.6768749952316284,
      "rewards/rollout_reward_func/std": 0.6514917612075806,
      "sampling/importance_sampling_ratio/max": 1.3485044240951538,
      "sampling/importance_sampling_ratio/mean": 0.8062540292739868,
      "sampling/importance_sampling_ratio/min": 3.197003550781119e-08,
      "sampling/sampling_logp_difference/max": 17.3125,
      "sampling/sampling_logp_difference/mean": 0.19177857041358948,
      "step": 416,
      "step_time": 12.628689462988405
    },
    {
      "clip_ratio/high_max": 0.02024529571644962,
      "clip_ratio/high_mean": 0.01012264785822481,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01012264785822481,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 621.0,
      "completions/max_terminated_length": 621.0,
      "completions/mean_length": 276.375,
      "completions/mean_terminated_length": 296.89288330078125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.48263458255678415,
      "epoch": 0.00834,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0457464456558228,
      "kl": 0.8342114454135299,
      "learning_rate": 6.94732783791515e-06,
      "loss": -0.0458,
      "num_tokens": 16825265.0,
      "reward": 0.5809375047683716,
      "reward_std": 0.6370315551757812,
      "rewards/rollout_reward_func/mean": 0.5809375047683716,
      "rewards/rollout_reward_func/std": 0.6654338836669922,
      "sampling/importance_sampling_ratio/max": 1.3138495683670044,
      "sampling/importance_sampling_ratio/mean": 0.9472640752792358,
      "sampling/importance_sampling_ratio/min": 0.40655866265296936,
      "sampling/sampling_logp_difference/max": 0.8092707395553589,
      "sampling/sampling_logp_difference/mean": 0.0378413125872612,
      "step": 417,
      "step_time": 12.605955008999445
    },
    {
      "clip_ratio/high_max": 0.010312499944120646,
      "clip_ratio/high_mean": 0.005156249972060323,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005156249972060323,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 216.1875,
      "completions/mean_terminated_length": 220.7666778564453,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4944626223295927,
      "epoch": 0.00836,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0597751140594482,
      "kl": 0.850645134691149,
      "learning_rate": 6.933941194414866e-06,
      "loss": -0.1179,
      "num_tokens": 16864780.0,
      "reward": 0.5706654787063599,
      "reward_std": 0.6895030736923218,
      "rewards/rollout_reward_func/mean": 0.5706654787063599,
      "rewards/rollout_reward_func/std": 0.7266435027122498,
      "sampling/importance_sampling_ratio/max": 1.4968596696853638,
      "sampling/importance_sampling_ratio/mean": 1.0321904420852661,
      "sampling/importance_sampling_ratio/min": 0.37201324105262756,
      "sampling/sampling_logp_difference/max": 1.0806715488433838,
      "sampling/sampling_logp_difference/mean": 0.03794832527637482,
      "step": 418,
      "step_time": 11.92280614899937
    },
    {
      "clip_ratio/high_max": 0.004464285913854837,
      "clip_ratio/high_mean": 0.0022321429569274187,
      "clip_ratio/low_mean": 0.0020833334419876337,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004315476398915052,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 248.15625,
      "completions/mean_terminated_length": 249.16668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4719155579805374,
      "epoch": 0.00838,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8413008451461792,
      "kl": 0.7625369275920093,
      "learning_rate": 6.920545529267882e-06,
      "loss": -0.0669,
      "num_tokens": 16904914.0,
      "reward": 0.7328125238418579,
      "reward_std": 0.41779428720474243,
      "rewards/rollout_reward_func/mean": 0.7328125238418579,
      "rewards/rollout_reward_func/std": 0.5310745239257812,
      "sampling/importance_sampling_ratio/max": 1.6254757642745972,
      "sampling/importance_sampling_ratio/mean": 0.9744167327880859,
      "sampling/importance_sampling_ratio/min": 0.28282126784324646,
      "sampling/sampling_logp_difference/max": 1.2734017372131348,
      "sampling/sampling_logp_difference/mean": 0.03658945858478546,
      "step": 419,
      "step_time": 12.5873405860184
    },
    {
      "clip_ratio/high_max": 0.00947229890152812,
      "clip_ratio/high_mean": 0.00473614945076406,
      "clip_ratio/low_mean": 0.0015625000232830644,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006298649590462446,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 664.0,
      "completions/max_terminated_length": 664.0,
      "completions/mean_length": 341.25,
      "completions/mean_terminated_length": 332.5185241699219,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5070328209549189,
      "epoch": 0.0084,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6650060415267944,
      "kl": 0.7640459910035133,
      "learning_rate": 6.907141019172076e-06,
      "loss": 0.0503,
      "num_tokens": 16950027.0,
      "reward": 0.6540625095367432,
      "reward_std": 0.6162254810333252,
      "rewards/rollout_reward_func/mean": 0.6540625095367432,
      "rewards/rollout_reward_func/std": 0.6068685054779053,
      "sampling/importance_sampling_ratio/max": 2.158294916152954,
      "sampling/importance_sampling_ratio/mean": 1.061568021774292,
      "sampling/importance_sampling_ratio/min": 0.12779514491558075,
      "sampling/sampling_logp_difference/max": 0.8505127429962158,
      "sampling/sampling_logp_difference/mean": 0.04011552035808563,
      "step": 420,
      "step_time": 12.675646514020627
    },
    {
      "clip_ratio/high_max": 0.009611312998458743,
      "clip_ratio/high_mean": 0.0048056564992293715,
      "clip_ratio/low_mean": 0.0020833334419876337,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006888989941217005,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 661.0,
      "completions/max_terminated_length": 661.0,
      "completions/mean_length": 322.53125,
      "completions/mean_terminated_length": 326.70965576171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4002712983638048,
      "epoch": 0.00842,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1951417922973633,
      "kl": 0.3766204174607992,
      "learning_rate": 6.893727840941997e-06,
      "loss": 0.0769,
      "num_tokens": 16993373.0,
      "reward": 0.6106249690055847,
      "reward_std": 0.6964248418807983,
      "rewards/rollout_reward_func/mean": 0.6106249690055847,
      "rewards/rollout_reward_func/std": 0.6731578707695007,
      "sampling/importance_sampling_ratio/max": 2.2397820949554443,
      "sampling/importance_sampling_ratio/mean": 0.9749435782432556,
      "sampling/importance_sampling_ratio/min": 0.34847861528396606,
      "sampling/sampling_logp_difference/max": 0.6271843910217285,
      "sampling/sampling_logp_difference/mean": 0.034197427332401276,
      "step": 421,
      "step_time": 12.277445694009657
    },
    {
      "clip_ratio/high_max": 0.004267939832061529,
      "clip_ratio/high_mean": 0.0021339699160307646,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031105324160307646,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 623.0,
      "completions/max_terminated_length": 623.0,
      "completions/mean_length": 290.75,
      "completions/mean_terminated_length": 285.9000244140625,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4465925972908735,
      "epoch": 0.00844,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5965332984924316,
      "kl": 0.41835908126085997,
      "learning_rate": 6.880306171506535e-06,
      "loss": 0.1121,
      "num_tokens": 17035481.0,
      "reward": 0.828125,
      "reward_std": 0.3690824508666992,
      "rewards/rollout_reward_func/mean": 0.828125,
      "rewards/rollout_reward_func/std": 0.4642888009548187,
      "sampling/importance_sampling_ratio/max": 1.7017936706542969,
      "sampling/importance_sampling_ratio/mean": 0.9381133317947388,
      "sampling/importance_sampling_ratio/min": 1.0704559924218696e-15,
      "sampling/sampling_logp_difference/max": 20.995786666870117,
      "sampling/sampling_logp_difference/mean": 0.11316445469856262,
      "step": 422,
      "step_time": 12.701519777983776
    },
    {
      "clip_ratio/high_max": 0.024218750186264515,
      "clip_ratio/high_mean": 0.012109375093132257,
      "clip_ratio/low_mean": 0.0035978618543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015707236947491765,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 779.0,
      "completions/max_terminated_length": 779.0,
      "completions/mean_length": 223.65625,
      "completions/mean_terminated_length": 220.48147583007812,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.43374966736882925,
      "epoch": 0.00846,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.241971969604492,
      "kl": 0.5563939334824681,
      "learning_rate": 6.866876187906582e-06,
      "loss": -0.098,
      "num_tokens": 17074119.0,
      "reward": 0.7622501850128174,
      "reward_std": 0.5702544450759888,
      "rewards/rollout_reward_func/mean": 0.7622501850128174,
      "rewards/rollout_reward_func/std": 0.6321989893913269,
      "sampling/importance_sampling_ratio/max": 2.4011049270629883,
      "sampling/importance_sampling_ratio/mean": 1.0061414241790771,
      "sampling/importance_sampling_ratio/min": 0.3183220326900482,
      "sampling/sampling_logp_difference/max": 0.9815754890441895,
      "sampling/sampling_logp_difference/mean": 0.03922092914581299,
      "step": 423,
      "step_time": 12.884381911004311
    },
    {
      "clip_ratio/high_max": 0.051636905409395695,
      "clip_ratio/high_mean": 0.02799868630245328,
      "clip_ratio/low_mean": 0.01953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.04752993630245328,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 1173.0,
      "completions/max_terminated_length": 1173.0,
      "completions/mean_length": 231.03125,
      "completions/mean_terminated_length": 237.45159912109375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.48893736861646175,
      "epoch": 0.00848,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9410653710365295,
      "kl": 1.0818658424541354,
      "learning_rate": 6.8534380672927e-06,
      "loss": -0.0249,
      "num_tokens": 17113623.0,
      "reward": 0.643348753452301,
      "reward_std": 0.7431066036224365,
      "rewards/rollout_reward_func/mean": 0.643348753452301,
      "rewards/rollout_reward_func/std": 0.7063663601875305,
      "sampling/importance_sampling_ratio/max": 1.4912580251693726,
      "sampling/importance_sampling_ratio/mean": 0.8621478080749512,
      "sampling/importance_sampling_ratio/min": 2.4916662757319158e-24,
      "sampling/sampling_logp_difference/max": 21.053050994873047,
      "sampling/sampling_logp_difference/mean": 0.15212012827396393,
      "step": 424,
      "step_time": 14.59533419397485
    },
    {
      "clip_ratio/high_max": 0.003289473708719015,
      "clip_ratio/high_mean": 0.0016447368543595076,
      "clip_ratio/low_mean": 0.00390625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0055509868543595076,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 641.0,
      "completions/max_terminated_length": 641.0,
      "completions/mean_length": 228.625,
      "completions/mean_terminated_length": 233.51724243164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.46877187490463257,
      "epoch": 0.0085,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1196378469467163,
      "kl": 0.43076953385025263,
      "learning_rate": 6.839991986922785e-06,
      "loss": -0.0314,
      "num_tokens": 17153011.0,
      "reward": 0.8274999856948853,
      "reward_std": 0.446412056684494,
      "rewards/rollout_reward_func/mean": 0.8274999856948853,
      "rewards/rollout_reward_func/std": 0.46525052189826965,
      "sampling/importance_sampling_ratio/max": 1.845848560333252,
      "sampling/importance_sampling_ratio/mean": 0.9524997472763062,
      "sampling/importance_sampling_ratio/min": 0.4908904731273651,
      "sampling/sampling_logp_difference/max": 0.7006587982177734,
      "sampling/sampling_logp_difference/mean": 0.0371415913105011,
      "step": 425,
      "step_time": 12.500782948030974
    },
    {
      "clip_ratio/high_max": 0.01953125,
      "clip_ratio/high_mean": 0.009765625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009765625,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 602.0,
      "completions/max_terminated_length": 602.0,
      "completions/mean_length": 257.03125,
      "completions/mean_terminated_length": 264.80645751953125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4081958597525954,
      "epoch": 0.00852,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0928820371627808,
      "kl": 0.3663415862247348,
      "learning_rate": 6.826538124159727e-06,
      "loss": -0.17,
      "num_tokens": 17194126.0,
      "reward": 0.7756249904632568,
      "reward_std": 0.43351855874061584,
      "rewards/rollout_reward_func/mean": 0.7756249904632568,
      "rewards/rollout_reward_func/std": 0.5038172602653503,
      "sampling/importance_sampling_ratio/max": 1.5865775346755981,
      "sampling/importance_sampling_ratio/mean": 1.053799033164978,
      "sampling/importance_sampling_ratio/min": 0.5691953301429749,
      "sampling/sampling_logp_difference/max": 0.4720759391784668,
      "sampling/sampling_logp_difference/mean": 0.031528178602457047,
      "step": 426,
      "step_time": 12.612342283959151
    },
    {
      "clip_ratio/high_max": 0.013671875,
      "clip_ratio/high_mean": 0.0068359375,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0224609375,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 522.0,
      "completions/mean_length": 207.03125,
      "completions/mean_terminated_length": 175.57693481445312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4992123432457447,
      "epoch": 0.00854,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2878791093826294,
      "kl": 5.671193053945899,
      "learning_rate": 6.813076656469068e-06,
      "loss": -0.1023,
      "num_tokens": 17233795.0,
      "reward": 0.6624999642372131,
      "reward_std": 0.5723401308059692,
      "rewards/rollout_reward_func/mean": 0.6624999642372131,
      "rewards/rollout_reward_func/std": 0.6754973530769348,
      "sampling/importance_sampling_ratio/max": 1.7179914712905884,
      "sampling/importance_sampling_ratio/mean": 0.8861059546470642,
      "sampling/importance_sampling_ratio/min": 2.1583648538764528e-17,
      "sampling/sampling_logp_difference/max": 22.72728157043457,
      "sampling/sampling_logp_difference/mean": 0.12958890199661255,
      "step": 427,
      "step_time": 12.120888422010466
    },
    {
      "clip_ratio/high_max": 0.0052083334885537624,
      "clip_ratio/high_mean": 0.0026041667442768812,
      "clip_ratio/low_mean": 0.0034722222480922937,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006076388992369175,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 630.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 249.28125,
      "completions/mean_terminated_length": 256.80645751953125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.43871577084064484,
      "epoch": 0.00856,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1199486255645752,
      "kl": 0.35207200795412064,
      "learning_rate": 6.799607761416671e-06,
      "loss": 0.0734,
      "num_tokens": 17273922.0,
      "reward": 0.6753125190734863,
      "reward_std": 0.48972630500793457,
      "rewards/rollout_reward_func/mean": 0.6753125190734863,
      "rewards/rollout_reward_func/std": 0.4773784875869751,
      "sampling/importance_sampling_ratio/max": 1.8972890377044678,
      "sampling/importance_sampling_ratio/mean": 1.0424954891204834,
      "sampling/importance_sampling_ratio/min": 0.35927119851112366,
      "sampling/sampling_logp_difference/max": 1.026465892791748,
      "sampling/sampling_logp_difference/mean": 0.03497738391160965,
      "step": 428,
      "step_time": 12.561651976022404
    },
    {
      "clip_ratio/high_max": 0.0029761905316263437,
      "clip_ratio/high_mean": 0.0014880952658131719,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0014880952658131719,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 643.0,
      "completions/max_terminated_length": 643.0,
      "completions/mean_length": 296.71875,
      "completions/mean_terminated_length": 296.71875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4065790455788374,
      "epoch": 0.00858,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4771548509597778,
      "kl": 1.1643222658894956,
      "learning_rate": 6.786131616666364e-06,
      "loss": 0.0074,
      "num_tokens": 17316563.0,
      "reward": 0.5447646379470825,
      "reward_std": 0.7231864333152771,
      "rewards/rollout_reward_func/mean": 0.5447646379470825,
      "rewards/rollout_reward_func/std": 0.696216344833374,
      "sampling/importance_sampling_ratio/max": 1.3581624031066895,
      "sampling/importance_sampling_ratio/mean": 0.9193655252456665,
      "sampling/importance_sampling_ratio/min": 0.34168288111686707,
      "sampling/sampling_logp_difference/max": 1.0304560661315918,
      "sampling/sampling_logp_difference/mean": 0.03895876556634903,
      "step": 429,
      "step_time": 11.749956549989292
    },
    {
      "clip_ratio/high_max": 0.003289473708719015,
      "clip_ratio/high_mean": 0.0016447368543595076,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0016447368543595076,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 664.0,
      "completions/max_terminated_length": 664.0,
      "completions/mean_length": 256.09375,
      "completions/mean_terminated_length": 271.5517272949219,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4999992726370692,
      "epoch": 0.0086,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1900756359100342,
      "kl": 0.4183416599407792,
      "learning_rate": 6.772648399977606e-06,
      "loss": -0.0658,
      "num_tokens": 17357743.0,
      "reward": 0.7778124809265137,
      "reward_std": 0.5677804350852966,
      "rewards/rollout_reward_func/mean": 0.7778124809265137,
      "rewards/rollout_reward_func/std": 0.5685393214225769,
      "sampling/importance_sampling_ratio/max": 1.9874062538146973,
      "sampling/importance_sampling_ratio/mean": 0.9932489395141602,
      "sampling/importance_sampling_ratio/min": 0.26361724734306335,
      "sampling/sampling_logp_difference/max": 1.054381251335144,
      "sampling/sampling_logp_difference/mean": 0.03591246157884598,
      "step": 430,
      "step_time": 12.452118802000768
    },
    {
      "clip_ratio/high_max": 0.0062500000931322575,
      "clip_ratio/high_mean": 0.0031250000465661287,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0053571430034935474,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 614.0,
      "completions/max_terminated_length": 614.0,
      "completions/mean_length": 255.84375,
      "completions/mean_terminated_length": 270.137939453125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4517997894436121,
      "epoch": 0.00862,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9174874424934387,
      "kl": 0.5016912519931793,
      "learning_rate": 6.759158289203141e-06,
      "loss": -0.0167,
      "num_tokens": 17399488.0,
      "reward": 0.7062499523162842,
      "reward_std": 0.565519392490387,
      "rewards/rollout_reward_func/mean": 0.7062499523162842,
      "rewards/rollout_reward_func/std": 0.5955845713615417,
      "sampling/importance_sampling_ratio/max": 1.5732961893081665,
      "sampling/importance_sampling_ratio/mean": 0.7954416275024414,
      "sampling/importance_sampling_ratio/min": 2.2447200282851512e-18,
      "sampling/sampling_logp_difference/max": 25.625,
      "sampling/sampling_logp_difference/mean": 0.362474650144577,
      "step": 431,
      "step_time": 12.345256859989604
    },
    {
      "clip_ratio/high_max": 0.0036764706019312143,
      "clip_ratio/high_mean": 0.0018382353009656072,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0018382353009656072,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 766.0,
      "completions/max_terminated_length": 766.0,
      "completions/mean_length": 269.78125,
      "completions/mean_terminated_length": 284.82757568359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4617954883724451,
      "epoch": 0.00864,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1845248937606812,
      "kl": 0.5446462631225586,
      "learning_rate": 6.745661462286648e-06,
      "loss": -0.066,
      "num_tokens": 17440661.0,
      "reward": 0.7368749976158142,
      "reward_std": 0.43682628870010376,
      "rewards/rollout_reward_func/mean": 0.7368749976158142,
      "rewards/rollout_reward_func/std": 0.5113198161125183,
      "sampling/importance_sampling_ratio/max": 1.7257156372070312,
      "sampling/importance_sampling_ratio/mean": 0.9687628149986267,
      "sampling/importance_sampling_ratio/min": 0.30608636140823364,
      "sampling/sampling_logp_difference/max": 1.1121015548706055,
      "sampling/sampling_logp_difference/mean": 0.0381525382399559,
      "step": 432,
      "step_time": 12.198267582978588
    },
    {
      "clip_ratio/high_max": 0.005629595601931214,
      "clip_ratio/high_mean": 0.002814797800965607,
      "clip_ratio/low_mean": 0.004199604853056371,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007014402654021978,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 291.5625,
      "completions/mean_terminated_length": 279.6785888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.535528801381588,
      "epoch": 0.00866,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0182440280914307,
      "kl": 0.8342572683468461,
      "learning_rate": 6.7321580972603996e-06,
      "loss": -0.2207,
      "num_tokens": 17484517.0,
      "reward": 0.6078751683235168,
      "reward_std": 0.7269859313964844,
      "rewards/rollout_reward_func/mean": 0.6078751683235168,
      "rewards/rollout_reward_func/std": 0.7280662655830383,
      "sampling/importance_sampling_ratio/max": 1.4293315410614014,
      "sampling/importance_sampling_ratio/mean": 0.8327892422676086,
      "sampling/importance_sampling_ratio/min": 4.857969027050046e-10,
      "sampling/sampling_logp_difference/max": 21.56136703491211,
      "sampling/sampling_logp_difference/mean": 0.1463223099708557,
      "step": 433,
      "step_time": 12.927738270998816
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009765625,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 678.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 253.84375,
      "completions/mean_terminated_length": 219.44827270507812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4706661254167557,
      "epoch": 0.00868,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2584129571914673,
      "kl": 0.6376317543908954,
      "learning_rate": 6.718648372242909e-06,
      "loss": -0.1091,
      "num_tokens": 17525211.0,
      "reward": 0.5441251993179321,
      "reward_std": 0.6396360993385315,
      "rewards/rollout_reward_func/mean": 0.5441251993179321,
      "rewards/rollout_reward_func/std": 0.6926717162132263,
      "sampling/importance_sampling_ratio/max": 1.5095497369766235,
      "sampling/importance_sampling_ratio/mean": 1.0758594274520874,
      "sampling/importance_sampling_ratio/min": 0.417688250541687,
      "sampling/sampling_logp_difference/max": 0.6929893493652344,
      "sampling/sampling_logp_difference/mean": 0.030007075518369675,
      "step": 434,
      "step_time": 12.519802852999419
    },
    {
      "clip_ratio/high_max": 0.0024999999441206455,
      "clip_ratio/high_mean": 0.0012499999720603228,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00571428588591516,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 595.0,
      "completions/max_terminated_length": 595.0,
      "completions/mean_length": 185.5,
      "completions/mean_terminated_length": 182.51612854003906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4278694875538349,
      "epoch": 0.0087,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8177950978279114,
      "kl": 0.5246847234666348,
      "learning_rate": 6.705132465436579e-06,
      "loss": -0.2251,
      "num_tokens": 17563645.0,
      "reward": 0.8637499809265137,
      "reward_std": 0.32077616453170776,
      "rewards/rollout_reward_func/mean": 0.8637499809265137,
      "rewards/rollout_reward_func/std": 0.46058690547943115,
      "sampling/importance_sampling_ratio/max": 1.472721815109253,
      "sampling/importance_sampling_ratio/mean": 0.9678487777709961,
      "sampling/importance_sampling_ratio/min": 0.35727375745773315,
      "sampling/sampling_logp_difference/max": 0.862879753112793,
      "sampling/sampling_logp_difference/mean": 0.03087221458554268,
      "step": 435,
      "step_time": 12.006180542026414
    },
    {
      "clip_ratio/high_max": 0.02238669595681131,
      "clip_ratio/high_mean": 0.011193347978405654,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011193347978405654,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 1045.0,
      "completions/max_terminated_length": 1045.0,
      "completions/mean_length": 258.84375,
      "completions/mean_terminated_length": 250.87095642089844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4264500830322504,
      "epoch": 0.00872,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1414268016815186,
      "kl": 0.8708685478195548,
      "learning_rate": 6.691610555125361e-06,
      "loss": -0.1164,
      "num_tokens": 17604745.0,
      "reward": 0.6388270854949951,
      "reward_std": 0.7765401601791382,
      "rewards/rollout_reward_func/mean": 0.6388270854949951,
      "rewards/rollout_reward_func/std": 0.7381476163864136,
      "sampling/importance_sampling_ratio/max": 1.4429430961608887,
      "sampling/importance_sampling_ratio/mean": 1.0449047088623047,
      "sampling/importance_sampling_ratio/min": 0.3932923376560211,
      "sampling/sampling_logp_difference/max": 0.9006106853485107,
      "sampling/sampling_logp_difference/mean": 0.0325118750333786,
      "step": 436,
      "step_time": 13.706894697024836
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.006026785937137902,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006026785937137902,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 250.1875,
      "completions/mean_terminated_length": 274.4137878417969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49315095972269773,
      "epoch": 0.00874,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0004795789718628,
      "kl": 0.8107326505705714,
      "learning_rate": 6.678082819672389e-06,
      "loss": -0.1226,
      "num_tokens": 17645642.0,
      "reward": 0.6381250023841858,
      "reward_std": 0.5142938494682312,
      "rewards/rollout_reward_func/mean": 0.6381250023841858,
      "rewards/rollout_reward_func/std": 0.622339129447937,
      "sampling/importance_sampling_ratio/max": 1.68315851688385,
      "sampling/importance_sampling_ratio/mean": 0.9599931240081787,
      "sampling/importance_sampling_ratio/min": 7.392235152996196e-13,
      "sampling/sampling_logp_difference/max": 18.06218719482422,
      "sampling/sampling_logp_difference/mean": 0.09235601127147675,
      "step": 437,
      "step_time": 12.473995750988252
    },
    {
      "clip_ratio/high_max": 0.01142857177183032,
      "clip_ratio/high_mean": 0.00571428588591516,
      "clip_ratio/low_mean": 0.0029611894860863686,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008675475372001529,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 1024.0,
      "completions/max_terminated_length": 1024.0,
      "completions/mean_length": 286.90625,
      "completions/mean_terminated_length": 295.6451416015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4580450728535652,
      "epoch": 0.00876,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6987200379371643,
      "kl": 1.3746018717065454,
      "learning_rate": 6.664549437517642e-06,
      "loss": -0.1358,
      "num_tokens": 17688497.0,
      "reward": 0.6738271713256836,
      "reward_std": 0.6036846041679382,
      "rewards/rollout_reward_func/mean": 0.6738271713256836,
      "rewards/rollout_reward_func/std": 0.6165447235107422,
      "sampling/importance_sampling_ratio/max": 1.4298616647720337,
      "sampling/importance_sampling_ratio/mean": 0.8794991970062256,
      "sampling/importance_sampling_ratio/min": 2.4557360471847388e-14,
      "sampling/sampling_logp_difference/max": 24.148040771484375,
      "sampling/sampling_logp_difference/mean": 0.10753939300775528,
      "step": 438,
      "step_time": 13.34952275903197
    },
    {
      "clip_ratio/high_max": 0.03283271915279329,
      "clip_ratio/high_mean": 0.016416359576396644,
      "clip_ratio/low_mean": 0.0013586956774815917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017775055253878236,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 629.0,
      "completions/max_terminated_length": 629.0,
      "completions/mean_length": 304.25,
      "completions/mean_terminated_length": 286.4814758300781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6271128989756107,
      "epoch": 0.00878,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3922617435455322,
      "kl": 0.6866352264769375,
      "learning_rate": 6.651010587175577e-06,
      "loss": -0.0232,
      "num_tokens": 17732448.0,
      "reward": 0.41749998927116394,
      "reward_std": 0.653376042842865,
      "rewards/rollout_reward_func/mean": 0.41749998927116394,
      "rewards/rollout_reward_func/std": 0.7153839468955994,
      "sampling/importance_sampling_ratio/max": 1.3022657632827759,
      "sampling/importance_sampling_ratio/mean": 0.8549620509147644,
      "sampling/importance_sampling_ratio/min": 1.8650346244724345e-15,
      "sampling/sampling_logp_difference/max": 23.342750549316406,
      "sampling/sampling_logp_difference/mean": 0.10445709526538849,
      "step": 439,
      "step_time": 12.732201478007482
    },
    {
      "clip_ratio/high_max": 0.008401537779718637,
      "clip_ratio/high_mean": 0.005177331273443997,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005177331273443997,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 613.0,
      "completions/max_terminated_length": 613.0,
      "completions/mean_length": 286.375,
      "completions/mean_terminated_length": 285.433349609375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.45879460126161575,
      "epoch": 0.0088,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5218888521194458,
      "kl": 0.313629693351686,
      "learning_rate": 6.637466447232784e-06,
      "loss": -0.2676,
      "num_tokens": 17774729.0,
      "reward": 0.7699999809265137,
      "reward_std": 0.3692001700401306,
      "rewards/rollout_reward_func/mean": 0.7699999809265137,
      "rewards/rollout_reward_func/std": 0.4404689371585846,
      "sampling/importance_sampling_ratio/max": 2.5107553005218506,
      "sampling/importance_sampling_ratio/mean": 1.0235626697540283,
      "sampling/importance_sampling_ratio/min": 0.3658435344696045,
      "sampling/sampling_logp_difference/max": 0.843754768371582,
      "sampling/sampling_logp_difference/mean": 0.034462518990039825,
      "step": 440,
      "step_time": 12.614741320008761
    },
    {
      "clip_ratio/high_max": 0.007812500232830644,
      "clip_ratio/high_mean": 0.003906250116415322,
      "clip_ratio/low_mean": 0.002016128972172737,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005922379088588059,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1087.0,
      "completions/max_terminated_length": 1087.0,
      "completions/mean_length": 310.78125,
      "completions/mean_terminated_length": 308.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4533058051019907,
      "epoch": 0.00882,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8672367334365845,
      "kl": 0.7871155776083469,
      "learning_rate": 6.623917196345622e-06,
      "loss": -0.1186,
      "num_tokens": 17817645.0,
      "reward": 0.5984374284744263,
      "reward_std": 0.5560826659202576,
      "rewards/rollout_reward_func/mean": 0.5984374284744263,
      "rewards/rollout_reward_func/std": 0.6773768663406372,
      "sampling/importance_sampling_ratio/max": 1.2960424423217773,
      "sampling/importance_sampling_ratio/mean": 0.8726605176925659,
      "sampling/importance_sampling_ratio/min": 7.91823678565444e-14,
      "sampling/sampling_logp_difference/max": 21.429973602294922,
      "sampling/sampling_logp_difference/mean": 0.10185518860816956,
      "step": 441,
      "step_time": 13.737686820008093
    },
    {
      "clip_ratio/high_max": 0.01478758198209107,
      "clip_ratio/high_mean": 0.007393790991045535,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007393790991045535,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 567.0,
      "completions/max_terminated_length": 567.0,
      "completions/mean_length": 234.6875,
      "completions/mean_terminated_length": 234.6875,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4325886648148298,
      "epoch": 0.00884,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2724672555923462,
      "kl": 0.41059909481555223,
      "learning_rate": 6.610363013237871e-06,
      "loss": -0.2049,
      "num_tokens": 17857793.0,
      "reward": 0.766915500164032,
      "reward_std": 0.4314410090446472,
      "rewards/rollout_reward_func/mean": 0.766915500164032,
      "rewards/rollout_reward_func/std": 0.6227301359176636,
      "sampling/importance_sampling_ratio/max": 1.5771316289901733,
      "sampling/importance_sampling_ratio/mean": 1.0125404596328735,
      "sampling/importance_sampling_ratio/min": 0.1797044575214386,
      "sampling/sampling_logp_difference/max": 0.9106993675231934,
      "sampling/sampling_logp_difference/mean": 0.03468480706214905,
      "step": 442,
      "step_time": 12.138507900963305
    },
    {
      "clip_ratio/high_max": 0.005217391299083829,
      "clip_ratio/high_mean": 0.0026086956495419145,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0026086956495419145,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 662.0,
      "completions/max_terminated_length": 662.0,
      "completions/mean_length": 229.5,
      "completions/mean_terminated_length": 221.21429443359375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5270122829824686,
      "epoch": 0.00886,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5981396436691284,
      "kl": 0.2925788569264114,
      "learning_rate": 6.59680407669837e-06,
      "loss": 0.1431,
      "num_tokens": 17897393.0,
      "reward": 0.5856249928474426,
      "reward_std": 0.6190099120140076,
      "rewards/rollout_reward_func/mean": 0.5856249928474426,
      "rewards/rollout_reward_func/std": 0.6110854148864746,
      "sampling/importance_sampling_ratio/max": 1.718266248703003,
      "sampling/importance_sampling_ratio/mean": 1.0789456367492676,
      "sampling/importance_sampling_ratio/min": 0.6931691765785217,
      "sampling/sampling_logp_difference/max": 0.34517645835876465,
      "sampling/sampling_logp_difference/mean": 0.032030969858169556,
      "step": 443,
      "step_time": 12.833302553975955
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 981.0,
      "completions/max_terminated_length": 561.0,
      "completions/mean_length": 230.09375,
      "completions/mean_terminated_length": 205.23077392578125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4837565366178751,
      "epoch": 0.00888,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.009749174118042,
      "kl": 0.32703355699777603,
      "learning_rate": 6.583240565578657e-06,
      "loss": 0.0774,
      "num_tokens": 17937347.0,
      "reward": 0.8621875047683716,
      "reward_std": 0.3417968153953552,
      "rewards/rollout_reward_func/mean": 0.8621875047683716,
      "rewards/rollout_reward_func/std": 0.5148863792419434,
      "sampling/importance_sampling_ratio/max": 1.6612497568130493,
      "sampling/importance_sampling_ratio/mean": 1.0628947019577026,
      "sampling/importance_sampling_ratio/min": 0.6941462755203247,
      "sampling/sampling_logp_difference/max": 0.4127078652381897,
      "sampling/sampling_logp_difference/mean": 0.029448989778757095,
      "step": 444,
      "step_time": 13.323704535971046
    },
    {
      "clip_ratio/high_max": 0.010416666977107525,
      "clip_ratio/high_mean": 0.0052083334885537624,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0071614584885537624,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1418.0,
      "completions/max_terminated_length": 1418.0,
      "completions/mean_length": 310.28125,
      "completions/mean_terminated_length": 323.5666809082031,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5314651094377041,
      "epoch": 0.0089,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9718708395957947,
      "kl": 0.7841149512678385,
      "learning_rate": 6.569672658790611e-06,
      "loss": -0.1571,
      "num_tokens": 17980354.0,
      "reward": 0.7121875286102295,
      "reward_std": 0.5561050176620483,
      "rewards/rollout_reward_func/mean": 0.7121875286102295,
      "rewards/rollout_reward_func/std": 0.6615413427352905,
      "sampling/importance_sampling_ratio/max": 1.3790301084518433,
      "sampling/importance_sampling_ratio/mean": 0.9323331117630005,
      "sampling/importance_sampling_ratio/min": 0.2833791971206665,
      "sampling/sampling_logp_difference/max": 1.2237834930419922,
      "sampling/sampling_logp_difference/mean": 0.03394157439470291,
      "step": 445,
      "step_time": 15.808352325009764
    },
    {
      "clip_ratio/high_max": 0.0028409091755747795,
      "clip_ratio/high_mean": 0.0014204545877873898,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0014204545877873898,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 624.0,
      "completions/max_terminated_length": 612.0,
      "completions/mean_length": 277.34375,
      "completions/mean_terminated_length": 267.73077392578125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.521335780620575,
      "epoch": 0.00892,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1169064044952393,
      "kl": 0.36898923572152853,
      "learning_rate": 6.556100535304097e-06,
      "loss": -0.1066,
      "num_tokens": 18022692.0,
      "reward": 0.7759374380111694,
      "reward_std": 0.4811587333679199,
      "rewards/rollout_reward_func/mean": 0.7759374380111694,
      "rewards/rollout_reward_func/std": 0.562315821647644,
      "sampling/importance_sampling_ratio/max": 1.8821139335632324,
      "sampling/importance_sampling_ratio/mean": 1.033188819885254,
      "sampling/importance_sampling_ratio/min": 0.5246210098266602,
      "sampling/sampling_logp_difference/max": 0.46129512786865234,
      "sampling/sampling_logp_difference/mean": 0.025638960301876068,
      "step": 446,
      "step_time": 12.21403763005219
    },
    {
      "clip_ratio/high_max": 0.004929315531626344,
      "clip_ratio/high_mean": 0.002464657765813172,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002464657765813172,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 642.0,
      "completions/max_terminated_length": 642.0,
      "completions/mean_length": 265.59375,
      "completions/mean_terminated_length": 236.7857208251953,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4604027010500431,
      "epoch": 0.00894,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5047088861465454,
      "kl": 0.7415442136116326,
      "learning_rate": 6.542524374144598e-06,
      "loss": -0.0754,
      "num_tokens": 18063185.0,
      "reward": 0.6328125,
      "reward_std": 0.6080414056777954,
      "rewards/rollout_reward_func/mean": 0.6328125,
      "rewards/rollout_reward_func/std": 0.6174464821815491,
      "sampling/importance_sampling_ratio/max": 1.9394611120224,
      "sampling/importance_sampling_ratio/mean": 0.9739242196083069,
      "sampling/importance_sampling_ratio/min": 0.32338815927505493,
      "sampling/sampling_logp_difference/max": 1.112077236175537,
      "sampling/sampling_logp_difference/mean": 0.03263751044869423,
      "step": 447,
      "step_time": 12.533016426023096
    },
    {
      "clip_ratio/high_max": 0.010774382157251239,
      "clip_ratio/high_mean": 0.005387191078625619,
      "clip_ratio/low_mean": 0.0013586956774815917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006745886756107211,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 656.0,
      "completions/mean_length": 176.03125,
      "completions/mean_terminated_length": 171.03448486328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.48664720356464386,
      "epoch": 0.00896,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3538912534713745,
      "kl": 0.2940802462399006,
      "learning_rate": 6.528944354390855e-06,
      "loss": 0.0352,
      "num_tokens": 18099752.0,
      "reward": 0.6943750381469727,
      "reward_std": 0.49003392457962036,
      "rewards/rollout_reward_func/mean": 0.6943750381469727,
      "rewards/rollout_reward_func/std": 0.6373125910758972,
      "sampling/importance_sampling_ratio/max": 1.7488443851470947,
      "sampling/importance_sampling_ratio/mean": 0.9757115840911865,
      "sampling/importance_sampling_ratio/min": 0.5739632248878479,
      "sampling/sampling_logp_difference/max": 0.5560324192047119,
      "sampling/sampling_logp_difference/mean": 0.03444618731737137,
      "step": 448,
      "step_time": 12.531112575976294
    },
    {
      "clip_ratio/high_max": 0.015017361380159855,
      "clip_ratio/high_mean": 0.0075086806900799274,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009461805690079927,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 660.0,
      "completions/mean_length": 191.96875,
      "completions/mean_terminated_length": 203.70001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5762611031532288,
      "epoch": 0.00898,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.148273229598999,
      "kl": 0.8563106381334364,
      "learning_rate": 6.515360655172512e-06,
      "loss": -0.0961,
      "num_tokens": 18138015.0,
      "reward": 0.574999988079071,
      "reward_std": 0.691824734210968,
      "rewards/rollout_reward_func/mean": 0.574999988079071,
      "rewards/rollout_reward_func/std": 0.7201477885246277,
      "sampling/importance_sampling_ratio/max": 1.4605813026428223,
      "sampling/importance_sampling_ratio/mean": 0.8838752508163452,
      "sampling/importance_sampling_ratio/min": 1.947928683292309e-16,
      "sampling/sampling_logp_difference/max": 23.252002716064453,
      "sampling/sampling_logp_difference/mean": 0.21543854475021362,
      "step": 449,
      "step_time": 11.964337174998946
    },
    {
      "clip_ratio/high_max": 0.005965909222140908,
      "clip_ratio/high_mean": 0.002982954611070454,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004936079611070454,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 680.0,
      "completions/max_terminated_length": 680.0,
      "completions/mean_length": 180.09375,
      "completions/mean_terminated_length": 197.0689697265625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.47687231563031673,
      "epoch": 0.009,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0941078662872314,
      "kl": 2.208732997532934,
      "learning_rate": 6.501773455667742e-06,
      "loss": -0.1115,
      "num_tokens": 18176056.0,
      "reward": 0.46357494592666626,
      "reward_std": 0.7127456068992615,
      "rewards/rollout_reward_func/mean": 0.46357494592666626,
      "rewards/rollout_reward_func/std": 0.8657038807868958,
      "sampling/importance_sampling_ratio/max": 2.807119131088257,
      "sampling/importance_sampling_ratio/mean": 0.9926393628120422,
      "sampling/importance_sampling_ratio/min": 0.275672048330307,
      "sampling/sampling_logp_difference/max": 1.2755951881408691,
      "sampling/sampling_logp_difference/mean": 0.0454406812787056,
      "step": 450,
      "step_time": 12.307411305984715
    },
    {
      "clip_ratio/high_max": 0.022919412003830075,
      "clip_ratio/high_mean": 0.011459706001915038,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011459706001915038,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 620.0,
      "completions/max_terminated_length": 589.0,
      "completions/mean_length": 261.875,
      "completions/mean_terminated_length": 250.32257080078125,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.5004891864955425,
      "epoch": 0.00902,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1015815734863281,
      "kl": 0.5469152769073844,
      "learning_rate": 6.488182935100893e-06,
      "loss": -0.0502,
      "num_tokens": 18218508.0,
      "reward": 0.8078124523162842,
      "reward_std": 0.5673302412033081,
      "rewards/rollout_reward_func/mean": 0.8078124523162842,
      "rewards/rollout_reward_func/std": 0.5629228949546814,
      "sampling/importance_sampling_ratio/max": 1.6446337699890137,
      "sampling/importance_sampling_ratio/mean": 0.8975223898887634,
      "sampling/importance_sampling_ratio/min": 5.5853873467211126e-15,
      "sampling/sampling_logp_difference/max": 24.403324127197266,
      "sampling/sampling_logp_difference/mean": 0.10898856073617935,
      "step": 451,
      "step_time": 12.350653360947035
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0166015625,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 892.0,
      "completions/max_terminated_length": 892.0,
      "completions/mean_length": 219.84375,
      "completions/mean_terminated_length": 221.40000915527344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.44545817002654076,
      "epoch": 0.00904,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3259961605072021,
      "kl": 0.95554478187114,
      "learning_rate": 6.474589272740116e-06,
      "loss": -0.3153,
      "num_tokens": 18257717.0,
      "reward": 0.5414737462997437,
      "reward_std": 0.63482666015625,
      "rewards/rollout_reward_func/mean": 0.5414737462997437,
      "rewards/rollout_reward_func/std": 0.7692280411720276,
      "sampling/importance_sampling_ratio/max": 2.7393786907196045,
      "sampling/importance_sampling_ratio/mean": 1.059190273284912,
      "sampling/importance_sampling_ratio/min": 0.4136923849582672,
      "sampling/sampling_logp_difference/max": 0.8980474472045898,
      "sampling/sampling_logp_difference/mean": 0.037836506962776184,
      "step": 452,
      "step_time": 12.635306427968317
    },
    {
      "clip_ratio/high_max": 0.019791666883975267,
      "clip_ratio/high_mean": 0.009895833441987634,
      "clip_ratio/low_mean": 0.0020833334419876337,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011979166883975267,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 519.0,
      "completions/max_terminated_length": 519.0,
      "completions/mean_length": 251.65625,
      "completions/mean_terminated_length": 251.65625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.48002403043210506,
      "epoch": 0.00906,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9407873153686523,
      "kl": 3.5647507421672344,
      "learning_rate": 6.46099264789501e-06,
      "loss": 0.0168,
      "num_tokens": 18299219.0,
      "reward": 0.8871874809265137,
      "reward_std": 0.34063535928726196,
      "rewards/rollout_reward_func/mean": 0.8871874809265137,
      "rewards/rollout_reward_func/std": 0.4497184157371521,
      "sampling/importance_sampling_ratio/max": 1.6354622840881348,
      "sampling/importance_sampling_ratio/mean": 1.0568770170211792,
      "sampling/importance_sampling_ratio/min": 0.2428397536277771,
      "sampling/sampling_logp_difference/max": 1.4154062271118164,
      "sampling/sampling_logp_difference/mean": 0.03141547739505768,
      "step": 453,
      "step_time": 12.4027753379778
    },
    {
      "clip_ratio/high_max": 0.03297697380185127,
      "clip_ratio/high_mean": 0.016488486900925636,
      "clip_ratio/low_mean": 0.003513088566251099,
      "clip_ratio/low_min": 0.001953125,
      "clip_ratio/region_mean": 0.020001575467176735,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 636.0,
      "completions/max_terminated_length": 636.0,
      "completions/mean_length": 266.3125,
      "completions/mean_terminated_length": 311.8800048828125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5249191746115685,
      "epoch": 0.00908,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3299165964126587,
      "kl": 0.5106087853200734,
      "learning_rate": 6.447393239914247e-06,
      "loss": -0.029,
      "num_tokens": 18339835.0,
      "reward": 0.6681249737739563,
      "reward_std": 0.6030310392379761,
      "rewards/rollout_reward_func/mean": 0.6681249737739563,
      "rewards/rollout_reward_func/std": 0.6164017915725708,
      "sampling/importance_sampling_ratio/max": 1.6974642276763916,
      "sampling/importance_sampling_ratio/mean": 0.9800405502319336,
      "sampling/importance_sampling_ratio/min": 0.43481945991516113,
      "sampling/sampling_logp_difference/max": 0.6924128532409668,
      "sampling/sampling_logp_difference/mean": 0.03431575000286102,
      "step": 454,
      "step_time": 12.615640621996135
    },
    {
      "clip_ratio/high_max": 0.02094655833207071,
      "clip_ratio/high_mean": 0.010473279166035354,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.026098279166035354,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 615.0,
      "completions/max_terminated_length": 615.0,
      "completions/mean_length": 273.21875,
      "completions/mean_terminated_length": 284.862060546875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5116786807775497,
      "epoch": 0.0091,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1355923414230347,
      "kl": 1.3547468893229961,
      "learning_rate": 6.433791228183211e-06,
      "loss": -0.0879,
      "num_tokens": 18382100.0,
      "reward": 0.6443749666213989,
      "reward_std": 0.6526139974594116,
      "rewards/rollout_reward_func/mean": 0.6443749666213989,
      "rewards/rollout_reward_func/std": 0.6704063415527344,
      "sampling/importance_sampling_ratio/max": 1.3625532388687134,
      "sampling/importance_sampling_ratio/mean": 0.9577420353889465,
      "sampling/importance_sampling_ratio/min": 0.37993964552879333,
      "sampling/sampling_logp_difference/max": 0.9394755363464355,
      "sampling/sampling_logp_difference/mean": 0.03696448355913162,
      "step": 455,
      "step_time": 12.235140944976592
    },
    {
      "clip_ratio/high_max": 0.027713068295270205,
      "clip_ratio/high_mean": 0.013856534147635102,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013856534147635102,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 829.0,
      "completions/max_terminated_length": 829.0,
      "completions/mean_length": 284.28125,
      "completions/mean_terminated_length": 312.03448486328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4560091905295849,
      "epoch": 0.00912,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1630887985229492,
      "kl": 0.7486433833837509,
      "learning_rate": 6.42018679212163e-06,
      "loss": 0.077,
      "num_tokens": 18423239.0,
      "reward": 0.5029475092887878,
      "reward_std": 0.7034891843795776,
      "rewards/rollout_reward_func/mean": 0.5029475092887878,
      "rewards/rollout_reward_func/std": 0.7429352402687073,
      "sampling/importance_sampling_ratio/max": 1.484503149986267,
      "sampling/importance_sampling_ratio/mean": 0.8738469481468201,
      "sampling/importance_sampling_ratio/min": 4.292864286270515e-09,
      "sampling/sampling_logp_difference/max": 19.312654495239258,
      "sampling/sampling_logp_difference/mean": 0.1329854279756546,
      "step": 456,
      "step_time": 12.81357858798583
    },
    {
      "clip_ratio/high_max": 0.01765046827495098,
      "clip_ratio/high_mean": 0.00882523413747549,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00882523413747549,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 558.0,
      "completions/max_terminated_length": 558.0,
      "completions/mean_length": 198.1875,
      "completions/mean_terminated_length": 195.90000915527344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4815651923418045,
      "epoch": 0.00914,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7558879256248474,
      "kl": 0.45629764441400766,
      "learning_rate": 6.406580111181216e-06,
      "loss": -0.143,
      "num_tokens": 18461889.0,
      "reward": 0.7421875,
      "reward_std": 0.6773371696472168,
      "rewards/rollout_reward_func/mean": 0.7421875,
      "rewards/rollout_reward_func/std": 0.6844009757041931,
      "sampling/importance_sampling_ratio/max": 1.838230013847351,
      "sampling/importance_sampling_ratio/mean": 0.9991293549537659,
      "sampling/importance_sampling_ratio/min": 0.37653928995132446,
      "sampling/sampling_logp_difference/max": 0.7059323787689209,
      "sampling/sampling_logp_difference/mean": 0.0366087332367897,
      "step": 457,
      "step_time": 12.016515480994713
    },
    {
      "clip_ratio/high_max": 0.01076007355004549,
      "clip_ratio/high_mean": 0.005380036775022745,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021005036775022745,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 630.0,
      "completions/max_terminated_length": 621.0,
      "completions/mean_length": 292.09375,
      "completions/mean_terminated_length": 281.19354248046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5128279216587543,
      "epoch": 0.00916,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8639600276947021,
      "kl": 0.5890715271234512,
      "learning_rate": 6.392971364843287e-06,
      "loss": -0.1803,
      "num_tokens": 18505019.0,
      "reward": 0.5521875023841858,
      "reward_std": 0.7680566310882568,
      "rewards/rollout_reward_func/mean": 0.5521875023841858,
      "rewards/rollout_reward_func/std": 0.7326695322990417,
      "sampling/importance_sampling_ratio/max": 1.4575775861740112,
      "sampling/importance_sampling_ratio/mean": 0.9526591300964355,
      "sampling/importance_sampling_ratio/min": 0.4269152581691742,
      "sampling/sampling_logp_difference/max": 0.8464069366455078,
      "sampling/sampling_logp_difference/mean": 0.029714062809944153,
      "step": 458,
      "step_time": 12.246270716030267
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 649.0,
      "completions/mean_length": 228.5625,
      "completions/mean_terminated_length": 206.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4571515154093504,
      "epoch": 0.00918,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9088395237922668,
      "kl": 0.5782118067145348,
      "learning_rate": 6.379360732616405e-06,
      "loss": 0.1019,
      "num_tokens": 18544362.0,
      "reward": 0.7409374713897705,
      "reward_std": 0.513913631439209,
      "rewards/rollout_reward_func/mean": 0.7409374713897705,
      "rewards/rollout_reward_func/std": 0.518287181854248,
      "sampling/importance_sampling_ratio/max": 1.4367625713348389,
      "sampling/importance_sampling_ratio/mean": 1.0164742469787598,
      "sampling/importance_sampling_ratio/min": 1.7088229025156651e-15,
      "sampling/sampling_logp_difference/max": 21.35503578186035,
      "sampling/sampling_logp_difference/mean": 0.13142192363739014,
      "step": 459,
      "step_time": 12.27502146102779
    },
    {
      "clip_ratio/high_max": 0.007031250046566129,
      "clip_ratio/high_mean": 0.0035156250232830644,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0035156250232830644,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 633.0,
      "completions/max_terminated_length": 633.0,
      "completions/mean_length": 196.84375,
      "completions/mean_terminated_length": 215.55172729492188,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5045338962227106,
      "epoch": 0.0092,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4366092681884766,
      "kl": 0.5370073774829507,
      "learning_rate": 6.365748394034013e-06,
      "loss": -0.0343,
      "num_tokens": 18582020.0,
      "reward": 0.7594521045684814,
      "reward_std": 0.44952136278152466,
      "rewards/rollout_reward_func/mean": 0.7594521045684814,
      "rewards/rollout_reward_func/std": 0.5281790494918823,
      "sampling/importance_sampling_ratio/max": 1.4361801147460938,
      "sampling/importance_sampling_ratio/mean": 1.0497034788131714,
      "sampling/importance_sampling_ratio/min": 0.3211389482021332,
      "sampling/sampling_logp_difference/max": 1.0501251220703125,
      "sampling/sampling_logp_difference/mean": 0.03185518831014633,
      "step": 460,
      "step_time": 12.375875953017385
    },
    {
      "clip_ratio/high_max": 0.0024999999441206455,
      "clip_ratio/high_mean": 0.0012499999720603228,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0012499999720603228,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 249.375,
      "completions/mean_terminated_length": 232.41378784179688,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4776557423174381,
      "epoch": 0.00922,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.315503478050232,
      "kl": 0.40786833595484495,
      "learning_rate": 6.352134528652057e-06,
      "loss": 0.1091,
      "num_tokens": 18622521.0,
      "reward": 0.6940624713897705,
      "reward_std": 0.5426313877105713,
      "rewards/rollout_reward_func/mean": 0.6940624713897705,
      "rewards/rollout_reward_func/std": 0.5957963466644287,
      "sampling/importance_sampling_ratio/max": 1.798397421836853,
      "sampling/importance_sampling_ratio/mean": 1.026429533958435,
      "sampling/importance_sampling_ratio/min": 0.33176225423812866,
      "sampling/sampling_logp_difference/max": 0.7106847763061523,
      "sampling/sampling_logp_difference/mean": 0.03204786032438278,
      "step": 461,
      "step_time": 12.133006560019567
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.0033735795877873898,
      "clip_ratio/low_mean": 0.0011574074160307646,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004530987003818154,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 862.0,
      "completions/max_terminated_length": 862.0,
      "completions/mean_length": 265.375,
      "completions/mean_terminated_length": 278.4285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5780180748552084,
      "epoch": 0.00924,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.282573938369751,
      "kl": 0.338323880918324,
      "learning_rate": 6.3385193160466255e-06,
      "loss": -0.0949,
      "num_tokens": 18664529.0,
      "reward": 0.6974999904632568,
      "reward_std": 0.6621494293212891,
      "rewards/rollout_reward_func/mean": 0.6974999904632568,
      "rewards/rollout_reward_func/std": 0.6434934735298157,
      "sampling/importance_sampling_ratio/max": 1.2831025123596191,
      "sampling/importance_sampling_ratio/mean": 0.863242506980896,
      "sampling/importance_sampling_ratio/min": 5.546860887273297e-11,
      "sampling/sampling_logp_difference/max": 23.802234649658203,
      "sampling/sampling_logp_difference/mean": 0.13225609064102173,
      "step": 462,
      "step_time": 13.14234844398743
    },
    {
      "clip_ratio/high_max": 0.014756944496184587,
      "clip_ratio/high_mean": 0.007378472248092294,
      "clip_ratio/low_mean": 0.00498383620288223,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012362308450974524,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 589.0,
      "completions/max_terminated_length": 576.0,
      "completions/mean_length": 210.28125,
      "completions/mean_terminated_length": 200.48275756835938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.48805265594273806,
      "epoch": 0.00926,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2269748449325562,
      "kl": 0.6313120424747467,
      "learning_rate": 6.324902935811576e-06,
      "loss": -0.0326,
      "num_tokens": 18703421.0,
      "reward": 0.5710002183914185,
      "reward_std": 0.7071930170059204,
      "rewards/rollout_reward_func/mean": 0.5710002183914185,
      "rewards/rollout_reward_func/std": 0.7377168536186218,
      "sampling/importance_sampling_ratio/max": 1.6119704246520996,
      "sampling/importance_sampling_ratio/mean": 0.9406559467315674,
      "sampling/importance_sampling_ratio/min": 6.977565747412484e-10,
      "sampling/sampling_logp_difference/max": 21.125532150268555,
      "sampling/sampling_logp_difference/mean": 0.13082130253314972,
      "step": 463,
      "step_time": 12.17600621197198
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.004166666767559946,
      "clip_ratio/low_min": 0.0026041667442768812,
      "clip_ratio/region_mean": 0.006119791767559946,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 302.21875,
      "completions/mean_terminated_length": 313.0333557128906,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5266995076090097,
      "epoch": 0.00928,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2841393947601318,
      "kl": 0.36699882335960865,
      "learning_rate": 6.311285567556168e-06,
      "loss": 0.001,
      "num_tokens": 18747318.0,
      "reward": 0.7412499785423279,
      "reward_std": 0.4535847306251526,
      "rewards/rollout_reward_func/mean": 0.7412499785423279,
      "rewards/rollout_reward_func/std": 0.45925429463386536,
      "sampling/importance_sampling_ratio/max": 1.7575708627700806,
      "sampling/importance_sampling_ratio/mean": 1.0016798973083496,
      "sampling/importance_sampling_ratio/min": 1.0319074067455247e-10,
      "sampling/sampling_logp_difference/max": 10.612998008728027,
      "sampling/sampling_logp_difference/mean": 0.07946731895208359,
      "step": 464,
      "step_time": 12.827103570016334
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 1135.0,
      "completions/max_terminated_length": 1135.0,
      "completions/mean_length": 260.0625,
      "completions/mean_terminated_length": 279.2758483886719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5731074921786785,
      "epoch": 0.0093,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2909696102142334,
      "kl": 0.7675462085753679,
      "learning_rate": 6.297667390902694e-06,
      "loss": -0.2613,
      "num_tokens": 18788799.0,
      "reward": 0.6325048208236694,
      "reward_std": 0.6602233648300171,
      "rewards/rollout_reward_func/mean": 0.6325048208236694,
      "rewards/rollout_reward_func/std": 0.7800052762031555,
      "sampling/importance_sampling_ratio/max": 1.5063742399215698,
      "sampling/importance_sampling_ratio/mean": 0.9931051135063171,
      "sampling/importance_sampling_ratio/min": 0.29444238543510437,
      "sampling/sampling_logp_difference/max": 1.1105785369873047,
      "sampling/sampling_logp_difference/mean": 0.040731098502874374,
      "step": 465,
      "step_time": 14.095768768005655
    },
    {
      "clip_ratio/high_max": 0.0031250000465661287,
      "clip_ratio/high_mean": 0.0015625000232830644,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015625000232830644,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 277.875,
      "completions/mean_terminated_length": 276.1071472167969,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.46882457472383976,
      "epoch": 0.00932,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.602121114730835,
      "kl": 0.2871092383284122,
      "learning_rate": 6.284048585484113e-06,
      "loss": 0.1558,
      "num_tokens": 18830091.0,
      "reward": 0.7734375,
      "reward_std": 0.4610168933868408,
      "rewards/rollout_reward_func/mean": 0.7734375,
      "rewards/rollout_reward_func/std": 0.43836653232574463,
      "sampling/importance_sampling_ratio/max": 2.118441581726074,
      "sampling/importance_sampling_ratio/mean": 1.0646097660064697,
      "sampling/importance_sampling_ratio/min": 0.4376635253429413,
      "sampling/sampling_logp_difference/max": 0.8251643180847168,
      "sampling/sampling_logp_difference/mean": 0.03532955422997475,
      "step": 466,
      "step_time": 12.41189802098961
    },
    {
      "clip_ratio/high_max": 0.0062806373462080956,
      "clip_ratio/high_mean": 0.0031403186731040478,
      "clip_ratio/low_mean": 0.0052083334885537624,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00834865216165781,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 283.9375,
      "completions/mean_terminated_length": 268.8666687011719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5372491702437401,
      "epoch": 0.00934,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1056138277053833,
      "kl": 0.511128825135529,
      "learning_rate": 6.270429330941671e-06,
      "loss": -0.0733,
      "num_tokens": 18872172.0,
      "reward": 0.6462500095367432,
      "reward_std": 0.4838739335536957,
      "rewards/rollout_reward_func/mean": 0.6462500095367432,
      "rewards/rollout_reward_func/std": 0.6504626274108887,
      "sampling/importance_sampling_ratio/max": 1.7433769702911377,
      "sampling/importance_sampling_ratio/mean": 0.9929654598236084,
      "sampling/importance_sampling_ratio/min": 0.36896488070487976,
      "sampling/sampling_logp_difference/max": 0.7936103343963623,
      "sampling/sampling_logp_difference/mean": 0.03434234485030174,
      "step": 467,
      "step_time": 12.567103191002388
    },
    {
      "clip_ratio/high_max": 0.07120028417557478,
      "clip_ratio/high_mean": 0.03690222604200244,
      "clip_ratio/low_mean": 0.0015625000232830644,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.038464726065285504,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 635.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 257.78125,
      "completions/mean_terminated_length": 269.2592468261719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5464379647746682,
      "epoch": 0.00936,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3588327169418335,
      "kl": 0.757067690603435,
      "learning_rate": 6.2568098069225436e-06,
      "loss": 0.0295,
      "num_tokens": 18914167.0,
      "reward": 0.5774999856948853,
      "reward_std": 0.5013295412063599,
      "rewards/rollout_reward_func/mean": 0.5774999856948853,
      "rewards/rollout_reward_func/std": 0.6140609979629517,
      "sampling/importance_sampling_ratio/max": 1.2505743503570557,
      "sampling/importance_sampling_ratio/mean": 0.843330979347229,
      "sampling/importance_sampling_ratio/min": 5.428824629405755e-23,
      "sampling/sampling_logp_difference/max": 23.061595916748047,
      "sampling/sampling_logp_difference/mean": 0.15188336372375488,
      "step": 468,
      "step_time": 12.897185704990989
    },
    {
      "clip_ratio/high_max": 0.010138889076188207,
      "clip_ratio/high_mean": 0.005069444538094103,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006046007038094103,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 660.0,
      "completions/mean_length": 219.03125,
      "completions/mean_terminated_length": 230.55555725097656,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.47209285385906696,
      "epoch": 0.00938,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0355180501937866,
      "kl": 0.39317901339381933,
      "learning_rate": 6.243190193077457e-06,
      "loss": -0.053,
      "num_tokens": 18954008.0,
      "reward": 0.546875,
      "reward_std": 0.7327767610549927,
      "rewards/rollout_reward_func/mean": 0.546875,
      "rewards/rollout_reward_func/std": 0.7141492962837219,
      "sampling/importance_sampling_ratio/max": 1.558166742324829,
      "sampling/importance_sampling_ratio/mean": 1.0114378929138184,
      "sampling/importance_sampling_ratio/min": 0.2951042354106903,
      "sampling/sampling_logp_difference/max": 0.7266082763671875,
      "sampling/sampling_logp_difference/mean": 0.031352266669273376,
      "step": 469,
      "step_time": 12.044781517019146
    },
    {
      "clip_ratio/high_max": 0.030729166930541396,
      "clip_ratio/high_mean": 0.016406250186264515,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016406250186264515,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 633.0,
      "completions/max_terminated_length": 599.0,
      "completions/mean_length": 239.25,
      "completions/mean_terminated_length": 237.11111450195312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.47521037328988314,
      "epoch": 0.0094,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.161536455154419,
      "kl": 0.3046142184175551,
      "learning_rate": 6.2295706690583325e-06,
      "loss": -0.1124,
      "num_tokens": 18994703.0,
      "reward": 0.7378125190734863,
      "reward_std": 0.42182737588882446,
      "rewards/rollout_reward_func/mean": 0.7378125190734863,
      "rewards/rollout_reward_func/std": 0.5142719745635986,
      "sampling/importance_sampling_ratio/max": 1.8299230337142944,
      "sampling/importance_sampling_ratio/mean": 1.0044023990631104,
      "sampling/importance_sampling_ratio/min": 0.3458952307701111,
      "sampling/sampling_logp_difference/max": 0.8846642971038818,
      "sampling/sampling_logp_difference/mean": 0.035548195242881775,
      "step": 470,
      "step_time": 12.494558732010773
    },
    {
      "clip_ratio/high_max": 0.01455965917557478,
      "clip_ratio/high_mean": 0.00727982958778739,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011744115501642227,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 541.0,
      "completions/max_terminated_length": 541.0,
      "completions/mean_length": 183.28125,
      "completions/mean_terminated_length": 175.03448486328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5101989731192589,
      "epoch": 0.00942,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9706793427467346,
      "kl": 0.4375730659812689,
      "learning_rate": 6.215951414515888e-06,
      "loss": -0.1894,
      "num_tokens": 19032075.0,
      "reward": 0.6365625262260437,
      "reward_std": 0.6765463352203369,
      "rewards/rollout_reward_func/mean": 0.6365625262260437,
      "rewards/rollout_reward_func/std": 0.704376220703125,
      "sampling/importance_sampling_ratio/max": 1.5906130075454712,
      "sampling/importance_sampling_ratio/mean": 0.9932798147201538,
      "sampling/importance_sampling_ratio/min": 0.3971588611602783,
      "sampling/sampling_logp_difference/max": 0.8992162942886353,
      "sampling/sampling_logp_difference/mean": 0.040277861058712006,
      "step": 471,
      "step_time": 12.140561002015602
    },
    {
      "clip_ratio/high_max": 0.008497807197272778,
      "clip_ratio/high_mean": 0.0057369989808648825,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0057369989808648825,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 236.5,
      "completions/mean_terminated_length": 195.82144165039062,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5633476264774799,
      "epoch": 0.00944,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3041249513626099,
      "kl": 0.5721097430214286,
      "learning_rate": 6.202332609097308e-06,
      "loss": 0.1232,
      "num_tokens": 19072345.0,
      "reward": 0.6346874833106995,
      "reward_std": 0.5517587065696716,
      "rewards/rollout_reward_func/mean": 0.6346874833106995,
      "rewards/rollout_reward_func/std": 0.5610818266868591,
      "sampling/importance_sampling_ratio/max": 1.8774229288101196,
      "sampling/importance_sampling_ratio/mean": 0.9207308292388916,
      "sampling/importance_sampling_ratio/min": 4.2460998054755113e-19,
      "sampling/sampling_logp_difference/max": 24.748605728149414,
      "sampling/sampling_logp_difference/mean": 0.1612236499786377,
      "step": 472,
      "step_time": 12.313660268991953
    },
    {
      "clip_ratio/high_max": 0.0036764706019312143,
      "clip_ratio/high_mean": 0.0018382353009656072,
      "clip_ratio/low_mean": 0.014144737040624022,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01598297234158963,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 626.0,
      "completions/max_terminated_length": 626.0,
      "completions/mean_length": 225.375,
      "completions/mean_terminated_length": 248.1785888671875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.47440906800329685,
      "epoch": 0.00946,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0637109279632568,
      "kl": 0.2738903700374067,
      "learning_rate": 6.188714432443833e-06,
      "loss": -0.1269,
      "num_tokens": 19111953.0,
      "reward": 0.8293750286102295,
      "reward_std": 0.3920072913169861,
      "rewards/rollout_reward_func/mean": 0.8293750286102295,
      "rewards/rollout_reward_func/std": 0.4680497348308563,
      "sampling/importance_sampling_ratio/max": 1.8570209741592407,
      "sampling/importance_sampling_ratio/mean": 1.0311264991760254,
      "sampling/importance_sampling_ratio/min": 0.45448851585388184,
      "sampling/sampling_logp_difference/max": 0.843773603439331,
      "sampling/sampling_logp_difference/mean": 0.034620676189661026,
      "step": 473,
      "step_time": 12.563937043029
    },
    {
      "clip_ratio/high_max": 0.00870028417557478,
      "clip_ratio/high_mean": 0.00435014208778739,
      "clip_ratio/low_mean": 0.008203125093132257,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012553267180919647,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 638.0,
      "completions/max_terminated_length": 638.0,
      "completions/mean_length": 281.53125,
      "completions/mean_terminated_length": 293.2692565917969,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5406616907566786,
      "epoch": 0.00948,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1129175424575806,
      "kl": 0.5059516057372093,
      "learning_rate": 6.175097064188427e-06,
      "loss": -0.1777,
      "num_tokens": 19154568.0,
      "reward": 0.502723753452301,
      "reward_std": 0.7757288217544556,
      "rewards/rollout_reward_func/mean": 0.502723753452301,
      "rewards/rollout_reward_func/std": 0.7448975443840027,
      "sampling/importance_sampling_ratio/max": 1.5395363569259644,
      "sampling/importance_sampling_ratio/mean": 0.9137399196624756,
      "sampling/importance_sampling_ratio/min": 9.424985855320812e-19,
      "sampling/sampling_logp_difference/max": 20.081295013427734,
      "sampling/sampling_logp_difference/mean": 0.11644287407398224,
      "step": 474,
      "step_time": 12.930019974999595
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 219.5,
      "completions/mean_terminated_length": 220.1428680419922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5152455540373921,
      "epoch": 0.0095,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0303200483322144,
      "kl": 1.9128483701497316,
      "learning_rate": 6.161480683953376e-06,
      "loss": -0.1733,
      "num_tokens": 19194958.0,
      "reward": 0.5753529667854309,
      "reward_std": 0.7353142499923706,
      "rewards/rollout_reward_func/mean": 0.5753529667854309,
      "rewards/rollout_reward_func/std": 0.7169037461280823,
      "sampling/importance_sampling_ratio/max": 1.9424365758895874,
      "sampling/importance_sampling_ratio/mean": 0.9928618669509888,
      "sampling/importance_sampling_ratio/min": 0.1860085278749466,
      "sampling/sampling_logp_difference/max": 1.2423367500305176,
      "sampling/sampling_logp_difference/mean": 0.03831392899155617,
      "step": 475,
      "step_time": 11.962975182992523
    },
    {
      "clip_ratio/high_max": 0.014330370118841529,
      "clip_ratio/high_mean": 0.0071651850594207644,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0071651850594207644,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 648.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 292.84375,
      "completions/mean_terminated_length": 287.6128845214844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5289229471236467,
      "epoch": 0.00952,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6269665956497192,
      "kl": 0.4541552113369107,
      "learning_rate": 6.147865471347945e-06,
      "loss": -0.0161,
      "num_tokens": 19237189.0,
      "reward": 0.6955362558364868,
      "reward_std": 0.5823593139648438,
      "rewards/rollout_reward_func/mean": 0.6955362558364868,
      "rewards/rollout_reward_func/std": 0.5934537053108215,
      "sampling/importance_sampling_ratio/max": 1.6816647052764893,
      "sampling/importance_sampling_ratio/mean": 0.9456833004951477,
      "sampling/importance_sampling_ratio/min": 0.4309815764427185,
      "sampling/sampling_logp_difference/max": 0.8131136894226074,
      "sampling/sampling_logp_difference/mean": 0.04053887724876404,
      "step": 476,
      "step_time": 12.672404669952812
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 641.0,
      "completions/max_terminated_length": 641.0,
      "completions/mean_length": 233.375,
      "completions/mean_terminated_length": 246.6428680419922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.480911485850811,
      "epoch": 0.00954,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4896790981292725,
      "kl": 0.5010779798030853,
      "learning_rate": 6.134251605965988e-06,
      "loss": 0.1958,
      "num_tokens": 19278015.0,
      "reward": 0.5874999761581421,
      "reward_std": 0.6197571754455566,
      "rewards/rollout_reward_func/mean": 0.5874999761581421,
      "rewards/rollout_reward_func/std": 0.7196369767189026,
      "sampling/importance_sampling_ratio/max": 2.4561829566955566,
      "sampling/importance_sampling_ratio/mean": 1.0034109354019165,
      "sampling/importance_sampling_ratio/min": 0.37166404724121094,
      "sampling/sampling_logp_difference/max": 1.0209014415740967,
      "sampling/sampling_logp_difference/mean": 0.03895949572324753,
      "step": 477,
      "step_time": 12.50833342304395
    },
    {
      "clip_ratio/high_max": 0.006953124888241291,
      "clip_ratio/high_mean": 0.0034765624441206455,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0034765624441206455,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 661.0,
      "completions/max_terminated_length": 661.0,
      "completions/mean_length": 380.1875,
      "completions/mean_terminated_length": 396.8148193359375,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5422965716570616,
      "epoch": 0.00956,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3418960571289062,
      "kl": 0.412858746945858,
      "learning_rate": 6.1206392673835955e-06,
      "loss": -0.0257,
      "num_tokens": 19324915.0,
      "reward": 0.5199999809265137,
      "reward_std": 0.6628816723823547,
      "rewards/rollout_reward_func/mean": 0.5199999809265137,
      "rewards/rollout_reward_func/std": 0.6879656314849854,
      "sampling/importance_sampling_ratio/max": 1.5955201387405396,
      "sampling/importance_sampling_ratio/mean": 0.944247841835022,
      "sampling/importance_sampling_ratio/min": 0.28518274426460266,
      "sampling/sampling_logp_difference/max": 0.7199306488037109,
      "sampling/sampling_logp_difference/mean": 0.03522159159183502,
      "step": 478,
      "step_time": 12.667950259972713
    },
    {
      "clip_ratio/high_max": 0.009027777938172221,
      "clip_ratio/high_mean": 0.004513888969086111,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004513888969086111,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 865.0,
      "completions/max_terminated_length": 865.0,
      "completions/mean_length": 259.4375,
      "completions/mean_terminated_length": 268.75860595703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.577535267919302,
      "epoch": 0.00958,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9720388650894165,
      "kl": 0.37106099631637335,
      "learning_rate": 6.1070286351567154e-06,
      "loss": 0.0207,
      "num_tokens": 19366801.0,
      "reward": 0.5715625286102295,
      "reward_std": 0.660754919052124,
      "rewards/rollout_reward_func/mean": 0.5715625286102295,
      "rewards/rollout_reward_func/std": 0.7112329602241516,
      "sampling/importance_sampling_ratio/max": 1.6093175411224365,
      "sampling/importance_sampling_ratio/mean": 0.9265922904014587,
      "sampling/importance_sampling_ratio/min": 9.774438677163239e-21,
      "sampling/sampling_logp_difference/max": 21.956623077392578,
      "sampling/sampling_logp_difference/mean": 0.13540437817573547,
      "step": 479,
      "step_time": 13.463321438015555
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 891.0,
      "completions/max_terminated_length": 643.0,
      "completions/mean_length": 279.1875,
      "completions/mean_terminated_length": 247.7333526611328,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4933468736708164,
      "epoch": 0.0096,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1446489095687866,
      "kl": 0.4211669694632292,
      "learning_rate": 6.093419888818785e-06,
      "loss": 0.023,
      "num_tokens": 19407791.0,
      "reward": 0.6362500190734863,
      "reward_std": 0.6232460737228394,
      "rewards/rollout_reward_func/mean": 0.6362500190734863,
      "rewards/rollout_reward_func/std": 0.6121866703033447,
      "sampling/importance_sampling_ratio/max": 1.312863826751709,
      "sampling/importance_sampling_ratio/mean": 0.992946982383728,
      "sampling/importance_sampling_ratio/min": 0.4053605794906616,
      "sampling/sampling_logp_difference/max": 0.710240364074707,
      "sampling/sampling_logp_difference/mean": 0.03420605510473251,
      "step": 480,
      "step_time": 12.967535842006328
    },
    {
      "clip_ratio/high_max": 0.016741071827709675,
      "clip_ratio/high_mean": 0.008370535913854837,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016183035913854837,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 997.0,
      "completions/max_terminated_length": 997.0,
      "completions/mean_length": 227.3125,
      "completions/mean_terminated_length": 244.57144165039062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49185838643461466,
      "epoch": 0.00962,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9555364847183228,
      "kl": 1.598540535196662,
      "learning_rate": 6.0798132078783714e-06,
      "loss": -0.1302,
      "num_tokens": 19447153.0,
      "reward": 0.7575024366378784,
      "reward_std": 0.5109760761260986,
      "rewards/rollout_reward_func/mean": 0.7575024366378784,
      "rewards/rollout_reward_func/std": 0.6011308431625366,
      "sampling/importance_sampling_ratio/max": 1.9028698205947876,
      "sampling/importance_sampling_ratio/mean": 0.9567884206771851,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.2228164672851562,
      "sampling/sampling_logp_difference/mean": 0.03843523561954498,
      "step": 481,
      "step_time": 13.36713346002216
    },
    {
      "clip_ratio/high_max": 0.015217391541227698,
      "clip_ratio/high_mean": 0.007608695770613849,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007608695770613849,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1267.0,
      "completions/max_terminated_length": 1267.0,
      "completions/mean_length": 248.96875,
      "completions/mean_terminated_length": 257.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5252981279045343,
      "epoch": 0.00964,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.604820728302002,
      "kl": 1.2138366736471653,
      "learning_rate": 6.0662087718167915e-06,
      "loss": -0.0398,
      "num_tokens": 19487232.0,
      "reward": 0.7295987606048584,
      "reward_std": 0.5310108661651611,
      "rewards/rollout_reward_func/mean": 0.7295987606048584,
      "rewards/rollout_reward_func/std": 0.6521875858306885,
      "sampling/importance_sampling_ratio/max": 1.7521840333938599,
      "sampling/importance_sampling_ratio/mean": 0.985211968421936,
      "sampling/importance_sampling_ratio/min": 1.7597241372402372e-11,
      "sampling/sampling_logp_difference/max": 19.941543579101562,
      "sampling/sampling_logp_difference/mean": 0.07978592813014984,
      "step": 482,
      "step_time": 15.591605683992384
    },
    {
      "clip_ratio/high_max": 0.018229166977107525,
      "clip_ratio/high_mean": 0.009114583488553762,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011067708488553762,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 820.0,
      "completions/max_terminated_length": 820.0,
      "completions/mean_length": 291.53125,
      "completions/mean_terminated_length": 283.6551818847656,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4486116785556078,
      "epoch": 0.00966,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1793714761734009,
      "kl": 0.592734482139349,
      "learning_rate": 6.0526067600857556e-06,
      "loss": -0.0158,
      "num_tokens": 19529897.0,
      "reward": 0.63866126537323,
      "reward_std": 0.6734381914138794,
      "rewards/rollout_reward_func/mean": 0.63866126537323,
      "rewards/rollout_reward_func/std": 0.7225295305252075,
      "sampling/importance_sampling_ratio/max": 1.520532488822937,
      "sampling/importance_sampling_ratio/mean": 0.9718408584594727,
      "sampling/importance_sampling_ratio/min": 0.29177165031433105,
      "sampling/sampling_logp_difference/max": 1.2708685398101807,
      "sampling/sampling_logp_difference/mean": 0.03944186121225357,
      "step": 483,
      "step_time": 12.643311843974516
    },
    {
      "clip_ratio/high_max": 0.01260504242964089,
      "clip_ratio/high_mean": 0.006302521214820445,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007279083714820445,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 254.46875,
      "completions/mean_terminated_length": 231.2857208251953,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.49572743847966194,
      "epoch": 0.00968,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3083726167678833,
      "kl": 0.5899682370945811,
      "learning_rate": 6.039007352104992e-06,
      "loss": -0.0152,
      "num_tokens": 19570746.0,
      "reward": 0.4059374928474426,
      "reward_std": 0.7621334791183472,
      "rewards/rollout_reward_func/mean": 0.4059374928474426,
      "rewards/rollout_reward_func/std": 0.7749124765396118,
      "sampling/importance_sampling_ratio/max": 1.246440052986145,
      "sampling/importance_sampling_ratio/mean": 0.9735715985298157,
      "sampling/importance_sampling_ratio/min": 0.4142396152019501,
      "sampling/sampling_logp_difference/max": 1.0067176818847656,
      "sampling/sampling_logp_difference/mean": 0.03158699348568916,
      "step": 484,
      "step_time": 12.565068464013166
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0020833334419876337,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004036458441987634,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 642.0,
      "completions/mean_length": 284.0,
      "completions/mean_terminated_length": 248.41378784179688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4995236750692129,
      "epoch": 0.0097,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2013012170791626,
      "kl": 0.622777777723968,
      "learning_rate": 6.025410727259885e-06,
      "loss": -0.0481,
      "num_tokens": 19613063.0,
      "reward": 0.5719155073165894,
      "reward_std": 0.6543756127357483,
      "rewards/rollout_reward_func/mean": 0.5719155073165894,
      "rewards/rollout_reward_func/std": 0.6737370491027832,
      "sampling/importance_sampling_ratio/max": 1.808329701423645,
      "sampling/importance_sampling_ratio/mean": 1.0101417303085327,
      "sampling/importance_sampling_ratio/min": 6.009995124073102e-20,
      "sampling/sampling_logp_difference/max": 17.049604415893555,
      "sampling/sampling_logp_difference/mean": 0.11327017843723297,
      "step": 485,
      "step_time": 13.125780942995334
    },
    {
      "clip_ratio/high_max": 0.0060614224057644606,
      "clip_ratio/high_mean": 0.0030307112028822303,
      "clip_ratio/low_mean": 0.004807692486792803,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007838403689675033,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 661.0,
      "completions/max_terminated_length": 661.0,
      "completions/mean_length": 282.90625,
      "completions/mean_terminated_length": 286.423095703125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5505399983376265,
      "epoch": 0.00972,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8877394199371338,
      "kl": 1.0495556006208062,
      "learning_rate": 6.01181706489911e-06,
      "loss": -0.1882,
      "num_tokens": 19656693.0,
      "reward": 0.6778124570846558,
      "reward_std": 0.5717473030090332,
      "rewards/rollout_reward_func/mean": 0.6778124570846558,
      "rewards/rollout_reward_func/std": 0.6615121364593506,
      "sampling/importance_sampling_ratio/max": 1.468066930770874,
      "sampling/importance_sampling_ratio/mean": 0.9924793839454651,
      "sampling/importance_sampling_ratio/min": 0.2908951938152313,
      "sampling/sampling_logp_difference/max": 0.9000253677368164,
      "sampling/sampling_logp_difference/mean": 0.0326635017991066,
      "step": 486,
      "step_time": 12.462583339991397
    },
    {
      "clip_ratio/high_max": 0.005434782709926367,
      "clip_ratio/high_mean": 0.0027173913549631834,
      "clip_ratio/low_mean": 0.0006510416860692203,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0033684330410324037,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1161.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 284.96875,
      "completions/mean_terminated_length": 258.65386962890625,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5058003626763821,
      "epoch": 0.00974,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.1322884559631348,
      "kl": 0.3138859933242202,
      "learning_rate": 5.99822654433226e-06,
      "loss": 0.1449,
      "num_tokens": 19699451.0,
      "reward": 0.7321874499320984,
      "reward_std": 0.5127275586128235,
      "rewards/rollout_reward_func/mean": 0.7321874499320984,
      "rewards/rollout_reward_func/std": 0.5972017049789429,
      "sampling/importance_sampling_ratio/max": 1.697255253791809,
      "sampling/importance_sampling_ratio/mean": 1.0150827169418335,
      "sampling/importance_sampling_ratio/min": 1.428453753722031e-18,
      "sampling/sampling_logp_difference/max": 21.339414596557617,
      "sampling/sampling_logp_difference/mean": 0.09116289764642715,
      "step": 487,
      "step_time": 15.142306769019342
    },
    {
      "clip_ratio/high_max": 0.016406250186264515,
      "clip_ratio/high_mean": 0.008203125093132257,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008203125093132257,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 676.0,
      "completions/max_terminated_length": 676.0,
      "completions/mean_length": 271.15625,
      "completions/mean_terminated_length": 274.3077087402344,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.562132578343153,
      "epoch": 0.00976,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.053108811378479,
      "kl": 0.4592092763632536,
      "learning_rate": 5.984639344827491e-06,
      "loss": 0.0931,
      "num_tokens": 19741735.0,
      "reward": 0.8075000047683716,
      "reward_std": 0.4147409498691559,
      "rewards/rollout_reward_func/mean": 0.8075000047683716,
      "rewards/rollout_reward_func/std": 0.4130999743938446,
      "sampling/importance_sampling_ratio/max": 1.4574356079101562,
      "sampling/importance_sampling_ratio/mean": 0.8446804285049438,
      "sampling/importance_sampling_ratio/min": 1.1504417898322572e-06,
      "sampling/sampling_logp_difference/max": 14.118867874145508,
      "sampling/sampling_logp_difference/mean": 0.09105387330055237,
      "step": 488,
      "step_time": 12.983007007002016
    },
    {
      "clip_ratio/high_max": 0.011083744233474135,
      "clip_ratio/high_mean": 0.005541872116737068,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006518434616737068,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 545.0,
      "completions/max_terminated_length": 545.0,
      "completions/mean_length": 165.0,
      "completions/mean_terminated_length": 145.65516662597656,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.44424000289291143,
      "epoch": 0.00978,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0354461669921875,
      "kl": 0.43405881710350513,
      "learning_rate": 5.971055645609147e-06,
      "loss": 0.0112,
      "num_tokens": 19779543.0,
      "reward": 0.7393749952316284,
      "reward_std": 0.5674850940704346,
      "rewards/rollout_reward_func/mean": 0.7393749952316284,
      "rewards/rollout_reward_func/std": 0.5787747502326965,
      "sampling/importance_sampling_ratio/max": 1.4881298542022705,
      "sampling/importance_sampling_ratio/mean": 0.9246457815170288,
      "sampling/importance_sampling_ratio/min": 1.2902952909149545e-23,
      "sampling/sampling_logp_difference/max": 26.847272872924805,
      "sampling/sampling_logp_difference/mean": 0.2509046494960785,
      "step": 489,
      "step_time": 11.95214776499779
    },
    {
      "clip_ratio/high_max": 0.0065972222946584225,
      "clip_ratio/high_mean": 0.0032986111473292112,
      "clip_ratio/low_mean": 0.002464657765813172,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005763268913142383,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 997.0,
      "completions/max_terminated_length": 997.0,
      "completions/mean_length": 243.84375,
      "completions/mean_terminated_length": 235.37930297851562,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4238915564492345,
      "epoch": 0.0098,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9987739324569702,
      "kl": 0.4911780022084713,
      "learning_rate": 5.957475625855404e-06,
      "loss": -0.0266,
      "num_tokens": 19819650.0,
      "reward": 0.5062524080276489,
      "reward_std": 0.8277736902236938,
      "rewards/rollout_reward_func/mean": 0.5062524080276489,
      "rewards/rollout_reward_func/std": 0.7971745729446411,
      "sampling/importance_sampling_ratio/max": 1.5134491920471191,
      "sampling/importance_sampling_ratio/mean": 0.942977786064148,
      "sampling/importance_sampling_ratio/min": 0.4116358458995819,
      "sampling/sampling_logp_difference/max": 0.6447745561599731,
      "sampling/sampling_logp_difference/mean": 0.03120432049036026,
      "step": 490,
      "step_time": 14.174922418998904
    },
    {
      "clip_ratio/high_max": 0.008333333535119891,
      "clip_ratio/high_mean": 0.004166666767559946,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004166666767559946,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 654.0,
      "completions/mean_length": 285.125,
      "completions/mean_terminated_length": 256.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5165462829172611,
      "epoch": 0.00982,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3690099716186523,
      "kl": 0.3008520118892193,
      "learning_rate": 5.943899464695905e-06,
      "loss": 0.0168,
      "num_tokens": 19861742.0,
      "reward": 0.7099999785423279,
      "reward_std": 0.5079376697540283,
      "rewards/rollout_reward_func/mean": 0.7099999785423279,
      "rewards/rollout_reward_func/std": 0.5395577549934387,
      "sampling/importance_sampling_ratio/max": 1.9906997680664062,
      "sampling/importance_sampling_ratio/mean": 1.009134292602539,
      "sampling/importance_sampling_ratio/min": 0.5116482973098755,
      "sampling/sampling_logp_difference/max": 0.6702733039855957,
      "sampling/sampling_logp_difference/mean": 0.03393366187810898,
      "step": 491,
      "step_time": 12.73599653999554
    },
    {
      "clip_ratio/high_max": 0.005624999990686774,
      "clip_ratio/high_mean": 0.002812499995343387,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0072767859091982245,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 255.09375,
      "completions/mean_terminated_length": 234.70370483398438,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.531410239636898,
      "epoch": 0.00984,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3029714822769165,
      "kl": 0.3238513325341046,
      "learning_rate": 5.930327341209392e-06,
      "loss": 0.015,
      "num_tokens": 19903450.0,
      "reward": 0.6675000190734863,
      "reward_std": 0.6063242554664612,
      "rewards/rollout_reward_func/mean": 0.6675000190734863,
      "rewards/rollout_reward_func/std": 0.5924416184425354,
      "sampling/importance_sampling_ratio/max": 2.698740243911743,
      "sampling/importance_sampling_ratio/mean": 1.0684597492218018,
      "sampling/importance_sampling_ratio/min": 0.5725768804550171,
      "sampling/sampling_logp_difference/max": 0.7058956623077393,
      "sampling/sampling_logp_difference/mean": 0.035224247723817825,
      "step": 492,
      "step_time": 12.294911739008967
    },
    {
      "clip_ratio/high_max": 0.014206361956894398,
      "clip_ratio/high_mean": 0.007103180978447199,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00970734772272408,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 282.09375,
      "completions/mean_terminated_length": 290.6773986816406,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.43488419195637107,
      "epoch": 0.00986,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9056732654571533,
      "kl": 0.5401133792474866,
      "learning_rate": 5.916759434421345e-06,
      "loss": -0.0201,
      "num_tokens": 19945443.0,
      "reward": 0.7030362486839294,
      "reward_std": 0.5124585628509521,
      "rewards/rollout_reward_func/mean": 0.7030362486839294,
      "rewards/rollout_reward_func/std": 0.5366871953010559,
      "sampling/importance_sampling_ratio/max": 1.4411534070968628,
      "sampling/importance_sampling_ratio/mean": 0.96360182762146,
      "sampling/importance_sampling_ratio/min": 0.5397756099700928,
      "sampling/sampling_logp_difference/max": 0.6216244697570801,
      "sampling/sampling_logp_difference/mean": 0.03611825034022331,
      "step": 493,
      "step_time": 12.969839627025067
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0096726194024086,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0116257444024086,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 609.0,
      "completions/max_terminated_length": 609.0,
      "completions/mean_length": 171.5,
      "completions/mean_terminated_length": 177.9310302734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4584157820791006,
      "epoch": 0.00988,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.43552827835083,
      "kl": 0.39004819095134735,
      "learning_rate": 5.903195923301632e-06,
      "loss": -0.0934,
      "num_tokens": 19982253.0,
      "reward": 0.7246874570846558,
      "reward_std": 0.4447484016418457,
      "rewards/rollout_reward_func/mean": 0.7246874570846558,
      "rewards/rollout_reward_func/std": 0.5820928812026978,
      "sampling/importance_sampling_ratio/max": 1.4254342317581177,
      "sampling/importance_sampling_ratio/mean": 1.0515131950378418,
      "sampling/importance_sampling_ratio/min": 0.5004634261131287,
      "sampling/sampling_logp_difference/max": 0.6632485389709473,
      "sampling/sampling_logp_difference/mean": 0.035799313336610794,
      "step": 494,
      "step_time": 11.803382893980597
    },
    {
      "clip_ratio/high_max": 0.008152173832058907,
      "clip_ratio/high_mean": 0.004076086916029453,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0066802536603063345,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 603.0,
      "completions/mean_length": 264.46875,
      "completions/mean_terminated_length": 255.48275756835938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5172843988984823,
      "epoch": 0.0099,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8453131914138794,
      "kl": 2.7011337857693434,
      "learning_rate": 5.88963698676213e-06,
      "loss": -0.1018,
      "num_tokens": 20023490.0,
      "reward": 0.5662499666213989,
      "reward_std": 0.597707986831665,
      "rewards/rollout_reward_func/mean": 0.5662499666213989,
      "rewards/rollout_reward_func/std": 0.6372255682945251,
      "sampling/importance_sampling_ratio/max": 1.3254023790359497,
      "sampling/importance_sampling_ratio/mean": 0.9311394691467285,
      "sampling/importance_sampling_ratio/min": 0.34930887818336487,
      "sampling/sampling_logp_difference/max": 1.054366111755371,
      "sampling/sampling_logp_difference/mean": 0.03547722101211548,
      "step": 495,
      "step_time": 12.233018312021159
    },
    {
      "clip_ratio/high_max": 0.005078125046566129,
      "clip_ratio/high_mean": 0.0025390625232830644,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0025390625232830644,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 681.0,
      "completions/max_terminated_length": 681.0,
      "completions/mean_length": 190.3125,
      "completions/mean_terminated_length": 196.8275909423828,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4962736591696739,
      "epoch": 0.00992,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9031919240951538,
      "kl": 0.3675548378378153,
      "learning_rate": 5.87608280365438e-06,
      "loss": 0.1305,
      "num_tokens": 20061111.0,
      "reward": 0.7228124737739563,
      "reward_std": 0.5018082857131958,
      "rewards/rollout_reward_func/mean": 0.7228124737739563,
      "rewards/rollout_reward_func/std": 0.5486477613449097,
      "sampling/importance_sampling_ratio/max": 1.614121675491333,
      "sampling/importance_sampling_ratio/mean": 1.0267574787139893,
      "sampling/importance_sampling_ratio/min": 6.004888188519382e-19,
      "sampling/sampling_logp_difference/max": 24.996654510498047,
      "sampling/sampling_logp_difference/mean": 0.1251257210969925,
      "step": 496,
      "step_time": 13.046388342001592
    },
    {
      "clip_ratio/high_max": 0.008984375046566129,
      "clip_ratio/high_mean": 0.004492187523283064,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004492187523283064,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 941.0,
      "completions/max_terminated_length": 941.0,
      "completions/mean_length": 295.34375,
      "completions/mean_terminated_length": 287.629638671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.504953196272254,
      "epoch": 0.00994,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0549015998840332,
      "kl": 0.5508249439299107,
      "learning_rate": 5.862533552767218e-06,
      "loss": -0.0913,
      "num_tokens": 20104105.0,
      "reward": 0.5868749618530273,
      "reward_std": 0.5163394808769226,
      "rewards/rollout_reward_func/mean": 0.5868749618530273,
      "rewards/rollout_reward_func/std": 0.669701099395752,
      "sampling/importance_sampling_ratio/max": 1.6968554258346558,
      "sampling/importance_sampling_ratio/mean": 0.9685809016227722,
      "sampling/importance_sampling_ratio/min": 0.4207969009876251,
      "sampling/sampling_logp_difference/max": 0.7207717895507812,
      "sampling/sampling_logp_difference/mean": 0.03607003390789032,
      "step": 497,
      "step_time": 13.351458850986091
    },
    {
      "clip_ratio/high_max": 0.008142738603055477,
      "clip_ratio/high_mean": 0.004071369301527739,
      "clip_ratio/low_mean": 0.00571428588591516,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009785655187442899,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 635.0,
      "completions/max_terminated_length": 635.0,
      "completions/mean_length": 225.875,
      "completions/mean_terminated_length": 218.3870849609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.503541387617588,
      "epoch": 0.00996,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5728458166122437,
      "kl": 0.6714431028813124,
      "learning_rate": 5.848989412824425e-06,
      "loss": -0.1467,
      "num_tokens": 20144516.0,
      "reward": 0.6109374761581421,
      "reward_std": 0.5956810712814331,
      "rewards/rollout_reward_func/mean": 0.6109374761581421,
      "rewards/rollout_reward_func/std": 0.7122039794921875,
      "sampling/importance_sampling_ratio/max": 1.4755048751831055,
      "sampling/importance_sampling_ratio/mean": 0.9101961851119995,
      "sampling/importance_sampling_ratio/min": 2.9818156832561726e-09,
      "sampling/sampling_logp_difference/max": 19.593677520751953,
      "sampling/sampling_logp_difference/mean": 0.16602692008018494,
      "step": 498,
      "step_time": 11.930218973997398
    },
    {
      "clip_ratio/high_max": 0.018058300716802478,
      "clip_ratio/high_mean": 0.009029150358401239,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013493436272256076,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 691.0,
      "completions/max_terminated_length": 691.0,
      "completions/mean_length": 176.21875,
      "completions/mean_terminated_length": 186.90000915527344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4867302207276225,
      "epoch": 0.00998,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0629953145980835,
      "kl": 0.7164223846048117,
      "learning_rate": 5.8354505624823585e-06,
      "loss": -0.1768,
      "num_tokens": 20182594.0,
      "reward": 0.4587499797344208,
      "reward_std": 0.7704893350601196,
      "rewards/rollout_reward_func/mean": 0.4587499797344208,
      "rewards/rollout_reward_func/std": 0.7969366908073425,
      "sampling/importance_sampling_ratio/max": 1.6765632629394531,
      "sampling/importance_sampling_ratio/mean": 0.894731879234314,
      "sampling/importance_sampling_ratio/min": 3.0300714339648864e-21,
      "sampling/sampling_logp_difference/max": 21.124988555908203,
      "sampling/sampling_logp_difference/mean": 0.2890772819519043,
      "step": 499,
      "step_time": 13.113257856064592
    },
    {
      "clip_ratio/high_max": 0.004267939832061529,
      "clip_ratio/high_mean": 0.0021339699160307646,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004366112872958183,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 977.0,
      "completions/max_terminated_length": 977.0,
      "completions/mean_length": 241.5,
      "completions/mean_terminated_length": 237.7096710205078,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3836466558277607,
      "epoch": 0.01,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9923121929168701,
      "kl": 1.062609069980681,
      "learning_rate": 5.821917180327612e-06,
      "loss": -0.066,
      "num_tokens": 20224045.0,
      "reward": 0.7362905144691467,
      "reward_std": 0.4619705080986023,
      "rewards/rollout_reward_func/mean": 0.7362905144691467,
      "rewards/rollout_reward_func/std": 0.5825890898704529,
      "sampling/importance_sampling_ratio/max": 1.5083789825439453,
      "sampling/importance_sampling_ratio/mean": 0.9117515087127686,
      "sampling/importance_sampling_ratio/min": 0.22295880317687988,
      "sampling/sampling_logp_difference/max": 0.965031623840332,
      "sampling/sampling_logp_difference/mean": 0.0372738242149353,
      "step": 500,
      "step_time": 13.279333097016206
    },
    {
      "clip_ratio/high_max": 0.0041082974057644606,
      "clip_ratio/high_mean": 0.0020541487028822303,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0046583154471591115,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 670.0,
      "completions/max_terminated_length": 670.0,
      "completions/mean_length": 272.15625,
      "completions/mean_terminated_length": 236.73077392578125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4867799300700426,
      "epoch": 0.01002,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3072515726089478,
      "kl": 0.3017266318202019,
      "learning_rate": 5.808389444874641e-06,
      "loss": 0.1189,
      "num_tokens": 20265465.0,
      "reward": 0.6943749785423279,
      "reward_std": 0.52289879322052,
      "rewards/rollout_reward_func/mean": 0.6943749785423279,
      "rewards/rollout_reward_func/std": 0.5387975573539734,
      "sampling/importance_sampling_ratio/max": 1.995779037475586,
      "sampling/importance_sampling_ratio/mean": 1.026234745979309,
      "sampling/importance_sampling_ratio/min": 0.6496878266334534,
      "sampling/sampling_logp_difference/max": 0.5593540668487549,
      "sampling/sampling_logp_difference/mean": 0.02827245369553566,
      "step": 501,
      "step_time": 13.550875666041975
    },
    {
      "clip_ratio/high_max": 0.007094254018738866,
      "clip_ratio/high_mean": 0.003547127009369433,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003547127009369433,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 602.0,
      "completions/max_terminated_length": 602.0,
      "completions/mean_length": 234.6875,
      "completions/mean_terminated_length": 259.0434875488281,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5294000888243318,
      "epoch": 0.01004,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6534490585327148,
      "kl": 0.41020580288022757,
      "learning_rate": 5.794867534563422e-06,
      "loss": -0.2167,
      "num_tokens": 20304447.0,
      "reward": 0.9191030263900757,
      "reward_std": 0.21477068960666656,
      "rewards/rollout_reward_func/mean": 0.9191030263900757,
      "rewards/rollout_reward_func/std": 0.28719452023506165,
      "sampling/importance_sampling_ratio/max": 1.7269561290740967,
      "sampling/importance_sampling_ratio/mean": 0.9825838804244995,
      "sampling/importance_sampling_ratio/min": 0.2685161232948303,
      "sampling/sampling_logp_difference/max": 1.1997923851013184,
      "sampling/sampling_logp_difference/mean": 0.035199735313653946,
      "step": 502,
      "step_time": 12.619868452005903
    },
    {
      "clip_ratio/high_max": 0.0067471591755747795,
      "clip_ratio/high_mean": 0.0033735795877873898,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0033735795877873898,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 802.0,
      "completions/max_terminated_length": 802.0,
      "completions/mean_length": 220.96875,
      "completions/mean_terminated_length": 214.1724090576172,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4183343150652945,
      "epoch": 0.01006,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5692899227142334,
      "kl": 1.0173662300221622,
      "learning_rate": 5.781351627757093e-06,
      "loss": -0.0441,
      "num_tokens": 20344799.0,
      "reward": 0.4740648865699768,
      "reward_std": 0.7657590508460999,
      "rewards/rollout_reward_func/mean": 0.4740648865699768,
      "rewards/rollout_reward_func/std": 0.8356283903121948,
      "sampling/importance_sampling_ratio/max": 1.6521607637405396,
      "sampling/importance_sampling_ratio/mean": 1.010325312614441,
      "sampling/importance_sampling_ratio/min": 0.37138643860816956,
      "sampling/sampling_logp_difference/max": 1.0365362167358398,
      "sampling/sampling_logp_difference/mean": 0.03307412564754486,
      "step": 503,
      "step_time": 12.584027429009438
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 664.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 224.90625,
      "completions/mean_terminated_length": 201.90000915527344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.46527902223169804,
      "epoch": 0.01008,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.35558021068573,
      "kl": 1.033523271791637,
      "learning_rate": 5.767841902739602e-06,
      "loss": -0.0348,
      "num_tokens": 20383802.0,
      "reward": 0.7005362510681152,
      "reward_std": 0.5996394157409668,
      "rewards/rollout_reward_func/mean": 0.7005362510681152,
      "rewards/rollout_reward_func/std": 0.6213235855102539,
      "sampling/importance_sampling_ratio/max": 1.3971738815307617,
      "sampling/importance_sampling_ratio/mean": 0.9965034127235413,
      "sampling/importance_sampling_ratio/min": 0.260945200920105,
      "sampling/sampling_logp_difference/max": 1.004492998123169,
      "sampling/sampling_logp_difference/mean": 0.03192553669214249,
      "step": 504,
      "step_time": 12.67749447200913
    },
    {
      "clip_ratio/high_max": 0.03515625,
      "clip_ratio/high_mean": 0.017578125,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0185546875,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 286.09375,
      "completions/mean_terminated_length": 287.96429443359375,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.48489243909716606,
      "epoch": 0.0101,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9843109250068665,
      "kl": 0.34397974982857704,
      "learning_rate": 5.754338537713353e-06,
      "loss": 0.0022,
      "num_tokens": 20426785.0,
      "reward": 0.8118749856948853,
      "reward_std": 0.47957462072372437,
      "rewards/rollout_reward_func/mean": 0.8118749856948853,
      "rewards/rollout_reward_func/std": 0.5508259534835815,
      "sampling/importance_sampling_ratio/max": 1.8306729793548584,
      "sampling/importance_sampling_ratio/mean": 0.9834244251251221,
      "sampling/importance_sampling_ratio/min": 0.2944827675819397,
      "sampling/sampling_logp_difference/max": 0.5956721305847168,
      "sampling/sampling_logp_difference/mean": 0.03297305852174759,
      "step": 505,
      "step_time": 13.042056854028488
    },
    {
      "clip_ratio/high_max": 0.025595238897949457,
      "clip_ratio/high_mean": 0.012797619448974729,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012797619448974729,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 244.1875,
      "completions/mean_terminated_length": 244.1875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.42086617927998304,
      "epoch": 0.01012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.00095796585083,
      "kl": 0.9408911867067218,
      "learning_rate": 5.740841710796861e-06,
      "loss": 0.118,
      "num_tokens": 20466571.0,
      "reward": 0.6071875095367432,
      "reward_std": 0.591804027557373,
      "rewards/rollout_reward_func/mean": 0.6071875095367432,
      "rewards/rollout_reward_func/std": 0.6068545579910278,
      "sampling/importance_sampling_ratio/max": 1.2102773189544678,
      "sampling/importance_sampling_ratio/mean": 0.9848717451095581,
      "sampling/importance_sampling_ratio/min": 0.4142346680164337,
      "sampling/sampling_logp_difference/max": 0.8455257415771484,
      "sampling/sampling_logp_difference/mean": 0.02877703681588173,
      "step": 506,
      "step_time": 11.960473143946729
    },
    {
      "clip_ratio/high_max": 0.005434782709926367,
      "clip_ratio/high_mean": 0.0027173913549631834,
      "clip_ratio/low_mean": 0.0010775862028822303,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0037949775578454137,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 660.0,
      "completions/mean_length": 215.25,
      "completions/mean_terminated_length": 222.85714721679688,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4859074577689171,
      "epoch": 0.01014,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1501355171203613,
      "kl": 0.3523573214188218,
      "learning_rate": 5.727351600022396e-06,
      "loss": 0.0658,
      "num_tokens": 20505836.0,
      "reward": 0.6781250238418579,
      "reward_std": 0.5319742560386658,
      "rewards/rollout_reward_func/mean": 0.6781250238418579,
      "rewards/rollout_reward_func/std": 0.5398532152175903,
      "sampling/importance_sampling_ratio/max": 1.3911583423614502,
      "sampling/importance_sampling_ratio/mean": 0.9558190107345581,
      "sampling/importance_sampling_ratio/min": 2.77695905367474e-19,
      "sampling/sampling_logp_difference/max": 25.369861602783203,
      "sampling/sampling_logp_difference/mean": 0.20666441321372986,
      "step": 507,
      "step_time": 12.65733512303268
    },
    {
      "clip_ratio/high_max": 0.01878063753247261,
      "clip_ratio/high_mean": 0.009390318766236305,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009390318766236305,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1042.0,
      "completions/max_terminated_length": 1042.0,
      "completions/mean_length": 284.96875,
      "completions/mean_terminated_length": 288.0333557128906,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4824395263567567,
      "epoch": 0.01016,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2611922025680542,
      "kl": 0.4459971571341157,
      "learning_rate": 5.7138683833336385e-06,
      "loss": -0.1292,
      "num_tokens": 20548773.0,
      "reward": 0.6409398913383484,
      "reward_std": 0.580615758895874,
      "rewards/rollout_reward_func/mean": 0.6409398913383484,
      "rewards/rollout_reward_func/std": 0.7223884463310242,
      "sampling/importance_sampling_ratio/max": 1.5182360410690308,
      "sampling/importance_sampling_ratio/mean": 0.8955122828483582,
      "sampling/importance_sampling_ratio/min": 1.998855616164109e-20,
      "sampling/sampling_logp_difference/max": 24.560504913330078,
      "sampling/sampling_logp_difference/mean": 0.13500837981700897,
      "step": 508,
      "step_time": 14.222489883002709
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00390625,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 315.3125,
      "completions/mean_terminated_length": 307.6896667480469,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4447146374732256,
      "epoch": 0.01018,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.179787278175354,
      "kl": 0.4030751148238778,
      "learning_rate": 5.70039223858333e-06,
      "loss": 0.1463,
      "num_tokens": 20593149.0,
      "reward": 0.5853124856948853,
      "reward_std": 0.581091046333313,
      "rewards/rollout_reward_func/mean": 0.5853124856948853,
      "rewards/rollout_reward_func/std": 0.6092511415481567,
      "sampling/importance_sampling_ratio/max": 2.0916805267333984,
      "sampling/importance_sampling_ratio/mean": 1.0229259729385376,
      "sampling/importance_sampling_ratio/min": 0.5656958222389221,
      "sampling/sampling_logp_difference/max": 0.4776124954223633,
      "sampling/sampling_logp_difference/mean": 0.029663272202014923,
      "step": 509,
      "step_time": 12.29260775198054
    },
    {
      "clip_ratio/high_max": 0.03134726965799928,
      "clip_ratio/high_mean": 0.01567363482899964,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01567363482899964,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 927.0,
      "completions/max_terminated_length": 883.0,
      "completions/mean_length": 290.53125,
      "completions/mean_terminated_length": 277.6071472167969,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4900618027895689,
      "epoch": 0.0102,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7788496017456055,
      "kl": 1.1878245091065764,
      "learning_rate": 5.686923343530932e-06,
      "loss": -0.1524,
      "num_tokens": 20635991.0,
      "reward": 0.8134399056434631,
      "reward_std": 0.41297847032546997,
      "rewards/rollout_reward_func/mean": 0.8134399056434631,
      "rewards/rollout_reward_func/std": 0.4889494478702545,
      "sampling/importance_sampling_ratio/max": 1.2577437162399292,
      "sampling/importance_sampling_ratio/mean": 0.8335211873054504,
      "sampling/importance_sampling_ratio/min": 4.0653553657135633e-13,
      "sampling/sampling_logp_difference/max": 28.706411361694336,
      "sampling/sampling_logp_difference/mean": 0.18321679532527924,
      "step": 510,
      "step_time": 13.826014349004254
    },
    {
      "clip_ratio/high_max": 0.010633680736646056,
      "clip_ratio/high_mean": 0.005316840368323028,
      "clip_ratio/low_mean": 0.005208333372138441,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010525173740461469,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 584.0,
      "completions/max_terminated_length": 584.0,
      "completions/mean_length": 236.375,
      "completions/mean_terminated_length": 239.55172729492188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4169011740013957,
      "epoch": 0.01022,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9977211356163025,
      "kl": 0.7280426416546106,
      "learning_rate": 5.673461875840275e-06,
      "loss": -0.135,
      "num_tokens": 20676810.0,
      "reward": 0.6731250286102295,
      "reward_std": 0.5191136598587036,
      "rewards/rollout_reward_func/mean": 0.6731250286102295,
      "rewards/rollout_reward_func/std": 0.5979180932044983,
      "sampling/importance_sampling_ratio/max": 1.396702527999878,
      "sampling/importance_sampling_ratio/mean": 0.9852056503295898,
      "sampling/importance_sampling_ratio/min": 0.53899085521698,
      "sampling/sampling_logp_difference/max": 0.36316537857055664,
      "sampling/sampling_logp_difference/mean": 0.030872289091348648,
      "step": 511,
      "step_time": 12.35328783300065
    },
    {
      "clip_ratio/high_max": 0.016741071827709675,
      "clip_ratio/high_mean": 0.008370535913854837,
      "clip_ratio/low_mean": 0.0027173913549631834,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011087927501648664,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 630.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 320.875,
      "completions/mean_terminated_length": 319.6551818847656,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.44376610312610865,
      "epoch": 0.01024,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.021292805671692,
      "kl": 0.7222929038107395,
      "learning_rate": 5.6600080130772166e-06,
      "loss": -0.0027,
      "num_tokens": 20720870.0,
      "reward": 0.6203124523162842,
      "reward_std": 0.6289123296737671,
      "rewards/rollout_reward_func/mean": 0.6203124523162842,
      "rewards/rollout_reward_func/std": 0.6107978820800781,
      "sampling/importance_sampling_ratio/max": 1.6019476652145386,
      "sampling/importance_sampling_ratio/mean": 1.0417838096618652,
      "sampling/importance_sampling_ratio/min": 0.4253799021244049,
      "sampling/sampling_logp_difference/max": 0.540992259979248,
      "sampling/sampling_logp_difference/mean": 0.03179177641868591,
      "step": 512,
      "step_time": 12.099878584995167
    },
    {
      "clip_ratio/high_max": 0.008184524020180106,
      "clip_ratio/high_mean": 0.004092262010090053,
      "clip_ratio/low_mean": 0.0010775862028822303,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005169848212972283,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 661.0,
      "completions/max_terminated_length": 636.0,
      "completions/mean_length": 262.9375,
      "completions/mean_terminated_length": 226.25001525878906,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4704522155225277,
      "epoch": 0.01026,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5510739088058472,
      "kl": 0.5033085653558373,
      "learning_rate": 5.646561932707302e-06,
      "loss": 0.07,
      "num_tokens": 20761667.0,
      "reward": 0.7371875047683716,
      "reward_std": 0.5856871008872986,
      "rewards/rollout_reward_func/mean": 0.7371875047683716,
      "rewards/rollout_reward_func/std": 0.5887944102287292,
      "sampling/importance_sampling_ratio/max": 1.4017162322998047,
      "sampling/importance_sampling_ratio/mean": 1.015608310699463,
      "sampling/importance_sampling_ratio/min": 0.31355389952659607,
      "sampling/sampling_logp_difference/max": 1.154618263244629,
      "sampling/sampling_logp_difference/mean": 0.030402274802327156,
      "step": 513,
      "step_time": 13.26071826502448
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.007936508161947131,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01574900816194713,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 233.53125,
      "completions/mean_terminated_length": 233.06668090820312,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.39535771030932665,
      "epoch": 0.01028,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1312023401260376,
      "kl": 0.37469199020415545,
      "learning_rate": 5.633123812093419e-06,
      "loss": -0.001,
      "num_tokens": 20801472.0,
      "reward": 0.77406245470047,
      "reward_std": 0.49869856238365173,
      "rewards/rollout_reward_func/mean": 0.77406245470047,
      "rewards/rollout_reward_func/std": 0.5143542885780334,
      "sampling/importance_sampling_ratio/max": 2.124547004699707,
      "sampling/importance_sampling_ratio/mean": 1.0169355869293213,
      "sampling/importance_sampling_ratio/min": 1.862022817624219e-20,
      "sampling/sampling_logp_difference/max": 24.490970611572266,
      "sampling/sampling_logp_difference/mean": 0.117557093501091,
      "step": 514,
      "step_time": 12.25947754495428
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 635.0,
      "completions/max_terminated_length": 627.0,
      "completions/mean_length": 185.8125,
      "completions/mean_terminated_length": 171.35714721679688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4504371816292405,
      "epoch": 0.0103,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0446624755859375,
      "kl": 0.8632713183760643,
      "learning_rate": 5.619693828493467e-06,
      "loss": -0.24,
      "num_tokens": 20838819.0,
      "reward": 0.6737499833106995,
      "reward_std": 0.5425999164581299,
      "rewards/rollout_reward_func/mean": 0.6737499833106995,
      "rewards/rollout_reward_func/std": 0.6588638424873352,
      "sampling/importance_sampling_ratio/max": 2.50164532661438,
      "sampling/importance_sampling_ratio/mean": 1.0703644752502441,
      "sampling/importance_sampling_ratio/min": 0.3795156180858612,
      "sampling/sampling_logp_difference/max": 0.9716615676879883,
      "sampling/sampling_logp_difference/mean": 0.0341218039393425,
      "step": 515,
      "step_time": 11.98454632695939
    },
    {
      "clip_ratio/high_max": 0.04959239135496318,
      "clip_ratio/high_mean": 0.02479619567748159,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02479619567748159,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1087.0,
      "completions/max_terminated_length": 1087.0,
      "completions/mean_length": 241.34375,
      "completions/mean_terminated_length": 256.3666687011719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.47695760801434517,
      "epoch": 0.01032,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0391013622283936,
      "kl": 0.8752263467758894,
      "learning_rate": 5.606272159058005e-06,
      "loss": 0.0062,
      "num_tokens": 20879246.0,
      "reward": 0.8221874833106995,
      "reward_std": 0.44620922207832336,
      "rewards/rollout_reward_func/mean": 0.8221874833106995,
      "rewards/rollout_reward_func/std": 0.4656922221183777,
      "sampling/importance_sampling_ratio/max": 1.5390456914901733,
      "sampling/importance_sampling_ratio/mean": 0.9636722803115845,
      "sampling/importance_sampling_ratio/min": 0.25429216027259827,
      "sampling/sampling_logp_difference/max": 1.1159443855285645,
      "sampling/sampling_logp_difference/mean": 0.03803683817386627,
      "step": 516,
      "step_time": 14.157501692970982
    },
    {
      "clip_ratio/high_max": 0.008470504777505994,
      "clip_ratio/high_mean": 0.004235252388752997,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006188377388752997,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 640.0,
      "completions/mean_length": 185.8125,
      "completions/mean_terminated_length": 165.60714721679688,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.45260122418403625,
      "epoch": 0.01034,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0217013359069824,
      "kl": 0.8584559324663132,
      "learning_rate": 5.5928589808279266e-06,
      "loss": 0.0844,
      "num_tokens": 20916327.0,
      "reward": 0.9175000190734863,
      "reward_std": 0.21484187245368958,
      "rewards/rollout_reward_func/mean": 0.9175000190734863,
      "rewards/rollout_reward_func/std": 0.28870120644569397,
      "sampling/importance_sampling_ratio/max": 1.7463374137878418,
      "sampling/importance_sampling_ratio/mean": 1.0453128814697266,
      "sampling/importance_sampling_ratio/min": 0.5151630640029907,
      "sampling/sampling_logp_difference/max": 0.6675753593444824,
      "sampling/sampling_logp_difference/mean": 0.029520878568291664,
      "step": 517,
      "step_time": 12.579673949963762
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006138392956927419,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 200.5,
      "completions/mean_terminated_length": 219.58621215820312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.38471975084394217,
      "epoch": 0.01036,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8748000860214233,
      "kl": 1.0341512197628617,
      "learning_rate": 5.5794544707321184e-06,
      "loss": -0.0865,
      "num_tokens": 20955421.0,
      "reward": 0.4841029942035675,
      "reward_std": 0.7720350027084351,
      "rewards/rollout_reward_func/mean": 0.4841029942035675,
      "rewards/rollout_reward_func/std": 0.7702271342277527,
      "sampling/importance_sampling_ratio/max": 1.3924381732940674,
      "sampling/importance_sampling_ratio/mean": 0.9506963491439819,
      "sampling/importance_sampling_ratio/min": 0.39308229088783264,
      "sampling/sampling_logp_difference/max": 0.904393196105957,
      "sampling/sampling_logp_difference/mean": 0.03367568925023079,
      "step": 518,
      "step_time": 12.47809115301061
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 919.0,
      "completions/max_terminated_length": 919.0,
      "completions/mean_length": 212.6875,
      "completions/mean_terminated_length": 225.80001831054688,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4259872641414404,
      "epoch": 0.01038,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7305752038955688,
      "kl": 0.7173825548961759,
      "learning_rate": 5.566058805585135e-06,
      "loss": -0.0305,
      "num_tokens": 20994337.0,
      "reward": 0.7368749976158142,
      "reward_std": 0.46732258796691895,
      "rewards/rollout_reward_func/mean": 0.7368749976158142,
      "rewards/rollout_reward_func/std": 0.5878305435180664,
      "sampling/importance_sampling_ratio/max": 1.2276256084442139,
      "sampling/importance_sampling_ratio/mean": 0.9397885799407959,
      "sampling/importance_sampling_ratio/min": 0.3711407482624054,
      "sampling/sampling_logp_difference/max": 0.8497190475463867,
      "sampling/sampling_logp_difference/mean": 0.034044113010168076,
      "step": 519,
      "step_time": 13.438957356018363
    },
    {
      "clip_ratio/high_max": 0.028515625512227416,
      "clip_ratio/high_mean": 0.014257812756113708,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014257812756113708,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 282.375,
      "completions/mean_terminated_length": 284.8000183105469,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4422909850254655,
      "epoch": 0.0104,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6302863359451294,
      "kl": 0.5982748847454786,
      "learning_rate": 5.55267216208485e-06,
      "loss": -0.0296,
      "num_tokens": 21037002.0,
      "reward": 0.7065624594688416,
      "reward_std": 0.4271407723426819,
      "rewards/rollout_reward_func/mean": 0.7065624594688416,
      "rewards/rollout_reward_func/std": 0.5452528595924377,
      "sampling/importance_sampling_ratio/max": 1.373436689376831,
      "sampling/importance_sampling_ratio/mean": 0.9487498998641968,
      "sampling/importance_sampling_ratio/min": 6.2080789592139016e-15,
      "sampling/sampling_logp_difference/max": 24.338857650756836,
      "sampling/sampling_logp_difference/mean": 0.1035454124212265,
      "step": 520,
      "step_time": 12.26070506501128
    },
    {
      "clip_ratio/high_max": 0.002016128972172737,
      "clip_ratio/high_mean": 0.0010080644860863686,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0010080644860863686,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 208.59375,
      "completions/mean_terminated_length": 203.2413787841797,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.38864006474614143,
      "epoch": 0.01042,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1407440900802612,
      "kl": 1.0043902890756726,
      "learning_rate": 5.539294716810144e-06,
      "loss": -0.0538,
      "num_tokens": 21076689.0,
      "reward": 0.6715624928474426,
      "reward_std": 0.6903233528137207,
      "rewards/rollout_reward_func/mean": 0.6715624928474426,
      "rewards/rollout_reward_func/std": 0.7011746168136597,
      "sampling/importance_sampling_ratio/max": 1.345316767692566,
      "sampling/importance_sampling_ratio/mean": 0.9028407335281372,
      "sampling/importance_sampling_ratio/min": 0.43621060252189636,
      "sampling/sampling_logp_difference/max": 0.8444395065307617,
      "sampling/sampling_logp_difference/mean": 0.03467527776956558,
      "step": 521,
      "step_time": 12.567680697015021
    },
    {
      "clip_ratio/high_max": 0.015323925530537963,
      "clip_ratio/high_mean": 0.0076619627652689815,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0076619627652689815,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 545.0,
      "completions/max_terminated_length": 545.0,
      "completions/mean_length": 159.78125,
      "completions/mean_terminated_length": 158.44444274902344,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.48451678082346916,
      "epoch": 0.01044,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5820759534835815,
      "kl": 0.44311715476214886,
      "learning_rate": 5.525926646218561e-06,
      "loss": -0.1004,
      "num_tokens": 21112802.0,
      "reward": 0.8603124618530273,
      "reward_std": 0.3808889389038086,
      "rewards/rollout_reward_func/mean": 0.8603124618530273,
      "rewards/rollout_reward_func/std": 0.47041836380958557,
      "sampling/importance_sampling_ratio/max": 1.4344428777694702,
      "sampling/importance_sampling_ratio/mean": 0.9255812168121338,
      "sampling/importance_sampling_ratio/min": 3.777881279855645e-21,
      "sampling/sampling_logp_difference/max": 26.68026351928711,
      "sampling/sampling_logp_difference/mean": 0.21927723288536072,
      "step": 522,
      "step_time": 12.481642964994535
    },
    {
      "clip_ratio/high_max": 0.02734375069849193,
      "clip_ratio/high_mean": 0.013671875349245965,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013671875349245965,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 283.65625,
      "completions/mean_terminated_length": 285.58062744140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4660273902118206,
      "epoch": 0.01046,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0546311140060425,
      "kl": 0.7729925001040101,
      "learning_rate": 5.512568126643991e-06,
      "loss": -0.1806,
      "num_tokens": 21155654.0,
      "reward": 0.715624988079071,
      "reward_std": 0.42502766847610474,
      "rewards/rollout_reward_func/mean": 0.715624988079071,
      "rewards/rollout_reward_func/std": 0.5852097868919373,
      "sampling/importance_sampling_ratio/max": 1.5586284399032593,
      "sampling/importance_sampling_ratio/mean": 0.9688190221786499,
      "sampling/importance_sampling_ratio/min": 0.35728809237480164,
      "sampling/sampling_logp_difference/max": 0.6976923942565918,
      "sampling/sampling_logp_difference/mean": 0.034321315586566925,
      "step": 523,
      "step_time": 12.396232206010609
    },
    {
      "clip_ratio/high_max": 0.02083333395421505,
      "clip_ratio/high_mean": 0.010416666977107525,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010416666977107525,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 1327.0,
      "completions/max_terminated_length": 1327.0,
      "completions/mean_length": 230.71875,
      "completions/mean_terminated_length": 247.74073791503906,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.4085651896893978,
      "epoch": 0.01048,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9595917463302612,
      "kl": 0.3399368319660425,
      "learning_rate": 5.499219334294335e-06,
      "loss": -0.0173,
      "num_tokens": 21194765.0,
      "reward": 0.7990624904632568,
      "reward_std": 0.42879533767700195,
      "rewards/rollout_reward_func/mean": 0.7990624904632568,
      "rewards/rollout_reward_func/std": 0.4865703284740448,
      "sampling/importance_sampling_ratio/max": 1.2505204677581787,
      "sampling/importance_sampling_ratio/mean": 0.9807529449462891,
      "sampling/importance_sampling_ratio/min": 0.38950076699256897,
      "sampling/sampling_logp_difference/max": 0.5576930046081543,
      "sampling/sampling_logp_difference/mean": 0.02902875654399395,
      "step": 524,
      "step_time": 15.554224976964178
    },
    {
      "clip_ratio/high_max": 0.011382980505004525,
      "clip_ratio/high_mean": 0.005691490252502263,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005691490252502263,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 624.0,
      "completions/max_terminated_length": 624.0,
      "completions/mean_length": 219.71875,
      "completions/mean_terminated_length": 212.16128540039062,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4456874430179596,
      "epoch": 0.0105,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2658065557479858,
      "kl": 0.5735981082543731,
      "learning_rate": 5.485880445249192e-06,
      "loss": -0.0842,
      "num_tokens": 21234369.0,
      "reward": 0.7628124952316284,
      "reward_std": 0.47531765699386597,
      "rewards/rollout_reward_func/mean": 0.7628124952316284,
      "rewards/rollout_reward_func/std": 0.5879610776901245,
      "sampling/importance_sampling_ratio/max": 1.6384309530258179,
      "sampling/importance_sampling_ratio/mean": 1.0905534029006958,
      "sampling/importance_sampling_ratio/min": 0.29855284094810486,
      "sampling/sampling_logp_difference/max": 0.5434079170227051,
      "sampling/sampling_logp_difference/mean": 0.030061732977628708,
      "step": 525,
      "step_time": 12.645480013990891
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 679.0,
      "completions/max_terminated_length": 679.0,
      "completions/mean_length": 289.1875,
      "completions/mean_terminated_length": 307.77777099609375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.552153492346406,
      "epoch": 0.01052,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0556790828704834,
      "kl": 0.41271738614887,
      "learning_rate": 5.472551635457521e-06,
      "loss": -0.0421,
      "num_tokens": 21276920.0,
      "reward": 0.5190624594688416,
      "reward_std": 0.6914987564086914,
      "rewards/rollout_reward_func/mean": 0.5190624594688416,
      "rewards/rollout_reward_func/std": 0.6859340667724609,
      "sampling/importance_sampling_ratio/max": 1.234389305114746,
      "sampling/importance_sampling_ratio/mean": 0.9030352830886841,
      "sampling/importance_sampling_ratio/min": 1.2034128483282458e-15,
      "sampling/sampling_logp_difference/max": 21.376121520996094,
      "sampling/sampling_logp_difference/mean": 0.09385383874177933,
      "step": 526,
      "step_time": 12.716868603019975
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 670.0,
      "completions/max_terminated_length": 670.0,
      "completions/mean_length": 246.25,
      "completions/mean_terminated_length": 240.90000915527344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.42125310748815536,
      "epoch": 0.01054,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3653303384780884,
      "kl": 0.7154541416093707,
      "learning_rate": 5.459233080735336e-06,
      "loss": 0.0928,
      "num_tokens": 21316923.0,
      "reward": 0.6390624642372131,
      "reward_std": 0.6146109104156494,
      "rewards/rollout_reward_func/mean": 0.6390624642372131,
      "rewards/rollout_reward_func/std": 0.6550909876823425,
      "sampling/importance_sampling_ratio/max": 1.4237477779388428,
      "sampling/importance_sampling_ratio/mean": 0.9800063967704773,
      "sampling/importance_sampling_ratio/min": 0.5703027844429016,
      "sampling/sampling_logp_difference/max": 0.5178694725036621,
      "sampling/sampling_logp_difference/mean": 0.03399931266903877,
      "step": 527,
      "step_time": 12.28524418095185
    },
    {
      "clip_ratio/high_max": 0.02398897078819573,
      "clip_ratio/high_mean": 0.011994485394097865,
      "clip_ratio/low_mean": 0.00390625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015900735394097865,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 275.6875,
      "completions/mean_terminated_length": 280.73077392578125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.432139465585351,
      "epoch": 0.01056,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0095610618591309,
      "kl": 0.6147180860862136,
      "learning_rate": 5.4459249567633776e-06,
      "loss": 0.0483,
      "num_tokens": 21358349.0,
      "reward": 0.6468750238418579,
      "reward_std": 0.5073697566986084,
      "rewards/rollout_reward_func/mean": 0.6468750238418579,
      "rewards/rollout_reward_func/std": 0.60666424036026,
      "sampling/importance_sampling_ratio/max": 1.380159854888916,
      "sampling/importance_sampling_ratio/mean": 0.8757867813110352,
      "sampling/importance_sampling_ratio/min": 1.0713530173234176e-05,
      "sampling/sampling_logp_difference/max": 11.628850936889648,
      "sampling/sampling_logp_difference/mean": 0.0805082768201828,
      "step": 528,
      "step_time": 12.488291551009752
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 227.0,
      "completions/mean_terminated_length": 213.74192810058594,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4462793515995145,
      "epoch": 0.01058,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3943713903427124,
      "kl": 0.3749307533726096,
      "learning_rate": 5.432627439084797e-06,
      "loss": 0.0835,
      "num_tokens": 21398251.0,
      "reward": 0.7631250023841858,
      "reward_std": 0.45072126388549805,
      "rewards/rollout_reward_func/mean": 0.7631250023841858,
      "rewards/rollout_reward_func/std": 0.4417591989040375,
      "sampling/importance_sampling_ratio/max": 1.2284762859344482,
      "sampling/importance_sampling_ratio/mean": 0.9199349880218506,
      "sampling/importance_sampling_ratio/min": 0.46904468536376953,
      "sampling/sampling_logp_difference/max": 0.7631180286407471,
      "sampling/sampling_logp_difference/mean": 0.036787014454603195,
      "step": 529,
      "step_time": 11.890196496984572
    },
    {
      "clip_ratio/high_max": 0.010881696827709675,
      "clip_ratio/high_mean": 0.005440848413854837,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005440848413854837,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 652.0,
      "completions/max_terminated_length": 652.0,
      "completions/mean_length": 215.59375,
      "completions/mean_terminated_length": 213.87095642089844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4087417433038354,
      "epoch": 0.0106,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9795897603034973,
      "kl": 0.39070712961256504,
      "learning_rate": 5.419340703102839e-06,
      "loss": -0.0921,
      "num_tokens": 21437284.0,
      "reward": 0.6156274080276489,
      "reward_std": 0.6488363742828369,
      "rewards/rollout_reward_func/mean": 0.6156274080276489,
      "rewards/rollout_reward_func/std": 0.6700294017791748,
      "sampling/importance_sampling_ratio/max": 1.5658860206604004,
      "sampling/importance_sampling_ratio/mean": 1.0345391035079956,
      "sampling/importance_sampling_ratio/min": 0.40141957998275757,
      "sampling/sampling_logp_difference/max": 0.5957798957824707,
      "sampling/sampling_logp_difference/mean": 0.027643408626317978,
      "step": 530,
      "step_time": 12.948982849018648
    },
    {
      "clip_ratio/high_max": 0.0026041667442768812,
      "clip_ratio/high_mean": 0.0013020833721384406,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0032552083721384406,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1130.0,
      "completions/max_terminated_length": 1130.0,
      "completions/mean_length": 279.34375,
      "completions/mean_terminated_length": 263.5333557128906,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.3700306685641408,
      "epoch": 0.01062,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2509407997131348,
      "kl": 0.4512772345915437,
      "learning_rate": 5.406064924078536e-06,
      "loss": 0.0745,
      "num_tokens": 21478996.0,
      "reward": 0.7724999785423279,
      "reward_std": 0.44013711810112,
      "rewards/rollout_reward_func/mean": 0.7724999785423279,
      "rewards/rollout_reward_func/std": 0.4971012473106384,
      "sampling/importance_sampling_ratio/max": 1.4755803346633911,
      "sampling/importance_sampling_ratio/mean": 1.0084322690963745,
      "sampling/importance_sampling_ratio/min": 0.5707242488861084,
      "sampling/sampling_logp_difference/max": 0.6127829551696777,
      "sampling/sampling_logp_difference/mean": 0.02351798117160797,
      "step": 531,
      "step_time": 13.73822121102421
    },
    {
      "clip_ratio/high_max": 0.018601190531626344,
      "clip_ratio/high_mean": 0.009300595265813172,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009300595265813172,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 625.0,
      "completions/max_terminated_length": 625.0,
      "completions/mean_length": 189.03125,
      "completions/mean_terminated_length": 173.8000030517578,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.43787859566509724,
      "epoch": 0.01064,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6110143661499023,
      "kl": 0.5845847288146615,
      "learning_rate": 5.392800277128386e-06,
      "loss": -0.0025,
      "num_tokens": 21516295.0,
      "reward": 0.8537499904632568,
      "reward_std": 0.3794577419757843,
      "rewards/rollout_reward_func/mean": 0.8537499904632568,
      "rewards/rollout_reward_func/std": 0.36834195256233215,
      "sampling/importance_sampling_ratio/max": 1.4396109580993652,
      "sampling/importance_sampling_ratio/mean": 0.9749913215637207,
      "sampling/importance_sampling_ratio/min": 0.3931799829006195,
      "sampling/sampling_logp_difference/max": 0.9505786895751953,
      "sampling/sampling_logp_difference/mean": 0.033804867416620255,
      "step": 532,
      "step_time": 11.752200554998126
    },
    {
      "clip_ratio/high_max": 0.024349256418645382,
      "clip_ratio/high_mean": 0.012174628209322691,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012174628209322691,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 631.0,
      "completions/max_terminated_length": 631.0,
      "completions/mean_length": 221.9375,
      "completions/mean_terminated_length": 223.03334045410156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4404160864651203,
      "epoch": 0.01066,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.253761887550354,
      "kl": 1.0121531393378973,
      "learning_rate": 5.3795469372220525e-06,
      "loss": 0.0298,
      "num_tokens": 21556400.0,
      "reward": 0.957812488079071,
      "reward_std": 0.19968663156032562,
      "rewards/rollout_reward_func/mean": 0.957812488079071,
      "rewards/rollout_reward_func/std": 0.24834592640399933,
      "sampling/importance_sampling_ratio/max": 1.168361783027649,
      "sampling/importance_sampling_ratio/mean": 0.9094431400299072,
      "sampling/importance_sampling_ratio/min": 0.2873983085155487,
      "sampling/sampling_logp_difference/max": 0.6765539646148682,
      "sampling/sampling_logp_difference/mean": 0.03213968873023987,
      "step": 533,
      "step_time": 13.15488531300798
    },
    {
      "clip_ratio/high_max": 0.010937500046566129,
      "clip_ratio/high_mean": 0.005468750023283064,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005468750023283064,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1000.0,
      "completions/max_terminated_length": 1000.0,
      "completions/mean_length": 314.53125,
      "completions/mean_terminated_length": 321.46429443359375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4013777421787381,
      "epoch": 0.01068,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0778136253356934,
      "kl": 0.334024447016418,
      "learning_rate": 5.366305079180043e-06,
      "loss": 0.0309,
      "num_tokens": 21599895.0,
      "reward": 0.7731249928474426,
      "reward_std": 0.45564723014831543,
      "rewards/rollout_reward_func/mean": 0.7731249928474426,
      "rewards/rollout_reward_func/std": 0.43343663215637207,
      "sampling/importance_sampling_ratio/max": 1.6789472103118896,
      "sampling/importance_sampling_ratio/mean": 0.9383560419082642,
      "sampling/importance_sampling_ratio/min": 0.32132378220558167,
      "sampling/sampling_logp_difference/max": 0.5795352458953857,
      "sampling/sampling_logp_difference/mean": 0.029195580631494522,
      "step": 534,
      "step_time": 13.354881287013995
    },
    {
      "clip_ratio/high_max": 0.004464285913854837,
      "clip_ratio/high_mean": 0.0022321429569274187,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022321429569274187,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 599.0,
      "completions/max_terminated_length": 599.0,
      "completions/mean_length": 275.125,
      "completions/mean_terminated_length": 260.1000061035156,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.42555877566337585,
      "epoch": 0.0107,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2190601825714111,
      "kl": 0.42080489452928305,
      "learning_rate": 5.35307487767142e-06,
      "loss": -0.0579,
      "num_tokens": 21642450.0,
      "reward": 0.6203124523162842,
      "reward_std": 0.5683451890945435,
      "rewards/rollout_reward_func/mean": 0.6203124523162842,
      "rewards/rollout_reward_func/std": 0.5617626309394836,
      "sampling/importance_sampling_ratio/max": 1.4526833295822144,
      "sampling/importance_sampling_ratio/mean": 1.0319452285766602,
      "sampling/importance_sampling_ratio/min": 1.1872944677925306e-14,
      "sampling/sampling_logp_difference/max": 20.445941925048828,
      "sampling/sampling_logp_difference/mean": 0.10557404160499573,
      "step": 535,
      "step_time": 12.509447111006011
    },
    {
      "clip_ratio/high_max": 0.013677205424755812,
      "clip_ratio/high_mean": 0.006838602712377906,
      "clip_ratio/low_mean": 0.0027225379599258304,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009561140672303736,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 668.0,
      "completions/max_terminated_length": 668.0,
      "completions/mean_length": 316.09375,
      "completions/mean_terminated_length": 315.63336181640625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5057194270193577,
      "epoch": 0.01072,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.429093837738037,
      "kl": 0.6763258827850223,
      "learning_rate": 5.339856507211485e-06,
      "loss": -0.1362,
      "num_tokens": 21686737.0,
      "reward": 0.6737499833106995,
      "reward_std": 0.6091959476470947,
      "rewards/rollout_reward_func/mean": 0.6737499833106995,
      "rewards/rollout_reward_func/std": 0.5937401652336121,
      "sampling/importance_sampling_ratio/max": 1.935964584350586,
      "sampling/importance_sampling_ratio/mean": 0.9614766836166382,
      "sampling/importance_sampling_ratio/min": 0.34971028566360474,
      "sampling/sampling_logp_difference/max": 0.7638809680938721,
      "sampling/sampling_logp_difference/mean": 0.034305572509765625,
      "step": 536,
      "step_time": 13.661436948008486
    },
    {
      "clip_ratio/high_max": 0.025520833441987634,
      "clip_ratio/high_mean": 0.014598651905544102,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014598651905544102,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 549.0,
      "completions/max_terminated_length": 549.0,
      "completions/mean_length": 192.40625,
      "completions/mean_terminated_length": 204.1666717529297,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4112771665677428,
      "epoch": 0.01074,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3286798000335693,
      "kl": 0.6288926070556045,
      "learning_rate": 5.326650142159479e-06,
      "loss": 0.0043,
      "num_tokens": 21724238.0,
      "reward": 0.7893749475479126,
      "reward_std": 0.521113395690918,
      "rewards/rollout_reward_func/mean": 0.7893749475479126,
      "rewards/rollout_reward_func/std": 0.519366443157196,
      "sampling/importance_sampling_ratio/max": 1.4456661939620972,
      "sampling/importance_sampling_ratio/mean": 0.9445327520370483,
      "sampling/importance_sampling_ratio/min": 0.45636531710624695,
      "sampling/sampling_logp_difference/max": 0.5710592269897461,
      "sampling/sampling_logp_difference/mean": 0.033012524247169495,
      "step": 537,
      "step_time": 11.704711798971402
    },
    {
      "clip_ratio/high_max": 0.0067471591755747795,
      "clip_ratio/high_mean": 0.0033735795877873898,
      "clip_ratio/low_mean": 0.005744485417380929,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009118065005168319,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 635.0,
      "completions/mean_length": 292.65625,
      "completions/mean_terminated_length": 284.1481628417969,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4987793527543545,
      "epoch": 0.01076,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5568597316741943,
      "kl": 0.3521883515641093,
      "learning_rate": 5.313455956716286e-06,
      "loss": -0.0066,
      "num_tokens": 21767305.0,
      "reward": 0.7806249856948853,
      "reward_std": 0.484897255897522,
      "rewards/rollout_reward_func/mean": 0.7806249856948853,
      "rewards/rollout_reward_func/std": 0.5010564923286438,
      "sampling/importance_sampling_ratio/max": 1.8223276138305664,
      "sampling/importance_sampling_ratio/mean": 1.10109543800354,
      "sampling/importance_sampling_ratio/min": 2.298567897975577e-16,
      "sampling/sampling_logp_difference/max": 22.660022735595703,
      "sampling/sampling_logp_difference/mean": 0.1078522801399231,
      "step": 538,
      "step_time": 12.918095130036818
    },
    {
      "clip_ratio/high_max": 0.030810041818767786,
      "clip_ratio/high_mean": 0.016707104165107012,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017683666665107012,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 662.0,
      "completions/max_terminated_length": 662.0,
      "completions/mean_length": 240.75,
      "completions/mean_terminated_length": 248.13792419433594,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.422562999650836,
      "epoch": 0.01078,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.738013505935669,
      "kl": 0.5410390952602029,
      "learning_rate": 5.3002741249221305e-06,
      "loss": -0.0868,
      "num_tokens": 21807295.0,
      "reward": 0.8321875333786011,
      "reward_std": 0.4747999608516693,
      "rewards/rollout_reward_func/mean": 0.8321875333786011,
      "rewards/rollout_reward_func/std": 0.4788476228713989,
      "sampling/importance_sampling_ratio/max": 1.4982728958129883,
      "sampling/importance_sampling_ratio/mean": 0.8919687867164612,
      "sampling/importance_sampling_ratio/min": 0.2680237293243408,
      "sampling/sampling_logp_difference/max": 0.7218585014343262,
      "sampling/sampling_logp_difference/mean": 0.04080859571695328,
      "step": 539,
      "step_time": 12.52587721402233
    },
    {
      "clip_ratio/high_max": 0.01320028048940003,
      "clip_ratio/high_mean": 0.006600140244700015,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011064426158554852,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 594.0,
      "completions/max_terminated_length": 594.0,
      "completions/mean_length": 211.9375,
      "completions/mean_terminated_length": 183.03448486328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.41989011131227016,
      "epoch": 0.0108,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8951796293258667,
      "kl": 0.6914418591186404,
      "learning_rate": 5.287104820654283e-06,
      "loss": -0.1988,
      "num_tokens": 21847152.0,
      "reward": 0.7990624904632568,
      "reward_std": 0.5482646226882935,
      "rewards/rollout_reward_func/mean": 0.7990624904632568,
      "rewards/rollout_reward_func/std": 0.5620719790458679,
      "sampling/importance_sampling_ratio/max": 1.7144241333007812,
      "sampling/importance_sampling_ratio/mean": 1.0708017349243164,
      "sampling/importance_sampling_ratio/min": 0.4277907609939575,
      "sampling/sampling_logp_difference/max": 0.615147590637207,
      "sampling/sampling_logp_difference/mean": 0.026716360822319984,
      "step": 540,
      "step_time": 11.844395743974019
    },
    {
      "clip_ratio/high_max": 0.016889881109818816,
      "clip_ratio/high_mean": 0.008444940554909408,
      "clip_ratio/low_mean": 0.005440848413854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01388578920159489,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 210.0625,
      "completions/mean_terminated_length": 201.36668395996094,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.42914543114602566,
      "epoch": 0.01082,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5894700288772583,
      "kl": 0.603661204688251,
      "learning_rate": 5.273948217624771e-06,
      "loss": 0.1073,
      "num_tokens": 21886026.0,
      "reward": 0.5413126945495605,
      "reward_std": 0.7044020295143127,
      "rewards/rollout_reward_func/mean": 0.5413126945495605,
      "rewards/rollout_reward_func/std": 0.7287976741790771,
      "sampling/importance_sampling_ratio/max": 1.255617380142212,
      "sampling/importance_sampling_ratio/mean": 0.9438135027885437,
      "sampling/importance_sampling_ratio/min": 0.4303102195262909,
      "sampling/sampling_logp_difference/max": 0.7150566577911377,
      "sampling/sampling_logp_difference/mean": 0.03581108897924423,
      "step": 541,
      "step_time": 12.520529625980998
    },
    {
      "clip_ratio/high_max": 0.005681818351149559,
      "clip_ratio/high_mean": 0.0028409091755747795,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0028409091755747795,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1063.0,
      "completions/max_terminated_length": 1063.0,
      "completions/mean_length": 276.28125,
      "completions/mean_terminated_length": 284.0714416503906,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.45139086432754993,
      "epoch": 0.01084,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8649607300758362,
      "kl": 0.49347532726824284,
      "learning_rate": 5.260804489378083e-06,
      "loss": -0.1221,
      "num_tokens": 21927452.0,
      "reward": 0.9000023603439331,
      "reward_std": 0.2895262837409973,
      "rewards/rollout_reward_func/mean": 0.9000023603439331,
      "rewards/rollout_reward_func/std": 0.4311012625694275,
      "sampling/importance_sampling_ratio/max": 1.5454261302947998,
      "sampling/importance_sampling_ratio/mean": 0.9489367008209229,
      "sampling/importance_sampling_ratio/min": 0.4008004367351532,
      "sampling/sampling_logp_difference/max": 0.8386783599853516,
      "sampling/sampling_logp_difference/mean": 0.035385504364967346,
      "step": 542,
      "step_time": 14.179755928969826
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0035978618543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0035978618543595076,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 1479.0,
      "completions/max_terminated_length": 1479.0,
      "completions/mean_length": 259.03125,
      "completions/mean_terminated_length": 263.6551818847656,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.42786249052733183,
      "epoch": 0.01086,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.078787088394165,
      "kl": 0.3581635784357786,
      "learning_rate": 5.2476738092888805e-06,
      "loss": 0.0231,
      "num_tokens": 21968487.0,
      "reward": 0.8665624856948853,
      "reward_std": 0.3243831396102905,
      "rewards/rollout_reward_func/mean": 0.8665624856948853,
      "rewards/rollout_reward_func/std": 0.3691979944705963,
      "sampling/importance_sampling_ratio/max": 1.8220747709274292,
      "sampling/importance_sampling_ratio/mean": 1.0170044898986816,
      "sampling/importance_sampling_ratio/min": 0.5708980560302734,
      "sampling/sampling_logp_difference/max": 0.7150988578796387,
      "sampling/sampling_logp_difference/mean": 0.03289464861154556,
      "step": 543,
      "step_time": 15.630667757999618
    },
    {
      "clip_ratio/high_max": 0.008297748630866408,
      "clip_ratio/high_mean": 0.004148874315433204,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004148874315433204,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 621.0,
      "completions/mean_length": 201.5625,
      "completions/mean_terminated_length": 164.03448486328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4679191457107663,
      "epoch": 0.01088,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2736637592315674,
      "kl": 0.45636992529034615,
      "learning_rate": 5.234556350559705e-06,
      "loss": -0.0065,
      "num_tokens": 22006459.0,
      "reward": 0.8284374475479126,
      "reward_std": 0.4662981629371643,
      "rewards/rollout_reward_func/mean": 0.8284374475479126,
      "rewards/rollout_reward_func/std": 0.4687740206718445,
      "sampling/importance_sampling_ratio/max": 2.692702531814575,
      "sampling/importance_sampling_ratio/mean": 0.9972357749938965,
      "sampling/importance_sampling_ratio/min": 8.46975945023587e-06,
      "sampling/sampling_logp_difference/max": 12.220892906188965,
      "sampling/sampling_logp_difference/mean": 0.060325659811496735,
      "step": 544,
      "step_time": 12.959015961998375
    },
    {
      "clip_ratio/high_max": 0.027157738571986556,
      "clip_ratio/high_mean": 0.013578869285993278,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013578869285993278,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 652.0,
      "completions/max_terminated_length": 652.0,
      "completions/mean_length": 249.90625,
      "completions/mean_terminated_length": 228.2413787841797,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.45631692465394735,
      "epoch": 0.0109,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4791350364685059,
      "kl": 0.39192727394402027,
      "learning_rate": 5.221452286218712e-06,
      "loss": -0.0316,
      "num_tokens": 22047000.0,
      "reward": 0.8378124833106995,
      "reward_std": 0.39111876487731934,
      "rewards/rollout_reward_func/mean": 0.8378124833106995,
      "rewards/rollout_reward_func/std": 0.4656298756599426,
      "sampling/importance_sampling_ratio/max": 1.8382691144943237,
      "sampling/importance_sampling_ratio/mean": 1.0273642539978027,
      "sampling/importance_sampling_ratio/min": 0.4745880365371704,
      "sampling/sampling_logp_difference/max": 0.586555004119873,
      "sampling/sampling_logp_difference/mean": 0.031348817050457,
      "step": 545,
      "step_time": 12.104616887052543
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0014880952658131719,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002464657765813172,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 604.0,
      "completions/mean_length": 176.40625,
      "completions/mean_terminated_length": 176.46429443359375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4537811651825905,
      "epoch": 0.01092,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.403264045715332,
      "kl": 0.35857151076197624,
      "learning_rate": 5.2083617891173625e-06,
      "loss": -0.1882,
      "num_tokens": 22082455.0,
      "reward": 0.8887499570846558,
      "reward_std": 0.14325907826423645,
      "rewards/rollout_reward_func/mean": 0.8887499570846558,
      "rewards/rollout_reward_func/std": 0.326345831155777,
      "sampling/importance_sampling_ratio/max": 1.49006986618042,
      "sampling/importance_sampling_ratio/mean": 1.0339235067367554,
      "sampling/importance_sampling_ratio/min": 2.031460013540709e-07,
      "sampling/sampling_logp_difference/max": 15.365191459655762,
      "sampling/sampling_logp_difference/mean": 0.06749515980482101,
      "step": 546,
      "step_time": 11.784862459040596
    },
    {
      "clip_ratio/high_max": 0.014850950799882412,
      "clip_ratio/high_mean": 0.007425475399941206,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007425475399941206,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 631.0,
      "completions/max_terminated_length": 631.0,
      "completions/mean_length": 284.6875,
      "completions/mean_terminated_length": 269.3214416503906,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.44166275672614574,
      "epoch": 0.01094,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.148522138595581,
      "kl": 0.701440048404038,
      "learning_rate": 5.195285031928168e-06,
      "loss": -0.0638,
      "num_tokens": 22125735.0,
      "reward": 0.8036612272262573,
      "reward_std": 0.49051448702812195,
      "rewards/rollout_reward_func/mean": 0.8036612272262573,
      "rewards/rollout_reward_func/std": 0.5036250352859497,
      "sampling/importance_sampling_ratio/max": 1.851218581199646,
      "sampling/importance_sampling_ratio/mean": 1.0104362964630127,
      "sampling/importance_sampling_ratio/min": 1.245663383997453e-06,
      "sampling/sampling_logp_difference/max": 5.66368293762207,
      "sampling/sampling_logp_difference/mean": 0.06021910905838013,
      "step": 547,
      "step_time": 13.220535709988326
    },
    {
      "clip_ratio/high_max": 0.0027173913549631834,
      "clip_ratio/high_mean": 0.0013586956774815917,
      "clip_ratio/low_mean": 0.0011160714784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002474767155945301,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 662.0,
      "completions/max_terminated_length": 662.0,
      "completions/mean_length": 274.625,
      "completions/mean_terminated_length": 271.3793029785156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5006561819463968,
      "epoch": 0.01096,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2761081457138062,
      "kl": 0.3115444639697671,
      "learning_rate": 5.182222187142388e-06,
      "loss": -0.081,
      "num_tokens": 22167977.0,
      "reward": 0.7087500095367432,
      "reward_std": 0.5844846963882446,
      "rewards/rollout_reward_func/mean": 0.7087500095367432,
      "rewards/rollout_reward_func/std": 0.5905259847640991,
      "sampling/importance_sampling_ratio/max": 1.5661287307739258,
      "sampling/importance_sampling_ratio/mean": 0.9922705888748169,
      "sampling/importance_sampling_ratio/min": 2.1678733461150124e-11,
      "sampling/sampling_logp_difference/max": 24.28125,
      "sampling/sampling_logp_difference/mean": 0.08185560256242752,
      "step": 548,
      "step_time": 12.268068632954964
    },
    {
      "clip_ratio/high_max": 0.0021551724057644606,
      "clip_ratio/high_mean": 0.0010775862028822303,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005541872116737068,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 192.5625,
      "completions/mean_terminated_length": 192.00001525878906,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.39624481461942196,
      "epoch": 0.01098,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.873187780380249,
      "kl": 0.46067607775330544,
      "learning_rate": 5.169173427067784e-06,
      "loss": -0.1243,
      "num_tokens": 22207282.0,
      "reward": 0.7093750238418579,
      "reward_std": 0.5591639280319214,
      "rewards/rollout_reward_func/mean": 0.7093750238418579,
      "rewards/rollout_reward_func/std": 0.5922778844833374,
      "sampling/importance_sampling_ratio/max": 1.6657975912094116,
      "sampling/importance_sampling_ratio/mean": 0.9778857231140137,
      "sampling/importance_sampling_ratio/min": 7.668278090011454e-19,
      "sampling/sampling_logp_difference/max": 26.9356632232666,
      "sampling/sampling_logp_difference/mean": 0.1571497768163681,
      "step": 549,
      "step_time": 11.949058559985133
    },
    {
      "clip_ratio/high_max": 0.04244791716337204,
      "clip_ratio/high_mean": 0.02122395858168602,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.025688244495540857,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 642.0,
      "completions/max_terminated_length": 634.0,
      "completions/mean_length": 217.71875,
      "completions/mean_terminated_length": 197.3928680419922,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4265681905671954,
      "epoch": 0.011,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8346673250198364,
      "kl": 0.556644769385457,
      "learning_rate": 5.156138923826317e-06,
      "loss": -0.0799,
      "num_tokens": 22246331.0,
      "reward": 0.6334375143051147,
      "reward_std": 0.5531155467033386,
      "rewards/rollout_reward_func/mean": 0.6334375143051147,
      "rewards/rollout_reward_func/std": 0.5606943368911743,
      "sampling/importance_sampling_ratio/max": 1.6815377473831177,
      "sampling/importance_sampling_ratio/mean": 0.9235894680023193,
      "sampling/importance_sampling_ratio/min": 2.372169802811186e-07,
      "sampling/sampling_logp_difference/max": 14.715633392333984,
      "sampling/sampling_logp_difference/mean": 0.11063949018716812,
      "step": 550,
      "step_time": 12.793493361983565
    },
    {
      "clip_ratio/high_max": 0.004464285913854837,
      "clip_ratio/high_mean": 0.0022321429569274187,
      "clip_ratio/low_mean": 0.0034722222480922937,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0057043652050197124,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 676.0,
      "completions/max_terminated_length": 606.0,
      "completions/mean_length": 185.15625,
      "completions/mean_terminated_length": 179.89654541015625,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.437892010435462,
      "epoch": 0.01102,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9080556631088257,
      "kl": 0.6456604264676571,
      "learning_rate": 5.143118849351898e-06,
      "loss": -0.0217,
      "num_tokens": 22283798.0,
      "reward": 0.8318749666213989,
      "reward_std": 0.3369309902191162,
      "rewards/rollout_reward_func/mean": 0.8318749666213989,
      "rewards/rollout_reward_func/std": 0.39204540848731995,
      "sampling/importance_sampling_ratio/max": 1.445454716682434,
      "sampling/importance_sampling_ratio/mean": 1.0168943405151367,
      "sampling/importance_sampling_ratio/min": 0.5337761640548706,
      "sampling/sampling_logp_difference/max": 0.5704841613769531,
      "sampling/sampling_logp_difference/mean": 0.02876581996679306,
      "step": 551,
      "step_time": 11.837383669961127
    },
    {
      "clip_ratio/high_max": 0.02941706799902022,
      "clip_ratio/high_mean": 0.015865941415540874,
      "clip_ratio/low_mean": 0.0013020833721384406,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017168024787679315,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 285.375,
      "completions/mean_terminated_length": 269.1071472167969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.48088682629168034,
      "epoch": 0.01104,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0133975744247437,
      "kl": 0.7187111172825098,
      "learning_rate": 5.1301133753881115e-06,
      "loss": -0.1727,
      "num_tokens": 22327065.0,
      "reward": 0.4212788939476013,
      "reward_std": 0.6871435046195984,
      "rewards/rollout_reward_func/mean": 0.4212788939476013,
      "rewards/rollout_reward_func/std": 0.7801230549812317,
      "sampling/importance_sampling_ratio/max": 1.4889198541641235,
      "sampling/importance_sampling_ratio/mean": 0.7751458287239075,
      "sampling/importance_sampling_ratio/min": 4.5218619822906865e-20,
      "sampling/sampling_logp_difference/max": 25.73434066772461,
      "sampling/sampling_logp_difference/mean": 0.19790098071098328,
      "step": 552,
      "step_time": 13.067028710021987
    },
    {
      "clip_ratio/high_max": 0.01992165483534336,
      "clip_ratio/high_mean": 0.00996082741767168,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00996082741767168,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 613.0,
      "completions/max_terminated_length": 613.0,
      "completions/mean_length": 183.78125,
      "completions/mean_terminated_length": 172.32257080078125,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.4457569532096386,
      "epoch": 0.01106,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1389700174331665,
      "kl": 0.7886724704876542,
      "learning_rate": 5.1171226734859505e-06,
      "loss": -0.0598,
      "num_tokens": 22364576.0,
      "reward": 0.7034375071525574,
      "reward_std": 0.6505909562110901,
      "rewards/rollout_reward_func/mean": 0.7034375071525574,
      "rewards/rollout_reward_func/std": 0.6569067239761353,
      "sampling/importance_sampling_ratio/max": 1.3339334726333618,
      "sampling/importance_sampling_ratio/mean": 0.9311773180961609,
      "sampling/importance_sampling_ratio/min": 6.605790256465594e-16,
      "sampling/sampling_logp_difference/max": 22.187170028686523,
      "sampling/sampling_logp_difference/mean": 0.11929961293935776,
      "step": 553,
      "step_time": 12.345524416014086
    },
    {
      "clip_ratio/high_max": 0.006221064832061529,
      "clip_ratio/high_mean": 0.0031105324160307646,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031105324160307646,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 632.0,
      "completions/max_terminated_length": 632.0,
      "completions/mean_length": 243.6875,
      "completions/mean_terminated_length": 246.89654541015625,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.45107242744416,
      "epoch": 0.01108,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9605954885482788,
      "kl": 0.8080345243215561,
      "learning_rate": 5.1041469150015535e-06,
      "loss": -0.095,
      "num_tokens": 22405394.0,
      "reward": 0.5709398984909058,
      "reward_std": 0.6981655955314636,
      "rewards/rollout_reward_func/mean": 0.5709398984909058,
      "rewards/rollout_reward_func/std": 0.691785991191864,
      "sampling/importance_sampling_ratio/max": 1.6430386304855347,
      "sampling/importance_sampling_ratio/mean": 0.9065662026405334,
      "sampling/importance_sampling_ratio/min": 2.4753045760837246e-32,
      "sampling/sampling_logp_difference/max": 26.827817916870117,
      "sampling/sampling_logp_difference/mean": 0.4135681390762329,
      "step": 554,
      "step_time": 12.119166766002309
    },
    {
      "clip_ratio/high_max": 0.0052083334885537624,
      "clip_ratio/high_mean": 0.0026041667442768812,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010416666744276881,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 1553.0,
      "completions/max_terminated_length": 1553.0,
      "completions/mean_length": 368.21875,
      "completions/mean_terminated_length": 372.3103332519531,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.45522618293762207,
      "epoch": 0.0111,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.632109522819519,
      "kl": 0.5718337446451187,
      "learning_rate": 5.091186271093949e-06,
      "loss": -0.1769,
      "num_tokens": 22451284.0,
      "reward": 0.9291029572486877,
      "reward_std": 0.26778966188430786,
      "rewards/rollout_reward_func/mean": 0.9291029572486877,
      "rewards/rollout_reward_func/std": 0.2920631766319275,
      "sampling/importance_sampling_ratio/max": 1.5537443161010742,
      "sampling/importance_sampling_ratio/mean": 0.8868870735168457,
      "sampling/importance_sampling_ratio/min": 5.375343503644103e-10,
      "sampling/sampling_logp_difference/max": 21.223098754882812,
      "sampling/sampling_logp_difference/mean": 0.13412512838840485,
      "step": 555,
      "step_time": 16.375944246014114
    },
    {
      "clip_ratio/high_max": 0.02362971263937652,
      "clip_ratio/high_mean": 0.01181485631968826,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01376798131968826,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 919.0,
      "completions/max_terminated_length": 919.0,
      "completions/mean_length": 229.59375,
      "completions/mean_terminated_length": 228.6785888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4550043698400259,
      "epoch": 0.01112,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.091576337814331,
      "kl": 0.38264618068933487,
      "learning_rate": 5.078240912722787e-06,
      "loss": -0.0,
      "num_tokens": 22491759.0,
      "reward": 0.590624988079071,
      "reward_std": 0.6796932220458984,
      "rewards/rollout_reward_func/mean": 0.590624988079071,
      "rewards/rollout_reward_func/std": 0.6598677039146423,
      "sampling/importance_sampling_ratio/max": 1.6010940074920654,
      "sampling/importance_sampling_ratio/mean": 0.9326369762420654,
      "sampling/importance_sampling_ratio/min": 2.5784488146114727e-25,
      "sampling/sampling_logp_difference/max": 27.29619598388672,
      "sampling/sampling_logp_difference/mean": 0.3343364894390106,
      "step": 556,
      "step_time": 13.6572209320002
    },
    {
      "clip_ratio/high_max": 0.0026041667442768812,
      "clip_ratio/high_mean": 0.0013020833721384406,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0013020833721384406,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 616.0,
      "completions/max_terminated_length": 585.0,
      "completions/mean_length": 237.40625,
      "completions/mean_terminated_length": 206.3199920654297,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4719028500840068,
      "epoch": 0.01114,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9429855346679688,
      "kl": 0.40678382851183414,
      "learning_rate": 5.065311010646101e-06,
      "loss": -0.1192,
      "num_tokens": 22532566.0,
      "reward": 0.7424999475479126,
      "reward_std": 0.5700705051422119,
      "rewards/rollout_reward_func/mean": 0.7424999475479126,
      "rewards/rollout_reward_func/std": 0.5746864080429077,
      "sampling/importance_sampling_ratio/max": 1.8110880851745605,
      "sampling/importance_sampling_ratio/mean": 1.0156662464141846,
      "sampling/importance_sampling_ratio/min": 8.428692258005735e-22,
      "sampling/sampling_logp_difference/max": 21.11191749572754,
      "sampling/sampling_logp_difference/mean": 0.14722473919391632,
      "step": 557,
      "step_time": 12.484881538970512
    },
    {
      "clip_ratio/high_max": 0.0029761905316263437,
      "clip_ratio/high_mean": 0.0014880952658131719,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003441220265813172,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 608.0,
      "completions/mean_length": 195.3125,
      "completions/mean_terminated_length": 187.75001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.44209033250808716,
      "epoch": 0.01116,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6044453382492065,
      "kl": 0.7040567584335804,
      "learning_rate": 5.052396735418037e-06,
      "loss": -0.2497,
      "num_tokens": 22570567.0,
      "reward": 0.7253124713897705,
      "reward_std": 0.5386699438095093,
      "rewards/rollout_reward_func/mean": 0.7253124713897705,
      "rewards/rollout_reward_func/std": 0.6038370728492737,
      "sampling/importance_sampling_ratio/max": 2.6955604553222656,
      "sampling/importance_sampling_ratio/mean": 1.0137910842895508,
      "sampling/importance_sampling_ratio/min": 0.30329179763793945,
      "sampling/sampling_logp_difference/max": 0.7655019760131836,
      "sampling/sampling_logp_difference/mean": 0.03850593417882919,
      "step": 558,
      "step_time": 12.386094821020379
    },
    {
      "clip_ratio/high_max": 0.007925724843516946,
      "clip_ratio/high_mean": 0.003962862421758473,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003962862421758473,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 604.0,
      "completions/max_terminated_length": 604.0,
      "completions/mean_length": 190.125,
      "completions/mean_terminated_length": 199.37930297851562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.48926649428904057,
      "epoch": 0.01118,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7452479004859924,
      "kl": 0.6226112460717559,
      "learning_rate": 5.039498257386617e-06,
      "loss": -0.0696,
      "num_tokens": 22608385.0,
      "reward": 0.7975000143051147,
      "reward_std": 0.4099368453025818,
      "rewards/rollout_reward_func/mean": 0.7975000143051147,
      "rewards/rollout_reward_func/std": 0.4966663122177124,
      "sampling/importance_sampling_ratio/max": 1.445473551750183,
      "sampling/importance_sampling_ratio/mean": 1.0169107913970947,
      "sampling/importance_sampling_ratio/min": 0.5518186688423157,
      "sampling/sampling_logp_difference/max": 0.5976829528808594,
      "sampling/sampling_logp_difference/mean": 0.028638195246458054,
      "step": 559,
      "step_time": 12.18429534502502
    },
    {
      "clip_ratio/high_max": 0.012849967461079359,
      "clip_ratio/high_mean": 0.0064249837305396795,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00902915047481656,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 299.90625,
      "completions/mean_terminated_length": 288.80645751953125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4678484760224819,
      "epoch": 0.0112,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2417089939117432,
      "kl": 0.4549908824265003,
      "learning_rate": 5.026615746691483e-06,
      "loss": -0.0896,
      "num_tokens": 22651616.0,
      "reward": 0.6746875047683716,
      "reward_std": 0.6253899931907654,
      "rewards/rollout_reward_func/mean": 0.6746875047683716,
      "rewards/rollout_reward_func/std": 0.625315248966217,
      "sampling/importance_sampling_ratio/max": 2.0449676513671875,
      "sampling/importance_sampling_ratio/mean": 0.8846181035041809,
      "sampling/importance_sampling_ratio/min": 1.5855238189033116e-07,
      "sampling/sampling_logp_difference/max": 15.593544006347656,
      "sampling/sampling_logp_difference/mean": 0.12160789966583252,
      "step": 560,
      "step_time": 12.713411261007423
    },
    {
      "clip_ratio/high_max": 0.01830357196740806,
      "clip_ratio/high_mean": 0.00915178598370403,
      "clip_ratio/low_mean": 0.004160839947871864,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013312625931575894,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 686.0,
      "completions/max_terminated_length": 686.0,
      "completions/mean_length": 228.71875,
      "completions/mean_terminated_length": 242.36668395996094,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.43786777649074793,
      "epoch": 0.01122,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.213416576385498,
      "kl": 0.5118887280113995,
      "learning_rate": 5.013749373261662e-06,
      "loss": 0.1757,
      "num_tokens": 22691518.0,
      "reward": 0.8365625143051147,
      "reward_std": 0.4657324552536011,
      "rewards/rollout_reward_func/mean": 0.8365625143051147,
      "rewards/rollout_reward_func/std": 0.4678751230239868,
      "sampling/importance_sampling_ratio/max": 1.3606666326522827,
      "sampling/importance_sampling_ratio/mean": 0.9360005855560303,
      "sampling/importance_sampling_ratio/min": 8.419747708957631e-17,
      "sampling/sampling_logp_difference/max": 22.088953018188477,
      "sampling/sampling_logp_difference/mean": 0.12571591138839722,
      "step": 561,
      "step_time": 12.988932786975056
    },
    {
      "clip_ratio/high_max": 0.004557291744276881,
      "clip_ratio/high_mean": 0.0022786458721384406,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022786458721384406,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 604.0,
      "completions/max_terminated_length": 604.0,
      "completions/mean_length": 193.25,
      "completions/mean_terminated_length": 190.0,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.49063601717352867,
      "epoch": 0.01124,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8668628931045532,
      "kl": 1.3128959825262427,
      "learning_rate": 5.0008993068133165e-06,
      "loss": -0.1763,
      "num_tokens": 22729560.0,
      "reward": 0.7956249713897705,
      "reward_std": 0.4054357707500458,
      "rewards/rollout_reward_func/mean": 0.7956249713897705,
      "rewards/rollout_reward_func/std": 0.5052430629730225,
      "sampling/importance_sampling_ratio/max": 1.5206319093704224,
      "sampling/importance_sampling_ratio/mean": 0.9563442468643188,
      "sampling/importance_sampling_ratio/min": 6.381362524343463e-16,
      "sampling/sampling_logp_difference/max": 22.375757217407227,
      "sampling/sampling_logp_difference/mean": 0.1280556619167328,
      "step": 562,
      "step_time": 11.842119861001265
    },
    {
      "clip_ratio/high_max": 0.04325980460271239,
      "clip_ratio/high_mean": 0.021629902301356196,
      "clip_ratio/low_mean": 0.0015625000232830644,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02319240232463926,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 831.0,
      "completions/max_terminated_length": 831.0,
      "completions/mean_length": 226.25,
      "completions/mean_terminated_length": 248.0,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.4992430806159973,
      "epoch": 0.01126,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3774313926696777,
      "kl": 0.40545625053346157,
      "learning_rate": 4.98806571684751e-06,
      "loss": -0.0037,
      "num_tokens": 22769083.0,
      "reward": 0.6356250047683716,
      "reward_std": 0.5259274244308472,
      "rewards/rollout_reward_func/mean": 0.6356250047683716,
      "rewards/rollout_reward_func/std": 0.5549421906471252,
      "sampling/importance_sampling_ratio/max": 1.4973163604736328,
      "sampling/importance_sampling_ratio/mean": 0.9479868412017822,
      "sampling/importance_sampling_ratio/min": 3.491028993662937e-11,
      "sampling/sampling_logp_difference/max": 9.657005310058594,
      "sampling/sampling_logp_difference/mean": 0.08435870707035065,
      "step": 563,
      "step_time": 12.571862681041239
    },
    {
      "clip_ratio/high_max": 0.0029761905316263437,
      "clip_ratio/high_mean": 0.0014880952658131719,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005952381179668009,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 676.0,
      "completions/max_terminated_length": 676.0,
      "completions/mean_length": 221.0,
      "completions/mean_terminated_length": 221.0,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.3647966468706727,
      "epoch": 0.01128,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8338630199432373,
      "kl": 1.0868151793256402,
      "learning_rate": 4.975248772647969e-06,
      "loss": -0.1017,
      "num_tokens": 22808017.0,
      "reward": 0.6988295316696167,
      "reward_std": 0.572288453578949,
      "rewards/rollout_reward_func/mean": 0.6988295316696167,
      "rewards/rollout_reward_func/std": 0.6850135326385498,
      "sampling/importance_sampling_ratio/max": 1.6937516927719116,
      "sampling/importance_sampling_ratio/mean": 1.0099139213562012,
      "sampling/importance_sampling_ratio/min": 0.2852031886577606,
      "sampling/sampling_logp_difference/max": 1.086594581604004,
      "sampling/sampling_logp_difference/mean": 0.03207094222307205,
      "step": 564,
      "step_time": 12.962563643988688
    },
    {
      "clip_ratio/high_max": 0.018342391354963183,
      "clip_ratio/high_mean": 0.009171195677481592,
      "clip_ratio/low_mean": 0.0012019231216982007,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010373118799179792,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 274.46875,
      "completions/mean_terminated_length": 273.433349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.48547530360519886,
      "epoch": 0.0113,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.060304045677185,
      "kl": 1.92185181658715,
      "learning_rate": 4.962448643278852e-06,
      "loss": -0.0955,
      "num_tokens": 22849209.0,
      "reward": 0.7030362486839294,
      "reward_std": 0.6376970410346985,
      "rewards/rollout_reward_func/mean": 0.7030362486839294,
      "rewards/rollout_reward_func/std": 0.6683452725410461,
      "sampling/importance_sampling_ratio/max": 1.2251311540603638,
      "sampling/importance_sampling_ratio/mean": 0.9337022304534912,
      "sampling/importance_sampling_ratio/min": 0.22371573746204376,
      "sampling/sampling_logp_difference/max": 0.9712810516357422,
      "sampling/sampling_logp_difference/mean": 0.0358453094959259,
      "step": 565,
      "step_time": 12.296088417962892
    },
    {
      "clip_ratio/high_max": 0.03144221263937652,
      "clip_ratio/high_mean": 0.01572110631968826,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01572110631968826,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 643.0,
      "completions/max_terminated_length": 643.0,
      "completions/mean_length": 311.1875,
      "completions/mean_terminated_length": 301.5714416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5439902981743217,
      "epoch": 0.01132,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.122413158416748,
      "kl": 0.41923298966139555,
      "learning_rate": 4.949665497582519e-06,
      "loss": 0.0843,
      "num_tokens": 22892283.0,
      "reward": 0.6435001492500305,
      "reward_std": 0.5564229488372803,
      "rewards/rollout_reward_func/mean": 0.6435001492500305,
      "rewards/rollout_reward_func/std": 0.5587396621704102,
      "sampling/importance_sampling_ratio/max": 1.656641960144043,
      "sampling/importance_sampling_ratio/mean": 0.8469980955123901,
      "sampling/importance_sampling_ratio/min": 1.4901285906413477e-27,
      "sampling/sampling_logp_difference/max": 25.1180419921875,
      "sampling/sampling_logp_difference/mean": 0.2286677062511444,
      "step": 566,
      "step_time": 12.489347288981662
    },
    {
      "clip_ratio/high_max": 0.0024038462433964014,
      "clip_ratio/high_mean": 0.0012019231216982007,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0168269231216982,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 605.0,
      "completions/max_terminated_length": 605.0,
      "completions/mean_length": 207.25,
      "completions/mean_terminated_length": 201.96429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4396178098395467,
      "epoch": 0.01134,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.453291416168213,
      "kl": 0.7807416748255491,
      "learning_rate": 4.936899504177303e-06,
      "loss": -0.0899,
      "num_tokens": 22931683.0,
      "reward": 0.6096874475479126,
      "reward_std": 0.6538482904434204,
      "rewards/rollout_reward_func/mean": 0.6096874475479126,
      "rewards/rollout_reward_func/std": 0.6762776374816895,
      "sampling/importance_sampling_ratio/max": 1.4345821142196655,
      "sampling/importance_sampling_ratio/mean": 0.998599112033844,
      "sampling/importance_sampling_ratio/min": 0.40212082862854004,
      "sampling/sampling_logp_difference/max": 0.7228329181671143,
      "sampling/sampling_logp_difference/mean": 0.031007403507828712,
      "step": 567,
      "step_time": 13.075688081036787
    },
    {
      "clip_ratio/high_max": 0.015340909361839294,
      "clip_ratio/high_mean": 0.007670454680919647,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023295454680919647,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 1544.0,
      "completions/max_terminated_length": 1544.0,
      "completions/mean_length": 317.78125,
      "completions/mean_terminated_length": 316.9310302734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5128846745938063,
      "epoch": 0.01136,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2182339429855347,
      "kl": 0.8245621081441641,
      "learning_rate": 4.9241508314552856e-06,
      "loss": -0.1726,
      "num_tokens": 22974368.0,
      "reward": 0.7208892703056335,
      "reward_std": 0.523176372051239,
      "rewards/rollout_reward_func/mean": 0.7208892703056335,
      "rewards/rollout_reward_func/std": 0.6538658142089844,
      "sampling/importance_sampling_ratio/max": 1.5133076906204224,
      "sampling/importance_sampling_ratio/mean": 0.8551524877548218,
      "sampling/importance_sampling_ratio/min": 1.2904602719743252e-08,
      "sampling/sampling_logp_difference/max": 17.718971252441406,
      "sampling/sampling_logp_difference/mean": 0.12146541476249695,
      "step": 568,
      "step_time": 15.90430634196673
    },
    {
      "clip_ratio/high_max": 0.0034722222480922937,
      "clip_ratio/high_mean": 0.0017361111240461469,
      "clip_ratio/low_mean": 0.0012499999720603228,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0029861110961064696,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 639.0,
      "completions/max_terminated_length": 639.0,
      "completions/mean_length": 328.09375,
      "completions/mean_terminated_length": 330.7930908203125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4855363853275776,
      "epoch": 0.01138,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6118673086166382,
      "kl": 0.5354102328419685,
      "learning_rate": 4.911419647580079e-06,
      "loss": 0.0492,
      "num_tokens": 23018434.0,
      "reward": 0.6462499499320984,
      "reward_std": 0.47079551219940186,
      "rewards/rollout_reward_func/mean": 0.6462499499320984,
      "rewards/rollout_reward_func/std": 0.5666069984436035,
      "sampling/importance_sampling_ratio/max": 1.508586049079895,
      "sampling/importance_sampling_ratio/mean": 0.987311840057373,
      "sampling/importance_sampling_ratio/min": 0.39746055006980896,
      "sampling/sampling_logp_difference/max": 0.9917230606079102,
      "sampling/sampling_logp_difference/mean": 0.0401836633682251,
      "step": 569,
      "step_time": 12.16187352995621
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0017361111240461469,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0017361111240461469,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 205.78125,
      "completions/mean_terminated_length": 194.9666748046875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.47867369558662176,
      "epoch": 0.0114,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0157300233840942,
      "kl": 0.35788136441260576,
      "learning_rate": 4.898706120484606e-06,
      "loss": 0.0266,
      "num_tokens": 23056585.0,
      "reward": 0.676562488079071,
      "reward_std": 0.3995835483074188,
      "rewards/rollout_reward_func/mean": 0.676562488079071,
      "rewards/rollout_reward_func/std": 0.5375968217849731,
      "sampling/importance_sampling_ratio/max": 1.7040514945983887,
      "sampling/importance_sampling_ratio/mean": 1.0714490413665771,
      "sampling/importance_sampling_ratio/min": 3.3882850242256768e-18,
      "sampling/sampling_logp_difference/max": 27.75775146484375,
      "sampling/sampling_logp_difference/mean": 0.12943720817565918,
      "step": 570,
      "step_time": 12.83347177298856
    },
    {
      "clip_ratio/high_max": 0.014756944496184587,
      "clip_ratio/high_mean": 0.007378472248092294,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007378472248092294,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 661.0,
      "completions/max_terminated_length": 661.0,
      "completions/mean_length": 263.84375,
      "completions/mean_terminated_length": 259.3103332519531,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5308666191995144,
      "epoch": 0.01142,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1941760778427124,
      "kl": 0.5954748345538974,
      "learning_rate": 4.886010417868881e-06,
      "loss": -0.0044,
      "num_tokens": 23098252.0,
      "reward": 0.7281249761581421,
      "reward_std": 0.599697470664978,
      "rewards/rollout_reward_func/mean": 0.7281249761581421,
      "rewards/rollout_reward_func/std": 0.5897098183631897,
      "sampling/importance_sampling_ratio/max": 1.6511660814285278,
      "sampling/importance_sampling_ratio/mean": 0.8896673917770386,
      "sampling/importance_sampling_ratio/min": 3.2797161988102024e-20,
      "sampling/sampling_logp_difference/max": 27.101016998291016,
      "sampling/sampling_logp_difference/mean": 0.13018520176410675,
      "step": 571,
      "step_time": 12.28416679694783
    },
    {
      "clip_ratio/high_max": 0.005922378972172737,
      "clip_ratio/high_mean": 0.0029611894860863686,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0029611894860863686,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 201.9375,
      "completions/mean_terminated_length": 200.42308044433594,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4817638285458088,
      "epoch": 0.01144,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1123281717300415,
      "kl": 0.3762042345479131,
      "learning_rate": 4.873332707197804e-06,
      "loss": -0.2095,
      "num_tokens": 23136077.0,
      "reward": 0.7368749976158142,
      "reward_std": 0.49937647581100464,
      "rewards/rollout_reward_func/mean": 0.7368749976158142,
      "rewards/rollout_reward_func/std": 0.5235143899917603,
      "sampling/importance_sampling_ratio/max": 1.9201865196228027,
      "sampling/importance_sampling_ratio/mean": 1.026036024093628,
      "sampling/importance_sampling_ratio/min": 0.3542875051498413,
      "sampling/sampling_logp_difference/max": 0.5824918746948242,
      "sampling/sampling_logp_difference/mean": 0.03739418834447861,
      "step": 572,
      "step_time": 12.588539360003779
    },
    {
      "clip_ratio/high_max": 0.03694358188658953,
      "clip_ratio/high_mean": 0.018471790943294764,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.026284290943294764,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 600.0,
      "completions/max_terminated_length": 600.0,
      "completions/mean_length": 193.875,
      "completions/mean_terminated_length": 198.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5095905726775527,
      "epoch": 0.01146,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.410250186920166,
      "kl": 0.9881428815424442,
      "learning_rate": 4.860673155698953e-06,
      "loss": -0.1642,
      "num_tokens": 23173685.0,
      "reward": 0.7916396856307983,
      "reward_std": 0.47971951961517334,
      "rewards/rollout_reward_func/mean": 0.7916396856307983,
      "rewards/rollout_reward_func/std": 0.5011290311813354,
      "sampling/importance_sampling_ratio/max": 1.5435665845870972,
      "sampling/importance_sampling_ratio/mean": 0.9946547746658325,
      "sampling/importance_sampling_ratio/min": 9.926219463014923e-18,
      "sampling/sampling_logp_difference/max": 21.30708122253418,
      "sampling/sampling_logp_difference/mean": 0.12291646003723145,
      "step": 573,
      "step_time": 12.537461411004188
    },
    {
      "clip_ratio/high_max": 0.012978558195754886,
      "clip_ratio/high_mean": 0.007791362353600562,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009744487353600562,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 916.0,
      "completions/max_terminated_length": 916.0,
      "completions/mean_length": 278.15625,
      "completions/mean_terminated_length": 325.66668701171875,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5230809543281794,
      "epoch": 0.01148,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.446498155593872,
      "kl": 1.003325373865664,
      "learning_rate": 4.848031930360365e-06,
      "loss": -0.1489,
      "num_tokens": 23215365.0,
      "reward": 0.7322646379470825,
      "reward_std": 0.5830914974212646,
      "rewards/rollout_reward_func/mean": 0.7322646379470825,
      "rewards/rollout_reward_func/std": 0.5910988450050354,
      "sampling/importance_sampling_ratio/max": 1.958389163017273,
      "sampling/importance_sampling_ratio/mean": 0.9864941835403442,
      "sampling/importance_sampling_ratio/min": 1.0018182106105228e-21,
      "sampling/sampling_logp_difference/max": 23.120134353637695,
      "sampling/sampling_logp_difference/mean": 0.12335969507694244,
      "step": 574,
      "step_time": 13.384581890990376
    },
    {
      "clip_ratio/high_max": 0.02257812488824129,
      "clip_ratio/high_mean": 0.011289062444120646,
      "clip_ratio/low_mean": 0.007697610300965607,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018986672745086253,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 631.0,
      "completions/max_terminated_length": 631.0,
      "completions/mean_length": 218.65625,
      "completions/mean_terminated_length": 238.6428680419922,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4417842496186495,
      "epoch": 0.0115,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2396820783615112,
      "kl": 0.5449198922142386,
      "learning_rate": 4.835409197928351e-06,
      "loss": -0.0616,
      "num_tokens": 23253622.0,
      "reward": 0.6343749761581421,
      "reward_std": 0.6481735706329346,
      "rewards/rollout_reward_func/mean": 0.6343749761581421,
      "rewards/rollout_reward_func/std": 0.668599545955658,
      "sampling/importance_sampling_ratio/max": 1.9186334609985352,
      "sampling/importance_sampling_ratio/mean": 1.0110137462615967,
      "sampling/importance_sampling_ratio/min": 0.43649277091026306,
      "sampling/sampling_logp_difference/max": 0.7728695869445801,
      "sampling/sampling_logp_difference/mean": 0.03314310312271118,
      "step": 575,
      "step_time": 12.5544504509744
    },
    {
      "clip_ratio/high_max": 0.017578125,
      "clip_ratio/high_mean": 0.0087890625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0087890625,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 690.0,
      "completions/mean_length": 238.59375,
      "completions/mean_terminated_length": 193.95834350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.611782617866993,
      "epoch": 0.01152,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.100648045539856,
      "kl": 0.435337845236063,
      "learning_rate": 4.822805124905282e-06,
      "loss": -0.1046,
      "num_tokens": 23293498.0,
      "reward": 0.5703125,
      "reward_std": 0.6774296164512634,
      "rewards/rollout_reward_func/mean": 0.5703125,
      "rewards/rollout_reward_func/std": 0.6657108068466187,
      "sampling/importance_sampling_ratio/max": 1.3310117721557617,
      "sampling/importance_sampling_ratio/mean": 0.9017680883407593,
      "sampling/importance_sampling_ratio/min": 0.39657747745513916,
      "sampling/sampling_logp_difference/max": 0.7653293609619141,
      "sampling/sampling_logp_difference/mean": 0.03592677786946297,
      "step": 576,
      "step_time": 13.01270858902717
    },
    {
      "clip_ratio/high_max": 0.0052083334885537624,
      "clip_ratio/high_mean": 0.0026041667442768812,
      "clip_ratio/low_mean": 0.0010416667209938169,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003645833465270698,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 630.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 244.59375,
      "completions/mean_terminated_length": 228.1428680419922,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.49631702061742544,
      "epoch": 0.01154,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2853920459747314,
      "kl": 0.38536819675937295,
      "learning_rate": 4.810219877547406e-06,
      "loss": 0.0814,
      "num_tokens": 23334076.0,
      "reward": 0.641915500164032,
      "reward_std": 0.545439600944519,
      "rewards/rollout_reward_func/mean": 0.641915500164032,
      "rewards/rollout_reward_func/std": 0.5602993965148926,
      "sampling/importance_sampling_ratio/max": 1.5453007221221924,
      "sampling/importance_sampling_ratio/mean": 1.0661706924438477,
      "sampling/importance_sampling_ratio/min": 0.6526913642883301,
      "sampling/sampling_logp_difference/max": 0.542391300201416,
      "sampling/sampling_logp_difference/mean": 0.028867874294519424,
      "step": 577,
      "step_time": 12.08595356902515
    },
    {
      "clip_ratio/high_max": 0.023809524718672037,
      "clip_ratio/high_mean": 0.011904762359336019,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011904762359336019,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 865.0,
      "completions/max_terminated_length": 865.0,
      "completions/mean_length": 330.3125,
      "completions/mean_terminated_length": 292.96295166015625,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.5334043465554714,
      "epoch": 0.01156,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2850067615509033,
      "kl": 0.4375659879297018,
      "learning_rate": 4.797653621862637e-06,
      "loss": 0.0225,
      "num_tokens": 23377998.0,
      "reward": 0.8046875,
      "reward_std": 0.3730553090572357,
      "rewards/rollout_reward_func/mean": 0.8046875,
      "rewards/rollout_reward_func/std": 0.4274076521396637,
      "sampling/importance_sampling_ratio/max": 2.0804765224456787,
      "sampling/importance_sampling_ratio/mean": 1.0167263746261597,
      "sampling/importance_sampling_ratio/min": 0.45917606353759766,
      "sampling/sampling_logp_difference/max": 0.575437068939209,
      "sampling/sampling_logp_difference/mean": 0.0379796102643013,
      "step": 578,
      "step_time": 13.796460322992061
    },
    {
      "clip_ratio/high_max": 0.02041903417557478,
      "clip_ratio/high_mean": 0.01020951708778739,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01020951708778739,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 923.0,
      "completions/max_terminated_length": 923.0,
      "completions/mean_length": 257.53125,
      "completions/mean_terminated_length": 246.38462829589844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4787189345806837,
      "epoch": 0.01158,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2079720497131348,
      "kl": 0.4017525874078274,
      "learning_rate": 4.785106523608388e-06,
      "loss": -0.1244,
      "num_tokens": 23419423.0,
      "reward": 0.9018749594688416,
      "reward_std": 0.2861764430999756,
      "rewards/rollout_reward_func/mean": 0.9018749594688416,
      "rewards/rollout_reward_func/std": 0.41492658853530884,
      "sampling/importance_sampling_ratio/max": 1.497599720954895,
      "sampling/importance_sampling_ratio/mean": 0.9793343544006348,
      "sampling/importance_sampling_ratio/min": 1.4895451571826196e-17,
      "sampling/sampling_logp_difference/max": 20.291057586669922,
      "sampling/sampling_logp_difference/mean": 0.10490830987691879,
      "step": 579,
      "step_time": 13.420827024034224
    },
    {
      "clip_ratio/high_max": 0.04320400138385594,
      "clip_ratio/high_mean": 0.02160200069192797,
      "clip_ratio/low_mean": 0.0011574074160307646,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022759408224374056,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 588.0,
      "completions/max_terminated_length": 588.0,
      "completions/mean_length": 253.6875,
      "completions/mean_terminated_length": 269.5517272949219,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5000816565006971,
      "epoch": 0.0116,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9771426916122437,
      "kl": 0.5924132270738482,
      "learning_rate": 4.7725787482893645e-06,
      "loss": -0.1155,
      "num_tokens": 23460738.0,
      "reward": 0.7263309955596924,
      "reward_std": 0.47287577390670776,
      "rewards/rollout_reward_func/mean": 0.7263309955596924,
      "rewards/rollout_reward_func/std": 0.7010914087295532,
      "sampling/importance_sampling_ratio/max": 1.8434946537017822,
      "sampling/importance_sampling_ratio/mean": 0.9944830536842346,
      "sampling/importance_sampling_ratio/min": 0.46007755398750305,
      "sampling/sampling_logp_difference/max": 0.5279152393341064,
      "sampling/sampling_logp_difference/mean": 0.03137335926294327,
      "step": 580,
      "step_time": 12.10200716998952
    },
    {
      "clip_ratio/high_max": 0.01168290339410305,
      "clip_ratio/high_mean": 0.007934085675515234,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007934085675515234,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 1337.0,
      "completions/max_terminated_length": 1337.0,
      "completions/mean_length": 294.28125,
      "completions/mean_terminated_length": 279.8095397949219,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5314036374911666,
      "epoch": 0.01162,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8609204888343811,
      "kl": 0.410955224186182,
      "learning_rate": 4.760070461155393e-06,
      "loss": -0.1593,
      "num_tokens": 23504419.0,
      "reward": 0.7700626850128174,
      "reward_std": 0.49464118480682373,
      "rewards/rollout_reward_func/mean": 0.7700626850128174,
      "rewards/rollout_reward_func/std": 0.5655350685119629,
      "sampling/importance_sampling_ratio/max": 1.2645224332809448,
      "sampling/importance_sampling_ratio/mean": 0.7581769227981567,
      "sampling/importance_sampling_ratio/min": 7.255956518917661e-38,
      "sampling/sampling_logp_difference/max": 24.582910537719727,
      "sampling/sampling_logp_difference/mean": 0.3067055344581604,
      "step": 581,
      "step_time": 16.367799945001025
    },
    {
      "clip_ratio/high_max": 0.009114583488553762,
      "clip_ratio/high_mean": 0.004557291744276881,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009021577658131719,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 658.0,
      "completions/mean_length": 264.1875,
      "completions/mean_terminated_length": 221.0416717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5148438923060894,
      "epoch": 0.01164,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5906381607055664,
      "kl": 1.5334338839165866,
      "learning_rate": 4.7475818271992335e-06,
      "loss": -0.2475,
      "num_tokens": 23546048.0,
      "reward": 0.6049999594688416,
      "reward_std": 0.7088608145713806,
      "rewards/rollout_reward_func/mean": 0.6049999594688416,
      "rewards/rollout_reward_func/std": 0.6971369385719299,
      "sampling/importance_sampling_ratio/max": 2.0602822303771973,
      "sampling/importance_sampling_ratio/mean": 0.9468488693237305,
      "sampling/importance_sampling_ratio/min": 0.37826353311538696,
      "sampling/sampling_logp_difference/max": 0.9637041091918945,
      "sampling/sampling_logp_difference/mean": 0.04052915424108505,
      "step": 582,
      "step_time": 12.72371259497595
    },
    {
      "clip_ratio/high_max": 0.007387907709926367,
      "clip_ratio/high_mean": 0.0036939538549631834,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004670516354963183,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 597.0,
      "completions/mean_length": 248.40625,
      "completions/mean_terminated_length": 237.375,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5475062821060419,
      "epoch": 0.01166,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5794111490249634,
      "kl": 0.41221610456705093,
      "learning_rate": 4.735113011154413e-06,
      "loss": -0.2129,
      "num_tokens": 23588261.0,
      "reward": 0.707812488079071,
      "reward_std": 0.5790907144546509,
      "rewards/rollout_reward_func/mean": 0.707812488079071,
      "rewards/rollout_reward_func/std": 0.6505022048950195,
      "sampling/importance_sampling_ratio/max": 1.613439917564392,
      "sampling/importance_sampling_ratio/mean": 1.0678341388702393,
      "sampling/importance_sampling_ratio/min": 0.4386431872844696,
      "sampling/sampling_logp_difference/max": 0.4623725414276123,
      "sampling/sampling_logp_difference/mean": 0.034633100032806396,
      "step": 583,
      "step_time": 12.539122750022216
    },
    {
      "clip_ratio/high_max": 0.011284722480922937,
      "clip_ratio/high_mean": 0.005642361240461469,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005642361240461469,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 612.0,
      "completions/max_terminated_length": 610.0,
      "completions/mean_length": 290.25,
      "completions/mean_terminated_length": 270.933349609375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.47552693262696266,
      "epoch": 0.01168,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2618556022644043,
      "kl": 0.35858024284243584,
      "learning_rate": 4.722664177493042e-06,
      "loss": 0.0796,
      "num_tokens": 23630931.0,
      "reward": 0.6856249570846558,
      "reward_std": 0.5325577855110168,
      "rewards/rollout_reward_func/mean": 0.6856249570846558,
      "rewards/rollout_reward_func/std": 0.5423540472984314,
      "sampling/importance_sampling_ratio/max": 1.4734028577804565,
      "sampling/importance_sampling_ratio/mean": 0.9571858048439026,
      "sampling/importance_sampling_ratio/min": 8.190583744551273e-23,
      "sampling/sampling_logp_difference/max": 17.39023208618164,
      "sampling/sampling_logp_difference/mean": 0.11553093791007996,
      "step": 584,
      "step_time": 13.370503034981084
    },
    {
      "clip_ratio/high_max": 0.009558446356095374,
      "clip_ratio/high_mean": 0.004779223178047687,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004779223178047687,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 999.0,
      "completions/max_terminated_length": 655.0,
      "completions/mean_length": 357.21875,
      "completions/mean_terminated_length": 307.79998779296875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5481118950992823,
      "epoch": 0.0117,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6877845525741577,
      "kl": 0.3435922171920538,
      "learning_rate": 4.710235490423655e-06,
      "loss": 0.1261,
      "num_tokens": 23676969.0,
      "reward": 0.6122280359268188,
      "reward_std": 0.6153496503829956,
      "rewards/rollout_reward_func/mean": 0.6122280359268188,
      "rewards/rollout_reward_func/std": 0.6288681030273438,
      "sampling/importance_sampling_ratio/max": 1.391730546951294,
      "sampling/importance_sampling_ratio/mean": 0.9433443546295166,
      "sampling/importance_sampling_ratio/min": 0.25118201971054077,
      "sampling/sampling_logp_difference/max": 0.7122683525085449,
      "sampling/sampling_logp_difference/mean": 0.03405687212944031,
      "step": 585,
      "step_time": 14.033344558003591
    },
    {
      "clip_ratio/high_max": 0.006510416744276881,
      "clip_ratio/high_mean": 0.0032552083721384406,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0032552083721384406,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 624.0,
      "completions/max_terminated_length": 624.0,
      "completions/mean_length": 169.5625,
      "completions/mean_terminated_length": 170.73333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4711027918383479,
      "epoch": 0.01172,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9897130131721497,
      "kl": 0.5721769416704774,
      "learning_rate": 4.697827113889034e-06,
      "loss": -0.1311,
      "num_tokens": 23713645.0,
      "reward": 0.7853530049324036,
      "reward_std": 0.4934535026550293,
      "rewards/rollout_reward_func/mean": 0.7853530049324036,
      "rewards/rollout_reward_func/std": 0.5612180233001709,
      "sampling/importance_sampling_ratio/max": 1.4472675323486328,
      "sampling/importance_sampling_ratio/mean": 0.9783871173858643,
      "sampling/importance_sampling_ratio/min": 0.5263667702674866,
      "sampling/sampling_logp_difference/max": 0.6254949569702148,
      "sampling/sampling_logp_difference/mean": 0.032563358545303345,
      "step": 586,
      "step_time": 11.742199116008123
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.004185267956927419,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008091517956927419,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 669.0,
      "completions/mean_length": 242.46875,
      "completions/mean_terminated_length": 243.69566345214844,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.49269686453044415,
      "epoch": 0.01174,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4880720376968384,
      "kl": 0.6468362547457218,
      "learning_rate": 4.685439211564055e-06,
      "loss": -0.0484,
      "num_tokens": 23754097.0,
      "reward": 0.7038126587867737,
      "reward_std": 0.5689084529876709,
      "rewards/rollout_reward_func/mean": 0.7038126587867737,
      "rewards/rollout_reward_func/std": 0.6954486966133118,
      "sampling/importance_sampling_ratio/max": 1.3677771091461182,
      "sampling/importance_sampling_ratio/mean": 1.0040777921676636,
      "sampling/importance_sampling_ratio/min": 0.5107840895652771,
      "sampling/sampling_logp_difference/max": 0.8525757789611816,
      "sampling/sampling_logp_difference/mean": 0.030538132414221764,
      "step": 587,
      "step_time": 13.602674904002924
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.00390625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00390625,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 1150.0,
      "completions/max_terminated_length": 1150.0,
      "completions/mean_length": 254.25,
      "completions/mean_terminated_length": 254.9199981689453,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5072314664721489,
      "epoch": 0.01176,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4694900512695312,
      "kl": 0.6428990019485354,
      "learning_rate": 4.67307194685352e-06,
      "loss": 0.1523,
      "num_tokens": 23794601.0,
      "reward": 0.6305362582206726,
      "reward_std": 0.5700230598449707,
      "rewards/rollout_reward_func/mean": 0.6305362582206726,
      "rewards/rollout_reward_func/std": 0.5629454851150513,
      "sampling/importance_sampling_ratio/max": 1.5708506107330322,
      "sampling/importance_sampling_ratio/mean": 0.9595957398414612,
      "sampling/importance_sampling_ratio/min": 5.306787940094688e-16,
      "sampling/sampling_logp_difference/max": 20.408809661865234,
      "sampling/sampling_logp_difference/mean": 0.09438114613294601,
      "step": 588,
      "step_time": 13.994737653978518
    },
    {
      "clip_ratio/high_max": 0.005989583441987634,
      "clip_ratio/high_mean": 0.002994791720993817,
      "clip_ratio/low_mean": 0.004448784748092294,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007443576585501432,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 664.0,
      "completions/max_terminated_length": 538.0,
      "completions/mean_length": 261.28125,
      "completions/mean_terminated_length": 212.7391357421875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5075719496235251,
      "epoch": 0.01178,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.561881422996521,
      "kl": 0.5752838626503944,
      "learning_rate": 4.660725482890016e-06,
      "loss": -0.0639,
      "num_tokens": 23835899.0,
      "reward": 0.6685001850128174,
      "reward_std": 0.6457993984222412,
      "rewards/rollout_reward_func/mean": 0.6685001850128174,
      "rewards/rollout_reward_func/std": 0.6642776727676392,
      "sampling/importance_sampling_ratio/max": 2.2930383682250977,
      "sampling/importance_sampling_ratio/mean": 1.0696430206298828,
      "sampling/importance_sampling_ratio/min": 0.3419818580150604,
      "sampling/sampling_logp_difference/max": 1.1267776489257812,
      "sampling/sampling_logp_difference/mean": 0.034041374921798706,
      "step": 589,
      "step_time": 12.91900224298297
    },
    {
      "clip_ratio/high_max": 0.015217391541227698,
      "clip_ratio/high_mean": 0.007608695770613849,
      "clip_ratio/low_mean": 0.0013020833721384406,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00891077914275229,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 585.0,
      "completions/mean_length": 241.53125,
      "completions/mean_terminated_length": 214.96551513671875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4770885678008199,
      "epoch": 0.0118,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9396638870239258,
      "kl": 0.3333112644031644,
      "learning_rate": 4.648399982531745e-06,
      "loss": -0.1176,
      "num_tokens": 23877095.0,
      "reward": 0.7981249690055847,
      "reward_std": 0.49856090545654297,
      "rewards/rollout_reward_func/mean": 0.7981249690055847,
      "rewards/rollout_reward_func/std": 0.49311017990112305,
      "sampling/importance_sampling_ratio/max": 1.4042408466339111,
      "sampling/importance_sampling_ratio/mean": 0.903442919254303,
      "sampling/importance_sampling_ratio/min": 1.5374330912979275e-16,
      "sampling/sampling_logp_difference/max": 20.773456573486328,
      "sampling/sampling_logp_difference/mean": 0.10374966263771057,
      "step": 590,
      "step_time": 12.141054243940744
    },
    {
      "clip_ratio/high_max": 0.005121237598359585,
      "clip_ratio/high_mean": 0.0025606187991797924,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0025606187991797924,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 253.5,
      "completions/mean_terminated_length": 231.07408142089844,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.48878402076661587,
      "epoch": 0.01182,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.088071346282959,
      "kl": 0.749884151853621,
      "learning_rate": 4.636095608360393e-06,
      "loss": 0.1164,
      "num_tokens": 23917198.0,
      "reward": 0.7668749690055847,
      "reward_std": 0.5042884349822998,
      "rewards/rollout_reward_func/mean": 0.7668749690055847,
      "rewards/rollout_reward_func/std": 0.5101450681686401,
      "sampling/importance_sampling_ratio/max": 1.849294662475586,
      "sampling/importance_sampling_ratio/mean": 0.9407432675361633,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 19.352571487426758,
      "sampling/sampling_logp_difference/mean": 0.13819926977157593,
      "step": 591,
      "step_time": 12.370433339005103
    },
    {
      "clip_ratio/high_max": 0.004999999888241291,
      "clip_ratio/high_mean": 0.0024999999441206455,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0024999999441206455,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 656.0,
      "completions/mean_length": 249.5625,
      "completions/mean_terminated_length": 235.9310302734375,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.4483937472105026,
      "epoch": 0.01184,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0488860607147217,
      "kl": 0.4737253710627556,
      "learning_rate": 4.623812522678968e-06,
      "loss": -0.0288,
      "num_tokens": 23958552.0,
      "reward": 0.5817862749099731,
      "reward_std": 0.6414059400558472,
      "rewards/rollout_reward_func/mean": 0.5817862749099731,
      "rewards/rollout_reward_func/std": 0.6351925730705261,
      "sampling/importance_sampling_ratio/max": 1.7288302183151245,
      "sampling/importance_sampling_ratio/mean": 1.0266342163085938,
      "sampling/importance_sampling_ratio/min": 0.34776657819747925,
      "sampling/sampling_logp_difference/max": 1.0662338733673096,
      "sampling/sampling_logp_difference/mean": 0.027868855744600296,
      "step": 592,
      "step_time": 14.336390933996881
    },
    {
      "clip_ratio/high_max": 0.007181677268818021,
      "clip_ratio/high_mean": 0.0035908386344090104,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0035908386344090104,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 658.0,
      "completions/max_terminated_length": 643.0,
      "completions/mean_length": 214.5,
      "completions/mean_terminated_length": 172.40740966796875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.3792462684214115,
      "epoch": 0.01186,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.301377296447754,
      "kl": 0.5119309965521097,
      "learning_rate": 4.611550887509677e-06,
      "loss": 0.0286,
      "num_tokens": 23997781.0,
      "reward": 0.7099999785423279,
      "reward_std": 0.538846492767334,
      "rewards/rollout_reward_func/mean": 0.7099999785423279,
      "rewards/rollout_reward_func/std": 0.5419439077377319,
      "sampling/importance_sampling_ratio/max": 1.2978843450546265,
      "sampling/importance_sampling_ratio/mean": 0.9726368188858032,
      "sampling/importance_sampling_ratio/min": 0.63768470287323,
      "sampling/sampling_logp_difference/max": 0.38654327392578125,
      "sampling/sampling_logp_difference/mean": 0.028411291539669037,
      "step": 593,
      "step_time": 12.123377447031089
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.0166015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0166015625,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 806.0,
      "completions/max_terminated_length": 806.0,
      "completions/mean_length": 338.9375,
      "completions/mean_terminated_length": 337.4814758300781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4657520707696676,
      "epoch": 0.01188,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0972208976745605,
      "kl": 1.0834138775244355,
      "learning_rate": 4.599310864591776e-06,
      "loss": -0.1489,
      "num_tokens": 24042801.0,
      "reward": 0.5769155025482178,
      "reward_std": 0.5163182020187378,
      "rewards/rollout_reward_func/mean": 0.5769155025482178,
      "rewards/rollout_reward_func/std": 0.6926434636116028,
      "sampling/importance_sampling_ratio/max": 1.5602924823760986,
      "sampling/importance_sampling_ratio/mean": 0.9305998682975769,
      "sampling/importance_sampling_ratio/min": 0.3669448792934418,
      "sampling/sampling_logp_difference/max": 1.0212006568908691,
      "sampling/sampling_logp_difference/mean": 0.03504693880677223,
      "step": 594,
      "step_time": 13.033043653995264
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 631.0,
      "completions/max_terminated_length": 631.0,
      "completions/mean_length": 220.125,
      "completions/mean_terminated_length": 233.1538543701172,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5112750642001629,
      "epoch": 0.0119,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1250807046890259,
      "kl": 0.5907795103266835,
      "learning_rate": 4.587092615379442e-06,
      "loss": -0.0713,
      "num_tokens": 24082155.0,
      "reward": 0.6640625,
      "reward_std": 0.6468400955200195,
      "rewards/rollout_reward_func/mean": 0.6640625,
      "rewards/rollout_reward_func/std": 0.6583942174911499,
      "sampling/importance_sampling_ratio/max": 1.6517540216445923,
      "sampling/importance_sampling_ratio/mean": 0.798384964466095,
      "sampling/importance_sampling_ratio/min": 4.1022662832868387e-14,
      "sampling/sampling_logp_difference/max": 20.701217651367188,
      "sampling/sampling_logp_difference/mean": 0.2032867819070816,
      "step": 595,
      "step_time": 13.673729802045273
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0009765625,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 662.0,
      "completions/mean_length": 261.8125,
      "completions/mean_terminated_length": 228.9285888671875,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4659054297953844,
      "epoch": 0.01192,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7804173231124878,
      "kl": 0.2939950665459037,
      "learning_rate": 4.574896301039641e-06,
      "loss": -0.1139,
      "num_tokens": 24123910.0,
      "reward": 0.8031250238418579,
      "reward_std": 0.3604245185852051,
      "rewards/rollout_reward_func/mean": 0.8031250238418579,
      "rewards/rollout_reward_func/std": 0.42790135741233826,
      "sampling/importance_sampling_ratio/max": 1.5798676013946533,
      "sampling/importance_sampling_ratio/mean": 1.100314736366272,
      "sampling/importance_sampling_ratio/min": 0.5477648973464966,
      "sampling/sampling_logp_difference/max": 0.3518061637878418,
      "sampling/sampling_logp_difference/mean": 0.02581821382045746,
      "step": 596,
      "step_time": 12.44678912202653
    },
    {
      "clip_ratio/high_max": 0.014880952890962362,
      "clip_ratio/high_mean": 0.007440476445481181,
      "clip_ratio/low_mean": 0.0029296875,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010370163945481181,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 316.03125,
      "completions/mean_terminated_length": 246.91305541992188,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5347176911309361,
      "epoch": 0.01194,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5651365518569946,
      "kl": 1.7945058392360806,
      "learning_rate": 4.562722082450004e-06,
      "loss": 0.1091,
      "num_tokens": 24167244.0,
      "reward": 0.5368750095367432,
      "reward_std": 0.6370484232902527,
      "rewards/rollout_reward_func/mean": 0.5368750095367432,
      "rewards/rollout_reward_func/std": 0.6188461184501648,
      "sampling/importance_sampling_ratio/max": 1.9375836849212646,
      "sampling/importance_sampling_ratio/mean": 0.9089797735214233,
      "sampling/importance_sampling_ratio/min": 2.3948705640376075e-30,
      "sampling/sampling_logp_difference/max": 20.007007598876953,
      "sampling/sampling_logp_difference/mean": 0.19101208448410034,
      "step": 597,
      "step_time": 12.792089616981684
    },
    {
      "clip_ratio/high_max": 0.01695312513038516,
      "clip_ratio/high_mean": 0.010121299186721444,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010121299186721444,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 586.0,
      "completions/mean_length": 308.6875,
      "completions/mean_terminated_length": 238.57142639160156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5870181005448103,
      "epoch": 0.01196,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6400727033615112,
      "kl": 0.33893001545220613,
      "learning_rate": 4.550570120196702e-06,
      "loss": -0.1109,
      "num_tokens": 24210086.0,
      "reward": 0.794374942779541,
      "reward_std": 0.33993908762931824,
      "rewards/rollout_reward_func/mean": 0.794374942779541,
      "rewards/rollout_reward_func/std": 0.414424866437912,
      "sampling/importance_sampling_ratio/max": 1.8637405633926392,
      "sampling/importance_sampling_ratio/mean": 1.038881540298462,
      "sampling/importance_sampling_ratio/min": 0.46637746691703796,
      "sampling/sampling_logp_difference/max": 0.6545865535736084,
      "sampling/sampling_logp_difference/mean": 0.036206573247909546,
      "step": 598,
      "step_time": 14.411617526988266
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006417410913854837,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 637.0,
      "completions/max_terminated_length": 629.0,
      "completions/mean_length": 187.75,
      "completions/mean_terminated_length": 158.5,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4411351718008518,
      "epoch": 0.01198,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9291195273399353,
      "kl": 0.5702699683606625,
      "learning_rate": 4.538440574572337e-06,
      "loss": -0.0641,
      "num_tokens": 24247038.0,
      "reward": 0.7618749737739563,
      "reward_std": 0.4179762005805969,
      "rewards/rollout_reward_func/mean": 0.7618749737739563,
      "rewards/rollout_reward_func/std": 0.5162047147750854,
      "sampling/importance_sampling_ratio/max": 1.2757153511047363,
      "sampling/importance_sampling_ratio/mean": 0.9560667276382446,
      "sampling/importance_sampling_ratio/min": 0.23490436375141144,
      "sampling/sampling_logp_difference/max": 1.4831762313842773,
      "sampling/sampling_logp_difference/mean": 0.031881626695394516,
      "step": 599,
      "step_time": 12.146461223979713
    },
    {
      "clip_ratio/high_max": 0.009114583488553762,
      "clip_ratio/high_mean": 0.004557291744276881,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004557291744276881,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 216.71875,
      "completions/mean_terminated_length": 217.42308044433594,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47868878953158855,
      "epoch": 0.012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.041338324546814,
      "kl": 0.35407929494977,
      "learning_rate": 4.5263336055738115e-06,
      "loss": 0.0663,
      "num_tokens": 24285769.0,
      "reward": 0.8015624284744263,
      "reward_std": 0.49061256647109985,
      "rewards/rollout_reward_func/mean": 0.8015624284744263,
      "rewards/rollout_reward_func/std": 0.48670125007629395,
      "sampling/importance_sampling_ratio/max": 1.3420685529708862,
      "sampling/importance_sampling_ratio/mean": 1.0059726238250732,
      "sampling/importance_sampling_ratio/min": 0.44919025897979736,
      "sampling/sampling_logp_difference/max": 0.5207095146179199,
      "sampling/sampling_logp_difference/mean": 0.027863113209605217,
      "step": 600,
      "step_time": 12.130938991031144
    },
    {
      "clip_ratio/high_max": 0.005439814878627658,
      "clip_ratio/high_mean": 0.002719907439313829,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002719907439313829,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 650.0,
      "completions/max_terminated_length": 650.0,
      "completions/mean_length": 276.625,
      "completions/mean_terminated_length": 272.5714416503906,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5129362838342786,
      "epoch": 0.01202,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2038869857788086,
      "kl": 0.3029701244086027,
      "learning_rate": 4.5142493729002316e-06,
      "loss": 0.0089,
      "num_tokens": 24328738.0,
      "reward": 0.8862500190734863,
      "reward_std": 0.24982352554798126,
      "rewards/rollout_reward_func/mean": 0.8862500190734863,
      "rewards/rollout_reward_func/std": 0.3320876955986023,
      "sampling/importance_sampling_ratio/max": 1.5337449312210083,
      "sampling/importance_sampling_ratio/mean": 0.9855589270591736,
      "sampling/importance_sampling_ratio/min": 8.787532557941776e-19,
      "sampling/sampling_logp_difference/max": 20.7332706451416,
      "sampling/sampling_logp_difference/mean": 0.11136622726917267,
      "step": 601,
      "step_time": 13.238141715002712
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0036057692486792803,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036057692486792803,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1013.0,
      "completions/max_terminated_length": 1013.0,
      "completions/mean_length": 250.875,
      "completions/mean_terminated_length": 225.07693481445312,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5140356365591288,
      "epoch": 0.01204,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2196321487426758,
      "kl": 0.5651195198297501,
      "learning_rate": 4.502188035950794e-06,
      "loss": -0.1212,
      "num_tokens": 24369858.0,
      "reward": 0.572603166103363,
      "reward_std": 0.5301845073699951,
      "rewards/rollout_reward_func/mean": 0.572603166103363,
      "rewards/rollout_reward_func/std": 0.7175350189208984,
      "sampling/importance_sampling_ratio/max": 1.8951491117477417,
      "sampling/importance_sampling_ratio/mean": 1.047990083694458,
      "sampling/importance_sampling_ratio/min": 0.23667146265506744,
      "sampling/sampling_logp_difference/max": 0.7166733741760254,
      "sampling/sampling_logp_difference/mean": 0.03531460836529732,
      "step": 602,
      "step_time": 13.449950510985218
    },
    {
      "clip_ratio/high_max": 0.0036764706019312143,
      "clip_ratio/high_mean": 0.0018382353009656072,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0018382353009656072,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 599.0,
      "completions/mean_length": 226.4375,
      "completions/mean_terminated_length": 209.20834350585938,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4254326242953539,
      "epoch": 0.01206,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8922042846679688,
      "kl": 0.5409058164805174,
      "learning_rate": 4.490149753822689e-06,
      "loss": -0.1567,
      "num_tokens": 24410439.0,
      "reward": 0.5490624904632568,
      "reward_std": 0.8221411108970642,
      "rewards/rollout_reward_func/mean": 0.5490624904632568,
      "rewards/rollout_reward_func/std": 0.805598795413971,
      "sampling/importance_sampling_ratio/max": 1.4434229135513306,
      "sampling/importance_sampling_ratio/mean": 0.8253496885299683,
      "sampling/importance_sampling_ratio/min": 3.327339700861265e-19,
      "sampling/sampling_logp_difference/max": 25.158748626708984,
      "sampling/sampling_logp_difference/mean": 0.1812780201435089,
      "step": 603,
      "step_time": 13.00854321802035
    },
    {
      "clip_ratio/high_max": 0.016161541221663356,
      "clip_ratio/high_mean": 0.008080770610831678,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008080770610831678,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 1022.0,
      "completions/max_terminated_length": 1022.0,
      "completions/mean_length": 254.34375,
      "completions/mean_terminated_length": 250.92593383789062,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4962821248918772,
      "epoch": 0.01208,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8947277665138245,
      "kl": 0.37859338242560625,
      "learning_rate": 4.478134685308997e-06,
      "loss": -0.094,
      "num_tokens": 24450797.0,
      "reward": 0.7028896808624268,
      "reward_std": 0.6120020151138306,
      "rewards/rollout_reward_func/mean": 0.7028896808624268,
      "rewards/rollout_reward_func/std": 0.6010019183158875,
      "sampling/importance_sampling_ratio/max": 1.5559180974960327,
      "sampling/importance_sampling_ratio/mean": 0.8942122459411621,
      "sampling/importance_sampling_ratio/min": 2.1428776975085384e-08,
      "sampling/sampling_logp_difference/max": 17.734525680541992,
      "sampling/sampling_logp_difference/mean": 0.11092380434274673,
      "step": 604,
      "step_time": 14.227965726036928
    },
    {
      "clip_ratio/high_max": 0.002016128972172737,
      "clip_ratio/high_mean": 0.0010080644860863686,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0019846269860863686,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 636.0,
      "completions/max_terminated_length": 633.0,
      "completions/mean_length": 247.96875,
      "completions/mean_terminated_length": 240.44444274902344,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.484774365555495,
      "epoch": 0.0121,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.310094952583313,
      "kl": 0.521658125333488,
      "learning_rate": 4.4661429888965894e-06,
      "loss": -0.0005,
      "num_tokens": 24490745.0,
      "reward": 0.6753124594688416,
      "reward_std": 0.5947327613830566,
      "rewards/rollout_reward_func/mean": 0.6753124594688416,
      "rewards/rollout_reward_func/std": 0.6196824312210083,
      "sampling/importance_sampling_ratio/max": 1.3163479566574097,
      "sampling/importance_sampling_ratio/mean": 0.9360994100570679,
      "sampling/importance_sampling_ratio/min": 2.2951858542358684e-16,
      "sampling/sampling_logp_difference/max": 21.295581817626953,
      "sampling/sampling_logp_difference/mean": 0.10999514162540436,
      "step": 605,
      "step_time": 12.479084341990529
    },
    {
      "clip_ratio/high_max": 0.005425347248092294,
      "clip_ratio/high_mean": 0.002712673624046147,
      "clip_ratio/low_mean": 0.0061848959885537624,
      "clip_ratio/low_min": 0.001953125,
      "clip_ratio/region_mean": 0.00889756961259991,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 687.0,
      "completions/max_terminated_length": 589.0,
      "completions/mean_length": 172.78125,
      "completions/mean_terminated_length": 140.83999633789062,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5032972027547657,
      "epoch": 0.01212,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0140671730041504,
      "kl": 0.5213170973584056,
      "learning_rate": 4.454174822764049e-06,
      "loss": -0.068,
      "num_tokens": 24526952.0,
      "reward": 0.7303124666213989,
      "reward_std": 0.5239158868789673,
      "rewards/rollout_reward_func/mean": 0.7303124666213989,
      "rewards/rollout_reward_func/std": 0.5353773236274719,
      "sampling/importance_sampling_ratio/max": 1.3624248504638672,
      "sampling/importance_sampling_ratio/mean": 0.9590317010879517,
      "sampling/importance_sampling_ratio/min": 3.7780167532991705e-18,
      "sampling/sampling_logp_difference/max": 21.51923179626465,
      "sampling/sampling_logp_difference/mean": 0.11403730511665344,
      "step": 606,
      "step_time": 13.084856135974405
    },
    {
      "clip_ratio/high_max": 0.008526545716449618,
      "clip_ratio/high_mean": 0.004263272858224809,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004263272858224809,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 638.0,
      "completions/max_terminated_length": 638.0,
      "completions/mean_length": 317.0,
      "completions/mean_terminated_length": 301.32000732421875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5587698351591825,
      "epoch": 0.01214,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.919272541999817,
      "kl": 0.2619572402909398,
      "learning_rate": 4.4422303447795816e-06,
      "loss": 0.2642,
      "num_tokens": 24571201.0,
      "reward": 0.7168749570846558,
      "reward_std": 0.4844686686992645,
      "rewards/rollout_reward_func/mean": 0.7168749570846558,
      "rewards/rollout_reward_func/std": 0.46531596779823303,
      "sampling/importance_sampling_ratio/max": 1.8047767877578735,
      "sampling/importance_sampling_ratio/mean": 1.012364387512207,
      "sampling/importance_sampling_ratio/min": 0.4672229588031769,
      "sampling/sampling_logp_difference/max": 0.581317663192749,
      "sampling/sampling_logp_difference/mean": 0.03232661634683609,
      "step": 607,
      "step_time": 12.663213037987589
    },
    {
      "clip_ratio/high_max": 0.027059659361839294,
      "clip_ratio/high_mean": 0.013529829680919647,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015368064981885254,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1049.0,
      "completions/max_terminated_length": 1049.0,
      "completions/mean_length": 283.28125,
      "completions/mean_terminated_length": 257.69232177734375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5063754350412637,
      "epoch": 0.01216,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.08811354637146,
      "kl": 0.4556442694738507,
      "learning_rate": 4.430309712498924e-06,
      "loss": 0.1524,
      "num_tokens": 24613116.0,
      "reward": 0.7446874380111694,
      "reward_std": 0.493059903383255,
      "rewards/rollout_reward_func/mean": 0.7446874380111694,
      "rewards/rollout_reward_func/std": 0.5184902548789978,
      "sampling/importance_sampling_ratio/max": 2.057450771331787,
      "sampling/importance_sampling_ratio/mean": 0.9815386533737183,
      "sampling/importance_sampling_ratio/min": 0.495927095413208,
      "sampling/sampling_logp_difference/max": 0.6081733703613281,
      "sampling/sampling_logp_difference/mean": 0.03660906106233597,
      "step": 608,
      "step_time": 13.574530947997118
    },
    {
      "clip_ratio/high_max": 0.003289473708719015,
      "clip_ratio/high_mean": 0.0016447368543595076,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0016447368543595076,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 620.0,
      "completions/max_terminated_length": 495.0,
      "completions/mean_length": 219.0,
      "completions/mean_terminated_length": 206.44444274902344,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47669930825941265,
      "epoch": 0.01218,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9776297807693481,
      "kl": 0.36781713366508484,
      "learning_rate": 4.418413083163273e-06,
      "loss": -0.1052,
      "num_tokens": 24651622.0,
      "reward": 0.8233487606048584,
      "reward_std": 0.37334197759628296,
      "rewards/rollout_reward_func/mean": 0.8233487606048584,
      "rewards/rollout_reward_func/std": 0.48039430379867554,
      "sampling/importance_sampling_ratio/max": 1.5303832292556763,
      "sampling/importance_sampling_ratio/mean": 0.9974859952926636,
      "sampling/importance_sampling_ratio/min": 0.3361875116825104,
      "sampling/sampling_logp_difference/max": 0.5528144836425781,
      "sampling/sampling_logp_difference/mean": 0.030918080359697342,
      "step": 609,
      "step_time": 12.765544333014986
    },
    {
      "clip_ratio/high_max": 0.010850694496184587,
      "clip_ratio/high_mean": 0.007070084102451801,
      "clip_ratio/low_mean": 0.0035978618543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010667945956811309,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 642.0,
      "completions/max_terminated_length": 642.0,
      "completions/mean_length": 300.59375,
      "completions/mean_terminated_length": 297.28570556640625,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5925496183335781,
      "epoch": 0.0122,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.071690559387207,
      "kl": 0.5149933509528637,
      "learning_rate": 4.406540613697213e-06,
      "loss": 0.055,
      "num_tokens": 24694965.0,
      "reward": 0.6356249451637268,
      "reward_std": 0.5039103627204895,
      "rewards/rollout_reward_func/mean": 0.6356249451637268,
      "rewards/rollout_reward_func/std": 0.6290542483329773,
      "sampling/importance_sampling_ratio/max": 1.8149250745773315,
      "sampling/importance_sampling_ratio/mean": 0.9562891125679016,
      "sampling/importance_sampling_ratio/min": 1.0913032595503308e-21,
      "sampling/sampling_logp_difference/max": 22.49399185180664,
      "sampling/sampling_logp_difference/mean": 0.1644473522901535,
      "step": 610,
      "step_time": 12.54947438002273
    },
    {
      "clip_ratio/high_max": 0.0041082974057644606,
      "clip_ratio/high_mean": 0.0020541487028822303,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0020541487028822303,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 656.0,
      "completions/mean_length": 192.8125,
      "completions/mean_terminated_length": 150.60870361328125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5401628725230694,
      "epoch": 0.01222,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3631367683410645,
      "kl": 0.28581874817609787,
      "learning_rate": 4.394692460706643e-06,
      "loss": -0.0496,
      "num_tokens": 24733000.0,
      "reward": 0.7965624928474426,
      "reward_std": 0.49419596791267395,
      "rewards/rollout_reward_func/mean": 0.7965624928474426,
      "rewards/rollout_reward_func/std": 0.48946717381477356,
      "sampling/importance_sampling_ratio/max": 1.5907542705535889,
      "sampling/importance_sampling_ratio/mean": 0.9889254570007324,
      "sampling/importance_sampling_ratio/min": 0.42623135447502136,
      "sampling/sampling_logp_difference/max": 0.6236093044281006,
      "sampling/sampling_logp_difference/mean": 0.03194160759449005,
      "step": 611,
      "step_time": 12.210145005999948
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 668.0,
      "completions/max_terminated_length": 668.0,
      "completions/mean_length": 255.65625,
      "completions/mean_terminated_length": 236.1851806640625,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5195886045694351,
      "epoch": 0.01224,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4613853693008423,
      "kl": 0.28919546445831656,
      "learning_rate": 4.382868780476709e-06,
      "loss": -0.0155,
      "num_tokens": 24774598.0,
      "reward": 0.7278124690055847,
      "reward_std": 0.5044220685958862,
      "rewards/rollout_reward_func/mean": 0.7278124690055847,
      "rewards/rollout_reward_func/std": 0.5123049020767212,
      "sampling/importance_sampling_ratio/max": 1.5949041843414307,
      "sampling/importance_sampling_ratio/mean": 1.055443286895752,
      "sampling/importance_sampling_ratio/min": 0.5011543035507202,
      "sampling/sampling_logp_difference/max": 0.6629366874694824,
      "sampling/sampling_logp_difference/mean": 0.03466970473527908,
      "step": 612,
      "step_time": 13.18522337100876
    },
    {
      "clip_ratio/high_max": 0.007378472248092294,
      "clip_ratio/high_mean": 0.003689236124046147,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004665798624046147,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 614.0,
      "completions/max_terminated_length": 614.0,
      "completions/mean_length": 232.53125,
      "completions/mean_terminated_length": 256.6000061035156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5126687865704298,
      "epoch": 0.01226,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2313454151153564,
      "kl": 0.3580290051177144,
      "learning_rate": 4.371069728969745e-06,
      "loss": 0.1027,
      "num_tokens": 24814366.0,
      "reward": 0.7602237462997437,
      "reward_std": 0.44191837310791016,
      "rewards/rollout_reward_func/mean": 0.7602237462997437,
      "rewards/rollout_reward_func/std": 0.5141225457191467,
      "sampling/importance_sampling_ratio/max": 1.426788568496704,
      "sampling/importance_sampling_ratio/mean": 0.9929186701774597,
      "sampling/importance_sampling_ratio/min": 3.6796431615779594e-20,
      "sampling/sampling_logp_difference/max": 22.877992630004883,
      "sampling/sampling_logp_difference/mean": 0.14049500226974487,
      "step": 613,
      "step_time": 12.472885302995564
    },
    {
      "clip_ratio/high_max": 0.03138764947652817,
      "clip_ratio/high_mean": 0.015693824738264084,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015693824738264084,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 856.0,
      "completions/max_terminated_length": 856.0,
      "completions/mean_length": 180.09375,
      "completions/mean_terminated_length": 171.86207580566406,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.42172240279614925,
      "epoch": 0.01228,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6557513475418091,
      "kl": 0.5671126465313137,
      "learning_rate": 4.359295461823221e-06,
      "loss": -0.1433,
      "num_tokens": 24851068.0,
      "reward": 0.6915624737739563,
      "reward_std": 0.6521115303039551,
      "rewards/rollout_reward_func/mean": 0.6915624737739563,
      "rewards/rollout_reward_func/std": 0.6561279892921448,
      "sampling/importance_sampling_ratio/max": 1.8099726438522339,
      "sampling/importance_sampling_ratio/mean": 0.9809868335723877,
      "sampling/importance_sampling_ratio/min": 0.35554826259613037,
      "sampling/sampling_logp_difference/max": 1.0812146663665771,
      "sampling/sampling_logp_difference/mean": 0.030588708817958832,
      "step": 614,
      "step_time": 12.54986305499915
    },
    {
      "clip_ratio/high_max": 0.033439066959545016,
      "clip_ratio/high_mean": 0.016719533479772508,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018557768780738115,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 692.0,
      "completions/mean_length": 284.53125,
      "completions/mean_terminated_length": 293.40740966796875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.529266950674355,
      "epoch": 0.0123,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.474965214729309,
      "kl": 0.33163464069366455,
      "learning_rate": 4.347546134347673e-06,
      "loss": 0.0988,
      "num_tokens": 24893493.0,
      "reward": 0.7437499761581421,
      "reward_std": 0.5083703994750977,
      "rewards/rollout_reward_func/mean": 0.7437499761581421,
      "rewards/rollout_reward_func/std": 0.5211014747619629,
      "sampling/importance_sampling_ratio/max": 1.5339381694793701,
      "sampling/importance_sampling_ratio/mean": 0.8085296154022217,
      "sampling/importance_sampling_ratio/min": 8.422738093665308e-24,
      "sampling/sampling_logp_difference/max": 23.843748092651367,
      "sampling/sampling_logp_difference/mean": 0.28735223412513733,
      "step": 615,
      "step_time": 13.351909952965798
    },
    {
      "clip_ratio/high_max": 0.03220078884623945,
      "clip_ratio/high_mean": 0.016100394423119724,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016100394423119724,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 815.0,
      "completions/max_terminated_length": 815.0,
      "completions/mean_length": 256.75,
      "completions/mean_terminated_length": 251.73077392578125,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.5922993626445532,
      "epoch": 0.01232,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6706255674362183,
      "kl": 0.8274971889331937,
      "learning_rate": 4.33582190152468e-06,
      "loss": -0.0434,
      "num_tokens": 24935163.0,
      "reward": 0.7374999523162842,
      "reward_std": 0.5774461627006531,
      "rewards/rollout_reward_func/mean": 0.7374999523162842,
      "rewards/rollout_reward_func/std": 0.5860639214515686,
      "sampling/importance_sampling_ratio/max": 1.93675696849823,
      "sampling/importance_sampling_ratio/mean": 0.9078227281570435,
      "sampling/importance_sampling_ratio/min": 1.9644709765069728e-25,
      "sampling/sampling_logp_difference/max": 23.772525787353516,
      "sampling/sampling_logp_difference/mean": 0.15665172040462494,
      "step": 616,
      "step_time": 13.306636761015397
    },
    {
      "clip_ratio/high_max": 0.007031250046566129,
      "clip_ratio/high_mean": 0.0035156250232830644,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0035156250232830644,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 1204.0,
      "completions/max_terminated_length": 1204.0,
      "completions/mean_length": 289.3125,
      "completions/mean_terminated_length": 253.0869598388672,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.580645352602005,
      "epoch": 0.01234,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8829461336135864,
      "kl": 0.2697654999792576,
      "learning_rate": 4.324122918004795e-06,
      "loss": 0.1463,
      "num_tokens": 24977037.0,
      "reward": 0.8571875095367432,
      "reward_std": 0.3864072561264038,
      "rewards/rollout_reward_func/mean": 0.8571875095367432,
      "rewards/rollout_reward_func/std": 0.44392117857933044,
      "sampling/importance_sampling_ratio/max": 2.1146817207336426,
      "sampling/importance_sampling_ratio/mean": 0.9598881602287292,
      "sampling/importance_sampling_ratio/min": 5.818956427816602e-09,
      "sampling/sampling_logp_difference/max": 18.78121566772461,
      "sampling/sampling_logp_difference/mean": 0.086983822286129,
      "step": 617,
      "step_time": 15.148699062003288
    },
    {
      "clip_ratio/high_max": 0.008091517956927419,
      "clip_ratio/high_mean": 0.004045758978463709,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005998883978463709,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 668.0,
      "completions/max_terminated_length": 668.0,
      "completions/mean_length": 232.96875,
      "completions/mean_terminated_length": 210.72000122070312,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5397284962236881,
      "epoch": 0.01236,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3432559967041016,
      "kl": 0.25959291215986013,
      "learning_rate": 4.3124493381055204e-06,
      "loss": -0.1897,
      "num_tokens": 25016627.0,
      "reward": 0.7090624570846558,
      "reward_std": 0.45333290100097656,
      "rewards/rollout_reward_func/mean": 0.7090624570846558,
      "rewards/rollout_reward_func/std": 0.5351716876029968,
      "sampling/importance_sampling_ratio/max": 1.4146591424942017,
      "sampling/importance_sampling_ratio/mean": 1.0146721601486206,
      "sampling/importance_sampling_ratio/min": 0.4431866407394409,
      "sampling/sampling_logp_difference/max": 0.5941176414489746,
      "sampling/sampling_logp_difference/mean": 0.03205258026719093,
      "step": 618,
      "step_time": 13.70723137902678
    },
    {
      "clip_ratio/high_max": 0.0024038462433964014,
      "clip_ratio/high_mean": 0.0012019231216982007,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0012019231216982007,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 682.0,
      "completions/max_terminated_length": 682.0,
      "completions/mean_length": 277.9375,
      "completions/mean_terminated_length": 252.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4880926664918661,
      "epoch": 0.01238,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0686254501342773,
      "kl": 0.6895125946030021,
      "learning_rate": 4.300801315809264e-06,
      "loss": -0.0075,
      "num_tokens": 25058101.0,
      "reward": 0.6484375,
      "reward_std": 0.6268197894096375,
      "rewards/rollout_reward_func/mean": 0.6484375,
      "rewards/rollout_reward_func/std": 0.6184428930282593,
      "sampling/importance_sampling_ratio/max": 1.5040462017059326,
      "sampling/importance_sampling_ratio/mean": 1.015794277191162,
      "sampling/importance_sampling_ratio/min": 0.4017060101032257,
      "sampling/sampling_logp_difference/max": 0.5123634338378906,
      "sampling/sampling_logp_difference/mean": 0.03303488343954086,
      "step": 619,
      "step_time": 12.348578431963688
    },
    {
      "clip_ratio/high_max": 0.009765625,
      "clip_ratio/high_mean": 0.0048828125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0048828125,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 651.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 194.40625,
      "completions/mean_terminated_length": 172.57144165039062,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.46744991559535265,
      "epoch": 0.0124,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0801821947097778,
      "kl": 0.31153961503878236,
      "learning_rate": 4.289179004761318e-06,
      "loss": 0.0145,
      "num_tokens": 25095897.0,
      "reward": 0.8574999570846558,
      "reward_std": 0.37863656878471375,
      "rewards/rollout_reward_func/mean": 0.8574999570846558,
      "rewards/rollout_reward_func/std": 0.3655398190021515,
      "sampling/importance_sampling_ratio/max": 1.5575536489486694,
      "sampling/importance_sampling_ratio/mean": 0.9732887744903564,
      "sampling/importance_sampling_ratio/min": 0.47846168279647827,
      "sampling/sampling_logp_difference/max": 0.5525853633880615,
      "sampling/sampling_logp_difference/mean": 0.03189120814204216,
      "step": 620,
      "step_time": 12.024468445044477
    },
    {
      "clip_ratio/high_max": 0.033203125,
      "clip_ratio/high_mean": 0.0166015625,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018439797800965607,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 622.0,
      "completions/max_terminated_length": 622.0,
      "completions/mean_length": 200.75,
      "completions/mean_terminated_length": 190.0370330810547,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.40916148060932755,
      "epoch": 0.01242,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.809978723526001,
      "kl": 0.574860867112875,
      "learning_rate": 4.277582558267819e-06,
      "loss": 0.0697,
      "num_tokens": 25133985.0,
      "reward": 0.6722279787063599,
      "reward_std": 0.6383928656578064,
      "rewards/rollout_reward_func/mean": 0.6722279787063599,
      "rewards/rollout_reward_func/std": 0.6588718295097351,
      "sampling/importance_sampling_ratio/max": 1.7208465337753296,
      "sampling/importance_sampling_ratio/mean": 0.9887014627456665,
      "sampling/importance_sampling_ratio/min": 0.4551844596862793,
      "sampling/sampling_logp_difference/max": 0.6274452209472656,
      "sampling/sampling_logp_difference/mean": 0.031036531552672386,
      "step": 621,
      "step_time": 12.45385315301246
    },
    {
      "clip_ratio/high_max": 0.013878106139600277,
      "clip_ratio/high_mean": 0.0069390530698001385,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00954321981407702,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 615.0,
      "completions/max_terminated_length": 615.0,
      "completions/mean_length": 191.5625,
      "completions/mean_terminated_length": 191.2413787841797,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.41744839027523994,
      "epoch": 0.01244,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6344752907752991,
      "kl": 0.34475054778158665,
      "learning_rate": 4.266012129293736e-06,
      "loss": -0.0501,
      "num_tokens": 25171306.0,
      "reward": 0.671875,
      "reward_std": 0.5482994318008423,
      "rewards/rollout_reward_func/mean": 0.671875,
      "rewards/rollout_reward_func/std": 0.6426178216934204,
      "sampling/importance_sampling_ratio/max": 1.5074139833450317,
      "sampling/importance_sampling_ratio/mean": 0.9925695657730103,
      "sampling/importance_sampling_ratio/min": 0.45931774377822876,
      "sampling/sampling_logp_difference/max": 0.45592832565307617,
      "sampling/sampling_logp_difference/mean": 0.026681948453187943,
      "step": 622,
      "step_time": 11.901037290037493
    },
    {
      "clip_ratio/high_max": 0.008576766354963183,
      "clip_ratio/high_mean": 0.004288383177481592,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005264945677481592,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 630.0,
      "completions/mean_length": 239.4375,
      "completions/mean_terminated_length": 215.71429443359375,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.6028543654829264,
      "epoch": 0.01246,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6146364212036133,
      "kl": 0.48394846078008413,
      "learning_rate": 4.254467870460848e-06,
      "loss": 0.0837,
      "num_tokens": 25212044.0,
      "reward": 0.7281249761581421,
      "reward_std": 0.5237429141998291,
      "rewards/rollout_reward_func/mean": 0.7281249761581421,
      "rewards/rollout_reward_func/std": 0.5398651957511902,
      "sampling/importance_sampling_ratio/max": 1.1864900588989258,
      "sampling/importance_sampling_ratio/mean": 0.8825137615203857,
      "sampling/importance_sampling_ratio/min": 1.1862870134225112e-18,
      "sampling/sampling_logp_difference/max": 18.19552993774414,
      "sampling/sampling_logp_difference/mean": 0.09807931631803513,
      "step": 623,
      "step_time": 13.555853240002762
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 584.0,
      "completions/max_terminated_length": 571.0,
      "completions/mean_length": 206.5625,
      "completions/mean_terminated_length": 212.36000061035156,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5286030182614923,
      "epoch": 0.01248,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3208808898925781,
      "kl": 0.34522461984306574,
      "learning_rate": 4.2429499340457344e-06,
      "loss": 0.0299,
      "num_tokens": 25250824.0,
      "reward": 0.7984374761581421,
      "reward_std": 0.40342098474502563,
      "rewards/rollout_reward_func/mean": 0.7984374761581421,
      "rewards/rollout_reward_func/std": 0.48405614495277405,
      "sampling/importance_sampling_ratio/max": 1.715543508529663,
      "sampling/importance_sampling_ratio/mean": 1.0032260417938232,
      "sampling/importance_sampling_ratio/min": 5.747210910236872e-22,
      "sampling/sampling_logp_difference/max": 23.25939178466797,
      "sampling/sampling_logp_difference/mean": 0.1079830527305603,
      "step": 624,
      "step_time": 12.466976085968781
    },
    {
      "clip_ratio/high_max": 0.0024038462433964014,
      "clip_ratio/high_mean": 0.0012019231216982007,
      "clip_ratio/low_mean": 0.0014880952658131719,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0026900183875113726,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 979.0,
      "completions/max_terminated_length": 979.0,
      "completions/mean_length": 246.78125,
      "completions/mean_terminated_length": 230.4285888671875,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.4954833649098873,
      "epoch": 0.0125,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2671587467193604,
      "kl": 0.36504484154284,
      "learning_rate": 4.231458471977764e-06,
      "loss": -0.0046,
      "num_tokens": 25291805.0,
      "reward": 0.6962499618530273,
      "reward_std": 0.4957740604877472,
      "rewards/rollout_reward_func/mean": 0.6962499618530273,
      "rewards/rollout_reward_func/std": 0.5411859750747681,
      "sampling/importance_sampling_ratio/max": 1.9450982809066772,
      "sampling/importance_sampling_ratio/mean": 0.9917658567428589,
      "sampling/importance_sampling_ratio/min": 3.5470918152504557e-16,
      "sampling/sampling_logp_difference/max": 19.145111083984375,
      "sampling/sampling_logp_difference/mean": 0.09580802917480469,
      "step": 625,
      "step_time": 13.60368804296013
    },
    {
      "clip_ratio/high_max": 0.0031250000465661287,
      "clip_ratio/high_mean": 0.0015625000232830644,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015625000232830644,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 599.0,
      "completions/max_terminated_length": 599.0,
      "completions/mean_length": 195.4375,
      "completions/mean_terminated_length": 194.04348754882812,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5146760982461274,
      "epoch": 0.01252,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.031713843345642,
      "kl": 0.546281230635941,
      "learning_rate": 4.219993635837086e-06,
      "loss": -0.1301,
      "num_tokens": 25330961.0,
      "reward": 0.7321875095367432,
      "reward_std": 0.6418102979660034,
      "rewards/rollout_reward_func/mean": 0.7321875095367432,
      "rewards/rollout_reward_func/std": 0.6419714093208313,
      "sampling/importance_sampling_ratio/max": 1.6271240711212158,
      "sampling/importance_sampling_ratio/mean": 0.9428456425666809,
      "sampling/importance_sampling_ratio/min": 0.39920690655708313,
      "sampling/sampling_logp_difference/max": 0.678685188293457,
      "sampling/sampling_logp_difference/mean": 0.03560907393693924,
      "step": 626,
      "step_time": 12.975824284992996
    },
    {
      "clip_ratio/high_max": 0.02278645895421505,
      "clip_ratio/high_mean": 0.011393229477107525,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011393229477107525,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 602.0,
      "completions/max_terminated_length": 602.0,
      "completions/mean_length": 206.4375,
      "completions/mean_terminated_length": 171.58621215820312,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.4657646035775542,
      "epoch": 0.01254,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7715516090393066,
      "kl": 1.029941977467388,
      "learning_rate": 4.208555576852641e-06,
      "loss": 0.2209,
      "num_tokens": 25369210.0,
      "reward": 0.7962499856948853,
      "reward_std": 0.42561089992523193,
      "rewards/rollout_reward_func/mean": 0.7962499856948853,
      "rewards/rollout_reward_func/std": 0.41055232286453247,
      "sampling/importance_sampling_ratio/max": 1.4709006547927856,
      "sampling/importance_sampling_ratio/mean": 0.9911445379257202,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.723546028137207,
      "sampling/sampling_logp_difference/mean": 0.03186333179473877,
      "step": 627,
      "step_time": 12.00896876702609
    },
    {
      "clip_ratio/high_max": 0.011543683474883437,
      "clip_ratio/high_mean": 0.005771841737441719,
      "clip_ratio/low_mean": 0.005884740501642227,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011656582239083946,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 657.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 233.34375,
      "completions/mean_terminated_length": 238.03448486328125,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.45523652993142605,
      "epoch": 0.01256,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7410451769828796,
      "kl": 0.34892272111028433,
      "learning_rate": 4.19714444590016e-06,
      "loss": -0.2205,
      "num_tokens": 25410377.0,
      "reward": 0.7121875286102295,
      "reward_std": 0.47230979800224304,
      "rewards/rollout_reward_func/mean": 0.7121875286102295,
      "rewards/rollout_reward_func/std": 0.598760724067688,
      "sampling/importance_sampling_ratio/max": 1.383239507675171,
      "sampling/importance_sampling_ratio/mean": 0.918788492679596,
      "sampling/importance_sampling_ratio/min": 4.301143233613557e-31,
      "sampling/sampling_logp_difference/max": 21.02988624572754,
      "sampling/sampling_logp_difference/mean": 0.262763112783432,
      "step": 628,
      "step_time": 12.392059158984921
    },
    {
      "clip_ratio/high_max": 0.005859375,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.0023352581774815917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005264945677481592,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 671.0,
      "completions/max_terminated_length": 671.0,
      "completions/mean_length": 355.875,
      "completions/mean_terminated_length": 349.34783935546875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5525628151372075,
      "epoch": 0.01258,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2931594848632812,
      "kl": 0.3016469869762659,
      "learning_rate": 4.185760393500174e-06,
      "loss": 0.0211,
      "num_tokens": 25456396.0,
      "reward": 0.6818749904632568,
      "reward_std": 0.4828013777732849,
      "rewards/rollout_reward_func/mean": 0.6818749904632568,
      "rewards/rollout_reward_func/std": 0.59498131275177,
      "sampling/importance_sampling_ratio/max": 1.611742377281189,
      "sampling/importance_sampling_ratio/mean": 1.0149261951446533,
      "sampling/importance_sampling_ratio/min": 0.4772132337093353,
      "sampling/sampling_logp_difference/max": 0.6615567207336426,
      "sampling/sampling_logp_difference/mean": 0.031769514083862305,
      "step": 629,
      "step_time": 14.073700168984942
    },
    {
      "clip_ratio/high_max": 0.01065340917557478,
      "clip_ratio/high_mean": 0.00532670458778739,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00630326708778739,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 675.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 269.0625,
      "completions/mean_terminated_length": 263.69232177734375,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5159411448985338,
      "epoch": 0.0126,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9075071811676025,
      "kl": 0.35273981280624866,
      "learning_rate": 4.17440356981603e-06,
      "loss": 0.0677,
      "num_tokens": 25497946.0,
      "reward": 0.7015624642372131,
      "reward_std": 0.42083996534347534,
      "rewards/rollout_reward_func/mean": 0.7015624642372131,
      "rewards/rollout_reward_func/std": 0.524718701839447,
      "sampling/importance_sampling_ratio/max": 1.269930362701416,
      "sampling/importance_sampling_ratio/mean": 0.8383061289787292,
      "sampling/importance_sampling_ratio/min": 4.6725588347991954e-36,
      "sampling/sampling_logp_difference/max": 23.437408447265625,
      "sampling/sampling_logp_difference/mean": 0.28909775614738464,
      "step": 630,
      "step_time": 12.629976460986654
    },
    {
      "clip_ratio/high_max": 0.009148848708719015,
      "clip_ratio/high_mean": 0.0055509868543595076,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0075041118543595076,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 602.0,
      "completions/max_terminated_length": 561.0,
      "completions/mean_length": 280.6875,
      "completions/mean_terminated_length": 276.67999267578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.526487547904253,
      "epoch": 0.01262,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7917975187301636,
      "kl": 0.4014758747071028,
      "learning_rate": 4.163074124651908e-06,
      "loss": 0.0549,
      "num_tokens": 25541238.0,
      "reward": 0.9856250286102295,
      "reward_std": 0.1143113225698471,
      "rewards/rollout_reward_func/mean": 0.9856250286102295,
      "rewards/rollout_reward_func/std": 0.1783244013786316,
      "sampling/importance_sampling_ratio/max": 1.8293527364730835,
      "sampling/importance_sampling_ratio/mean": 0.916845977306366,
      "sampling/importance_sampling_ratio/min": 9.274483958598978e-10,
      "sampling/sampling_logp_difference/max": 20.81830596923828,
      "sampling/sampling_logp_difference/mean": 0.11465317010879517,
      "step": 631,
      "step_time": 12.414007744984701
    },
    {
      "clip_ratio/high_max": 0.033203125,
      "clip_ratio/high_mean": 0.0166015625,
      "clip_ratio/low_mean": 0.0012019231216982007,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017803486436605453,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1762.0,
      "completions/max_terminated_length": 1762.0,
      "completions/mean_length": 313.9375,
      "completions/mean_terminated_length": 315.54168701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5480643883347511,
      "epoch": 0.01264,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0041017532348633,
      "kl": 0.4409613171592355,
      "learning_rate": 4.15177220745085e-06,
      "loss": -0.078,
      "num_tokens": 25585291.0,
      "reward": 0.5456249713897705,
      "reward_std": 0.36620092391967773,
      "rewards/rollout_reward_func/mean": 0.5456249713897705,
      "rewards/rollout_reward_func/std": 0.8030021786689758,
      "sampling/importance_sampling_ratio/max": 1.7287545204162598,
      "sampling/importance_sampling_ratio/mean": 0.9515678882598877,
      "sampling/importance_sampling_ratio/min": 3.882941967023562e-17,
      "sampling/sampling_logp_difference/max": 18.316600799560547,
      "sampling/sampling_logp_difference/mean": 0.10612385720014572,
      "step": 632,
      "step_time": 17.62044707001769
    },
    {
      "clip_ratio/high_max": 0.010286458535119891,
      "clip_ratio/high_mean": 0.005143229267559946,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005143229267559946,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 570.0,
      "completions/max_terminated_length": 570.0,
      "completions/mean_length": 190.71875,
      "completions/mean_terminated_length": 168.5,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5514382645487785,
      "epoch": 0.01266,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3710920810699463,
      "kl": 0.4121554112061858,
      "learning_rate": 4.140497967292782e-06,
      "loss": -0.1346,
      "num_tokens": 25623807.0,
      "reward": 0.7268750071525574,
      "reward_std": 0.46808046102523804,
      "rewards/rollout_reward_func/mean": 0.7268750071525574,
      "rewards/rollout_reward_func/std": 0.4495333433151245,
      "sampling/importance_sampling_ratio/max": 1.7290202379226685,
      "sampling/importance_sampling_ratio/mean": 0.9900645613670349,
      "sampling/importance_sampling_ratio/min": 5.112053733760138e-19,
      "sampling/sampling_logp_difference/max": 20.90449333190918,
      "sampling/sampling_logp_difference/mean": 0.11150877177715302,
      "step": 633,
      "step_time": 12.29807826600154
    },
    {
      "clip_ratio/high_max": 0.014787946827709675,
      "clip_ratio/high_mean": 0.007393973413854837,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007393973413854837,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 590.0,
      "completions/max_terminated_length": 590.0,
      "completions/mean_length": 197.53125,
      "completions/mean_terminated_length": 169.1999969482422,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5054810522124171,
      "epoch": 0.01268,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0637441873550415,
      "kl": 0.5974683482199907,
      "learning_rate": 4.1292515528925584e-06,
      "loss": 0.0208,
      "num_tokens": 25661613.0,
      "reward": 0.7618749737739563,
      "reward_std": 0.4825093150138855,
      "rewards/rollout_reward_func/mean": 0.7618749737739563,
      "rewards/rollout_reward_func/std": 0.4989371597766876,
      "sampling/importance_sampling_ratio/max": 1.8216336965560913,
      "sampling/importance_sampling_ratio/mean": 0.9819199442863464,
      "sampling/importance_sampling_ratio/min": 0.39279764890670776,
      "sampling/sampling_logp_difference/max": 0.8008525371551514,
      "sampling/sampling_logp_difference/mean": 0.03053533285856247,
      "step": 634,
      "step_time": 12.246384973972454
    },
    {
      "clip_ratio/high_max": 0.0044531249441206455,
      "clip_ratio/high_mean": 0.0022265624720603228,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022265624720603228,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 945.0,
      "completions/max_terminated_length": 642.0,
      "completions/mean_length": 244.625,
      "completions/mean_terminated_length": 195.9130401611328,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5522070527076721,
      "epoch": 0.0127,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6337816715240479,
      "kl": 0.32115186285227537,
      "learning_rate": 4.11803311259798e-06,
      "loss": -0.133,
      "num_tokens": 25701570.0,
      "reward": 0.8243750333786011,
      "reward_std": 0.344422310590744,
      "rewards/rollout_reward_func/mean": 0.8243750333786011,
      "rewards/rollout_reward_func/std": 0.39417481422424316,
      "sampling/importance_sampling_ratio/max": 1.6629810333251953,
      "sampling/importance_sampling_ratio/mean": 1.0074495077133179,
      "sampling/importance_sampling_ratio/min": 6.584422911760734e-26,
      "sampling/sampling_logp_difference/max": 20.07196617126465,
      "sampling/sampling_logp_difference/mean": 0.12750335037708282,
      "step": 635,
      "step_time": 14.514326909033116
    },
    {
      "clip_ratio/high_max": 0.02184103289619088,
      "clip_ratio/high_mean": 0.01092051644809544,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01287364144809544,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 686.0,
      "completions/max_terminated_length": 661.0,
      "completions/mean_length": 319.09375,
      "completions/mean_terminated_length": 274.4347839355469,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5217218864709139,
      "epoch": 0.01272,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8035814762115479,
      "kl": 1.1579428473487496,
      "learning_rate": 4.106842794387864e-06,
      "loss": 0.0971,
      "num_tokens": 25744794.0,
      "reward": 0.6971874833106995,
      "reward_std": 0.5085797309875488,
      "rewards/rollout_reward_func/mean": 0.6971874833106995,
      "rewards/rollout_reward_func/std": 0.5225756764411926,
      "sampling/importance_sampling_ratio/max": 1.7830475568771362,
      "sampling/importance_sampling_ratio/mean": 0.9812479615211487,
      "sampling/importance_sampling_ratio/min": 0.2886425852775574,
      "sampling/sampling_logp_difference/max": 0.767822265625,
      "sampling/sampling_logp_difference/mean": 0.035141974687576294,
      "step": 636,
      "step_time": 12.793081462019472
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.005440848413854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006417410913854837,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 654.0,
      "completions/max_terminated_length": 654.0,
      "completions/mean_length": 234.1875,
      "completions/mean_terminated_length": 187.59999084472656,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.46098513156175613,
      "epoch": 0.01274,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.5211427211761475,
      "kl": 0.5235399808734655,
      "learning_rate": 4.095680745870069e-06,
      "loss": -0.1443,
      "num_tokens": 25784787.0,
      "reward": 0.64906245470047,
      "reward_std": 0.5432247519493103,
      "rewards/rollout_reward_func/mean": 0.64906245470047,
      "rewards/rollout_reward_func/std": 0.7060492038726807,
      "sampling/importance_sampling_ratio/max": 1.8767929077148438,
      "sampling/importance_sampling_ratio/mean": 0.9998689293861389,
      "sampling/importance_sampling_ratio/min": 0.4463171362876892,
      "sampling/sampling_logp_difference/max": 0.8086655139923096,
      "sampling/sampling_logp_difference/mean": 0.030020227655768394,
      "step": 637,
      "step_time": 13.140541886998108
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0013586956774815917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0013586956774815917,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 682.0,
      "completions/max_terminated_length": 682.0,
      "completions/mean_length": 283.84375,
      "completions/mean_terminated_length": 286.59259033203125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.492624687962234,
      "epoch": 0.01276,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2161608934402466,
      "kl": 0.9753650529310107,
      "learning_rate": 4.084547114279564e-06,
      "loss": -0.1667,
      "num_tokens": 25827865.0,
      "reward": 0.5159374475479126,
      "reward_std": 0.7493351697921753,
      "rewards/rollout_reward_func/mean": 0.5159374475479126,
      "rewards/rollout_reward_func/std": 0.7409926056861877,
      "sampling/importance_sampling_ratio/max": 1.62056565284729,
      "sampling/importance_sampling_ratio/mean": 0.9639008045196533,
      "sampling/importance_sampling_ratio/min": 0.4063987731933594,
      "sampling/sampling_logp_difference/max": 0.9051151275634766,
      "sampling/sampling_logp_difference/mean": 0.029882458969950676,
      "step": 638,
      "step_time": 13.150188840023475
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00390625,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 679.0,
      "completions/max_terminated_length": 679.0,
      "completions/mean_length": 244.40625,
      "completions/mean_terminated_length": 258.5666809082031,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4288575630635023,
      "epoch": 0.01278,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7747998237609863,
      "kl": 0.348608055151999,
      "learning_rate": 4.073442046476472e-06,
      "loss": 0.0859,
      "num_tokens": 25868440.0,
      "reward": 0.7403124570846558,
      "reward_std": 0.5325278043746948,
      "rewards/rollout_reward_func/mean": 0.7403124570846558,
      "rewards/rollout_reward_func/std": 0.5182880163192749,
      "sampling/importance_sampling_ratio/max": 2.220425844192505,
      "sampling/importance_sampling_ratio/mean": 0.9561629891395569,
      "sampling/importance_sampling_ratio/min": 1.010294259924163e-24,
      "sampling/sampling_logp_difference/max": 24.159618377685547,
      "sampling/sampling_logp_difference/mean": 0.37431028485298157,
      "step": 639,
      "step_time": 12.101744464001968
    },
    {
      "clip_ratio/high_max": 0.01065340917557478,
      "clip_ratio/high_mean": 0.00532670458778739,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00532670458778739,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 1032.0,
      "completions/max_terminated_length": 648.0,
      "completions/mean_length": 335.9375,
      "completions/mean_terminated_length": 271.9565124511719,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5258694496005774,
      "epoch": 0.0128,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3681988716125488,
      "kl": 0.40337555948644876,
      "learning_rate": 4.062365688944147e-06,
      "loss": -0.2216,
      "num_tokens": 25913271.0,
      "reward": 0.8287500143051147,
      "reward_std": 0.4056257903575897,
      "rewards/rollout_reward_func/mean": 0.8287500143051147,
      "rewards/rollout_reward_func/std": 0.39430418610572815,
      "sampling/importance_sampling_ratio/max": 2.1107852458953857,
      "sampling/importance_sampling_ratio/mean": 1.0442858934402466,
      "sampling/importance_sampling_ratio/min": 0.26700520515441895,
      "sampling/sampling_logp_difference/max": 0.5990829467773438,
      "sampling/sampling_logp_difference/mean": 0.03319328650832176,
      "step": 640,
      "step_time": 14.75180468398321
    },
    {
      "clip_ratio/high_max": 0.011443995637819171,
      "clip_ratio/high_mean": 0.0057219978189095855,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0057219978189095855,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 968.0,
      "completions/max_terminated_length": 968.0,
      "completions/mean_length": 298.53125,
      "completions/mean_terminated_length": 282.6521911621094,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5423181029036641,
      "epoch": 0.01282,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.042029619216919,
      "kl": 1.6949415523558855,
      "learning_rate": 4.051318187787231e-06,
      "loss": -0.1482,
      "num_tokens": 25955593.0,
      "reward": 0.6415624618530273,
      "reward_std": 0.7571772336959839,
      "rewards/rollout_reward_func/mean": 0.6415624618530273,
      "rewards/rollout_reward_func/std": 0.7276878952980042,
      "sampling/importance_sampling_ratio/max": 1.6064164638519287,
      "sampling/importance_sampling_ratio/mean": 0.7568527460098267,
      "sampling/importance_sampling_ratio/min": 3.053160073937869e-14,
      "sampling/sampling_logp_difference/max": 20.531179428100586,
      "sampling/sampling_logp_difference/mean": 0.09941942989826202,
      "step": 641,
      "step_time": 14.34555857401574
    },
    {
      "clip_ratio/high_max": 0.008666088106110692,
      "clip_ratio/high_mean": 0.004333044053055346,
      "clip_ratio/low_mean": 0.020312500186264515,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02464554423931986,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 615.0,
      "completions/max_terminated_length": 615.0,
      "completions/mean_length": 248.9375,
      "completions/mean_terminated_length": 221.68182373046875,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.5331151150166988,
      "epoch": 0.01284,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7779539823532104,
      "kl": 0.34299719613045454,
      "learning_rate": 4.040299688729734e-06,
      "loss": -0.0723,
      "num_tokens": 25996463.0,
      "reward": 0.8878124952316284,
      "reward_std": 0.35779091715812683,
      "rewards/rollout_reward_func/mean": 0.8878124952316284,
      "rewards/rollout_reward_func/std": 0.4193599820137024,
      "sampling/importance_sampling_ratio/max": 1.3918652534484863,
      "sampling/importance_sampling_ratio/mean": 0.8824934959411621,
      "sampling/importance_sampling_ratio/min": 2.6002182673226785e-18,
      "sampling/sampling_logp_difference/max": 26.618288040161133,
      "sampling/sampling_logp_difference/mean": 0.24931152164936066,
      "step": 642,
      "step_time": 12.557795547982096
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.005063657416030765,
      "clip_ratio/low_min": 0.001953125,
      "clip_ratio/region_mean": 0.006040219916030765,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 783.0,
      "completions/max_terminated_length": 558.0,
      "completions/mean_length": 245.875,
      "completions/mean_terminated_length": 243.13636779785156,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5219305250793695,
      "epoch": 0.01286,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.511185646057129,
      "kl": 0.5575648061931133,
      "learning_rate": 4.029310337113106e-06,
      "loss": 0.1989,
      "num_tokens": 26037924.0,
      "reward": 0.9440624713897705,
      "reward_std": 0.2114982306957245,
      "rewards/rollout_reward_func/mean": 0.9440624713897705,
      "rewards/rollout_reward_func/std": 0.351630836725235,
      "sampling/importance_sampling_ratio/max": 1.7022514343261719,
      "sampling/importance_sampling_ratio/mean": 0.8744444847106934,
      "sampling/importance_sampling_ratio/min": 1.675865274054474e-17,
      "sampling/sampling_logp_difference/max": 25.309715270996094,
      "sampling/sampling_logp_difference/mean": 0.10104858130216599,
      "step": 643,
      "step_time": 14.100211668992415
    },
    {
      "clip_ratio/high_max": 0.005859375,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.00048828125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00341796875,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 1729.0,
      "completions/max_terminated_length": 563.0,
      "completions/mean_length": 308.9375,
      "completions/mean_terminated_length": 221.51998901367188,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4788389215245843,
      "epoch": 0.01288,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3528164625167847,
      "kl": 0.5306055713444948,
      "learning_rate": 4.018350277894324e-06,
      "loss": -0.0228,
      "num_tokens": 26081307.0,
      "reward": 0.7400000095367432,
      "reward_std": 0.5317533612251282,
      "rewards/rollout_reward_func/mean": 0.7400000095367432,
      "rewards/rollout_reward_func/std": 0.5753372311592102,
      "sampling/importance_sampling_ratio/max": 1.6912721395492554,
      "sampling/importance_sampling_ratio/mean": 1.024867296218872,
      "sampling/importance_sampling_ratio/min": 0.607761800289154,
      "sampling/sampling_logp_difference/max": 0.6336288452148438,
      "sampling/sampling_logp_difference/mean": 0.03259403258562088,
      "step": 644,
      "step_time": 17.008598452972365
    },
    {
      "clip_ratio/high_max": 0.004185267956927419,
      "clip_ratio/high_mean": 0.0020926339784637094,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0020926339784637094,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 1112.0,
      "completions/max_terminated_length": 1112.0,
      "completions/mean_length": 241.28125,
      "completions/mean_terminated_length": 189.43478393554688,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.47764470241963863,
      "epoch": 0.0129,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9428596496582031,
      "kl": 0.7857241360470653,
      "learning_rate": 4.007419655643977e-06,
      "loss": -0.0272,
      "num_tokens": 26120472.0,
      "reward": 0.7893749475479126,
      "reward_std": 0.5233283042907715,
      "rewards/rollout_reward_func/mean": 0.7893749475479126,
      "rewards/rollout_reward_func/std": 0.6246130466461182,
      "sampling/importance_sampling_ratio/max": 2.2819390296936035,
      "sampling/importance_sampling_ratio/mean": 0.9373723864555359,
      "sampling/importance_sampling_ratio/min": 0.3376237750053406,
      "sampling/sampling_logp_difference/max": 1.0712857246398926,
      "sampling/sampling_logp_difference/mean": 0.03760301321744919,
      "step": 645,
      "step_time": 13.685095608962001
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0063281250186264515,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008281250018626451,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 671.0,
      "completions/max_terminated_length": 671.0,
      "completions/mean_length": 303.0625,
      "completions/mean_terminated_length": 303.0833435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5928616030141711,
      "epoch": 0.01292,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6239919662475586,
      "kl": 0.44119742047041655,
      "learning_rate": 3.996518614544363e-06,
      "loss": -0.0272,
      "num_tokens": 26164218.0,
      "reward": 0.5156654715538025,
      "reward_std": 0.644976019859314,
      "rewards/rollout_reward_func/mean": 0.5156654715538025,
      "rewards/rollout_reward_func/std": 0.678822934627533,
      "sampling/importance_sampling_ratio/max": 1.596645712852478,
      "sampling/importance_sampling_ratio/mean": 0.8975968360900879,
      "sampling/importance_sampling_ratio/min": 2.194442134268732e-28,
      "sampling/sampling_logp_difference/max": 27.665857315063477,
      "sampling/sampling_logp_difference/mean": 0.30131256580352783,
      "step": 646,
      "step_time": 13.730127479997464
    },
    {
      "clip_ratio/high_max": 0.005893640452995896,
      "clip_ratio/high_mean": 0.002946820226497948,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002946820226497948,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 1139.0,
      "completions/max_terminated_length": 1139.0,
      "completions/mean_length": 252.53125,
      "completions/mean_terminated_length": 238.27999877929688,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47967411018908024,
      "epoch": 0.01294,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2212510108947754,
      "kl": 0.3242537258192897,
      "learning_rate": 3.985647298387584e-06,
      "loss": -0.0707,
      "num_tokens": 26204054.0,
      "reward": 0.6424999833106995,
      "reward_std": 0.5538876056671143,
      "rewards/rollout_reward_func/mean": 0.6424999833106995,
      "rewards/rollout_reward_func/std": 0.5444618463516235,
      "sampling/importance_sampling_ratio/max": 2.5261645317077637,
      "sampling/importance_sampling_ratio/mean": 1.0630093812942505,
      "sampling/importance_sampling_ratio/min": 0.5884976387023926,
      "sampling/sampling_logp_difference/max": 0.79388427734375,
      "sampling/sampling_logp_difference/mean": 0.03029664233326912,
      "step": 647,
      "step_time": 14.578450436005369
    },
    {
      "clip_ratio/high_max": 0.008795797592028975,
      "clip_ratio/high_mean": 0.004397898796014488,
      "clip_ratio/low_mean": 0.009905134327709675,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014303033123724163,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 667.0,
      "completions/mean_length": 215.4375,
      "completions/mean_terminated_length": 200.6521759033203,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5543844252824783,
      "epoch": 0.01296,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1026347875595093,
      "kl": 1.1691994611173868,
      "learning_rate": 3.9748058505736474e-06,
      "loss": -0.2164,
      "num_tokens": 26243447.0,
      "reward": 0.5275510549545288,
      "reward_std": 0.8266417384147644,
      "rewards/rollout_reward_func/mean": 0.5275510549545288,
      "rewards/rollout_reward_func/std": 0.8026811480522156,
      "sampling/importance_sampling_ratio/max": 2.2944977283477783,
      "sampling/importance_sampling_ratio/mean": 0.8585614562034607,
      "sampling/importance_sampling_ratio/min": 1.4578782224717673e-18,
      "sampling/sampling_logp_difference/max": 20.818790435791016,
      "sampling/sampling_logp_difference/mean": 0.25127506256103516,
      "step": 648,
      "step_time": 12.44270133599639
    },
    {
      "clip_ratio/high_max": 0.0034722222480922937,
      "clip_ratio/high_mean": 0.0017361111240461469,
      "clip_ratio/low_mean": 0.0011574074160307646,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0028935185400769114,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 677.0,
      "completions/max_terminated_length": 677.0,
      "completions/mean_length": 248.375,
      "completions/mean_terminated_length": 248.84616088867188,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5085415113717318,
      "epoch": 0.01298,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1116852760314941,
      "kl": 0.6289421338587999,
      "learning_rate": 3.9639944141085785e-06,
      "loss": -0.1898,
      "num_tokens": 26284343.0,
      "reward": 0.762499988079071,
      "reward_std": 0.5148347616195679,
      "rewards/rollout_reward_func/mean": 0.762499988079071,
      "rewards/rollout_reward_func/std": 0.5871692299842834,
      "sampling/importance_sampling_ratio/max": 1.5845909118652344,
      "sampling/importance_sampling_ratio/mean": 0.814691424369812,
      "sampling/importance_sampling_ratio/min": 6.420309191298215e-23,
      "sampling/sampling_logp_difference/max": 20.572954177856445,
      "sampling/sampling_logp_difference/mean": 0.21118222177028656,
      "step": 649,
      "step_time": 13.855829565014574
    },
    {
      "clip_ratio/high_max": 0.010190318804234266,
      "clip_ratio/high_mean": 0.005095159402117133,
      "clip_ratio/low_mean": 0.0029296875,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008024846902117133,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 607.0,
      "completions/mean_length": 244.875,
      "completions/mean_terminated_length": 200.0,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5937076229602098,
      "epoch": 0.013,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1768957376480103,
      "kl": 0.7548494054935873,
      "learning_rate": 3.953213131602535e-06,
      "loss": -0.1969,
      "num_tokens": 26325768.0,
      "reward": 0.5113126635551453,
      "reward_std": 0.6920714378356934,
      "rewards/rollout_reward_func/mean": 0.5113126635551453,
      "rewards/rollout_reward_func/std": 0.6893973350524902,
      "sampling/importance_sampling_ratio/max": 1.6177371740341187,
      "sampling/importance_sampling_ratio/mean": 0.9023336172103882,
      "sampling/importance_sampling_ratio/min": 2.746678750259044e-20,
      "sampling/sampling_logp_difference/max": 20.05056381225586,
      "sampling/sampling_logp_difference/mean": 0.2126399129629135,
      "step": 650,
      "step_time": 12.637635797960684
    },
    {
      "clip_ratio/high_max": 0.04462852003052831,
      "clip_ratio/high_mean": 0.023290822515264153,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.025243947515264153,
      "completions/clipped_ratio": 0.40625,
      "completions/max_length": 701.0,
      "completions/max_terminated_length": 678.0,
      "completions/mean_length": 261.90625,
      "completions/mean_terminated_length": 215.42105102539062,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5637107528746128,
      "epoch": 0.01302,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.6516976356506348,
      "kl": 0.25286534428596497,
      "learning_rate": 3.942462145267919e-06,
      "loss": 0.1111,
      "num_tokens": 26367756.0,
      "reward": 0.7925000190734863,
      "reward_std": 0.3794803023338318,
      "rewards/rollout_reward_func/mean": 0.7925000190734863,
      "rewards/rollout_reward_func/std": 0.4309554100036621,
      "sampling/importance_sampling_ratio/max": 1.9300872087478638,
      "sampling/importance_sampling_ratio/mean": 0.9909601211547852,
      "sampling/importance_sampling_ratio/min": 5.584900726994034e-29,
      "sampling/sampling_logp_difference/max": 28.965049743652344,
      "sampling/sampling_logp_difference/mean": 0.12322109192609787,
      "step": 651,
      "step_time": 13.462400696982513
    },
    {
      "clip_ratio/high_max": 0.014307823264971375,
      "clip_ratio/high_mean": 0.007153911632485688,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008130474132485688,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1038.0,
      "completions/max_terminated_length": 1038.0,
      "completions/mean_length": 243.5625,
      "completions/mean_terminated_length": 217.69232177734375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5335935559123755,
      "epoch": 0.01304,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4511502981185913,
      "kl": 0.469424344599247,
      "learning_rate": 3.93174159691751e-06,
      "loss": 0.0379,
      "num_tokens": 26408810.0,
      "reward": 0.8624999523162842,
      "reward_std": 0.32091107964515686,
      "rewards/rollout_reward_func/mean": 0.8624999523162842,
      "rewards/rollout_reward_func/std": 0.36751121282577515,
      "sampling/importance_sampling_ratio/max": 1.257032871246338,
      "sampling/importance_sampling_ratio/mean": 0.8523473143577576,
      "sampling/importance_sampling_ratio/min": 5.45072933445224e-26,
      "sampling/sampling_logp_difference/max": 26.99254035949707,
      "sampling/sampling_logp_difference/mean": 0.25969815254211426,
      "step": 652,
      "step_time": 14.46380248197238
    },
    {
      "clip_ratio/high_max": 0.021183300530537963,
      "clip_ratio/high_mean": 0.010591650265268981,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010591650265268981,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 923.0,
      "completions/max_terminated_length": 923.0,
      "completions/mean_length": 216.21875,
      "completions/mean_terminated_length": 231.46429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3906334089115262,
      "epoch": 0.01306,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7844971418380737,
      "kl": 0.3668788615614176,
      "learning_rate": 3.921051627962586e-06,
      "loss": -0.119,
      "num_tokens": 26447648.0,
      "reward": 0.7140624523162842,
      "reward_std": 0.631829559803009,
      "rewards/rollout_reward_func/mean": 0.7140624523162842,
      "rewards/rollout_reward_func/std": 0.636522114276886,
      "sampling/importance_sampling_ratio/max": 1.7737332582473755,
      "sampling/importance_sampling_ratio/mean": 1.0499554872512817,
      "sampling/importance_sampling_ratio/min": 9.9319112928099e-16,
      "sampling/sampling_logp_difference/max": 26.989627838134766,
      "sampling/sampling_logp_difference/mean": 0.1025414988398552,
      "step": 653,
      "step_time": 12.867088328988757
    },
    {
      "clip_ratio/high_max": 0.0069444444961845875,
      "clip_ratio/high_mean": 0.0034722222480922937,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0034722222480922937,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1116.0,
      "completions/max_terminated_length": 628.0,
      "completions/mean_length": 288.25,
      "completions/mean_terminated_length": 250.61538696289062,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.5569934528321028,
      "epoch": 0.01308,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4722658395767212,
      "kl": 0.40740840695798397,
      "learning_rate": 3.9103923794110645e-06,
      "loss": -0.1372,
      "num_tokens": 26490061.0,
      "reward": 0.6759374737739563,
      "reward_std": 0.681288480758667,
      "rewards/rollout_reward_func/mean": 0.6759374737739563,
      "rewards/rollout_reward_func/std": 0.6546698808670044,
      "sampling/importance_sampling_ratio/max": 1.5871808528900146,
      "sampling/importance_sampling_ratio/mean": 0.9722810387611389,
      "sampling/importance_sampling_ratio/min": 9.631301629169954e-15,
      "sampling/sampling_logp_difference/max": 21.310565948486328,
      "sampling/sampling_logp_difference/mean": 0.09009857475757599,
      "step": 654,
      "step_time": 14.478101918983157
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 679.0,
      "completions/max_terminated_length": 679.0,
      "completions/mean_length": 246.53125,
      "completions/mean_terminated_length": 242.19232177734375,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4455076758749783,
      "epoch": 0.0131,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5863077640533447,
      "kl": 0.5172448880039155,
      "learning_rate": 3.89976399186564e-06,
      "loss": 0.0654,
      "num_tokens": 26530867.0,
      "reward": 0.6190624237060547,
      "reward_std": 0.5675363540649414,
      "rewards/rollout_reward_func/mean": 0.6190624237060547,
      "rewards/rollout_reward_func/std": 0.6579849720001221,
      "sampling/importance_sampling_ratio/max": 2.1118557453155518,
      "sampling/importance_sampling_ratio/mean": 0.9929773211479187,
      "sampling/importance_sampling_ratio/min": 0.38423606753349304,
      "sampling/sampling_logp_difference/max": 0.957521915435791,
      "sampling/sampling_logp_difference/mean": 0.027804769575595856,
      "step": 655,
      "step_time": 12.88084891003382
    },
    {
      "clip_ratio/high_max": 0.01260653417557478,
      "clip_ratio/high_mean": 0.00630326708778739,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00630326708778739,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 960.0,
      "completions/max_terminated_length": 960.0,
      "completions/mean_length": 237.65625,
      "completions/mean_terminated_length": 230.478271484375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4961145920678973,
      "epoch": 0.01312,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4906868934631348,
      "kl": 0.4206564025953412,
      "learning_rate": 3.8891666055219305e-06,
      "loss": -0.0639,
      "num_tokens": 26570565.0,
      "reward": 0.7253124713897705,
      "reward_std": 0.49947547912597656,
      "rewards/rollout_reward_func/mean": 0.7253124713897705,
      "rewards/rollout_reward_func/std": 0.5715851187705994,
      "sampling/importance_sampling_ratio/max": 2.0810530185699463,
      "sampling/importance_sampling_ratio/mean": 1.0011961460113525,
      "sampling/importance_sampling_ratio/min": 0.32130250334739685,
      "sampling/sampling_logp_difference/max": 0.6024518013000488,
      "sampling/sampling_logp_difference/mean": 0.03333193436264992,
      "step": 656,
      "step_time": 13.263278442012961
    },
    {
      "clip_ratio/high_max": 0.0114297850523144,
      "clip_ratio/high_mean": 0.0057148925261572,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0057148925261572,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 1163.0,
      "completions/max_terminated_length": 652.0,
      "completions/mean_length": 259.25,
      "completions/mean_terminated_length": 205.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6600025203078985,
      "epoch": 0.01314,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4030274152755737,
      "kl": 0.3860687408596277,
      "learning_rate": 3.878600360166621e-06,
      "loss": -0.0405,
      "num_tokens": 26612742.0,
      "reward": 0.6725000143051147,
      "reward_std": 0.5298144817352295,
      "rewards/rollout_reward_func/mean": 0.6725000143051147,
      "rewards/rollout_reward_func/std": 0.5353322625160217,
      "sampling/importance_sampling_ratio/max": 1.6080098152160645,
      "sampling/importance_sampling_ratio/mean": 0.8399484157562256,
      "sampling/importance_sampling_ratio/min": 3.4886985984338085e-19,
      "sampling/sampling_logp_difference/max": 21.673446655273438,
      "sampling/sampling_logp_difference/mean": 0.2380831092596054,
      "step": 657,
      "step_time": 16.104583748965524
    },
    {
      "clip_ratio/high_max": 0.0029761905316263437,
      "clip_ratio/high_mean": 0.0014880952658131719,
      "clip_ratio/low_mean": 0.0013020833721384406,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0027901786379516125,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 954.0,
      "completions/max_terminated_length": 603.0,
      "completions/mean_length": 254.03125,
      "completions/mean_terminated_length": 221.69232177734375,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5386718828231096,
      "epoch": 0.01316,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7302895784378052,
      "kl": 0.9053495209664106,
      "learning_rate": 3.868065395175636e-06,
      "loss": -0.0664,
      "num_tokens": 26652975.0,
      "reward": 0.7340624928474426,
      "reward_std": 0.48753872513771057,
      "rewards/rollout_reward_func/mean": 0.7340624928474426,
      "rewards/rollout_reward_func/std": 0.5895150899887085,
      "sampling/importance_sampling_ratio/max": 1.8957247734069824,
      "sampling/importance_sampling_ratio/mean": 1.0167195796966553,
      "sampling/importance_sampling_ratio/min": 0.22261814773082733,
      "sampling/sampling_logp_difference/max": 0.607579231262207,
      "sampling/sampling_logp_difference/mean": 0.03032919019460678,
      "step": 658,
      "step_time": 13.37992796600156
    },
    {
      "clip_ratio/high_max": 0.0052083334885537624,
      "clip_ratio/high_mean": 0.0036817528307437897,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036817528307437897,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 664.0,
      "completions/max_terminated_length": 626.0,
      "completions/mean_length": 229.15625,
      "completions/mean_terminated_length": 202.85714721679688,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.543981259688735,
      "epoch": 0.01318,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1238160133361816,
      "kl": 0.30927570909261703,
      "learning_rate": 3.857561849512283e-06,
      "loss": -0.1332,
      "num_tokens": 26691920.0,
      "reward": 0.7071874737739563,
      "reward_std": 0.5818419456481934,
      "rewards/rollout_reward_func/mean": 0.7071874737739563,
      "rewards/rollout_reward_func/std": 0.5929055213928223,
      "sampling/importance_sampling_ratio/max": 1.287011742591858,
      "sampling/importance_sampling_ratio/mean": 0.9279627799987793,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.4551877975463867,
      "sampling/sampling_logp_difference/mean": 0.03408265486359596,
      "step": 659,
      "step_time": 12.542231423940393
    },
    {
      "clip_ratio/high_max": 0.0060614224057644606,
      "clip_ratio/high_mean": 0.0030307112028822303,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00400727370288223,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 654.0,
      "completions/mean_length": 207.75,
      "completions/mean_terminated_length": 190.88462829589844,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5926008690148592,
      "epoch": 0.0132,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6266796588897705,
      "kl": 1.8480177726596594,
      "learning_rate": 3.847089861725429e-06,
      "loss": 0.0541,
      "num_tokens": 26729909.0,
      "reward": 0.5965625047683716,
      "reward_std": 0.6497849822044373,
      "rewards/rollout_reward_func/mean": 0.5965625047683716,
      "rewards/rollout_reward_func/std": 0.6861138939857483,
      "sampling/importance_sampling_ratio/max": 1.62844717502594,
      "sampling/importance_sampling_ratio/mean": 0.960669219493866,
      "sampling/importance_sampling_ratio/min": 6.546182324505823e-14,
      "sampling/sampling_logp_difference/max": 21.67974853515625,
      "sampling/sampling_logp_difference/mean": 0.09602956473827362,
      "step": 660,
      "step_time": 12.99655976901704
    },
    {
      "clip_ratio/high_max": 0.017153532709926367,
      "clip_ratio/high_mean": 0.008576766354963183,
      "clip_ratio/low_mean": 0.007161458372138441,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015738224959932268,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 752.0,
      "completions/max_terminated_length": 752.0,
      "completions/mean_length": 325.625,
      "completions/mean_terminated_length": 297.8500061035156,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.612508911639452,
      "epoch": 0.01322,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.112795352935791,
      "kl": 0.3488847855478525,
      "learning_rate": 3.836649569947673e-06,
      "loss": 0.1257,
      "num_tokens": 26774031.0,
      "reward": 0.6750405430793762,
      "reward_std": 0.4311358630657196,
      "rewards/rollout_reward_func/mean": 0.6750405430793762,
      "rewards/rollout_reward_func/std": 0.5964766144752502,
      "sampling/importance_sampling_ratio/max": 1.7558510303497314,
      "sampling/importance_sampling_ratio/mean": 0.9701039791107178,
      "sampling/importance_sampling_ratio/min": 5.350972962974774e-19,
      "sampling/sampling_logp_difference/max": 24.587970733642578,
      "sampling/sampling_logp_difference/mean": 0.09769964963197708,
      "step": 661,
      "step_time": 13.40111484999943
    },
    {
      "clip_ratio/high_max": 0.006623641354963183,
      "clip_ratio/high_mean": 0.0033118206774815917,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0033118206774815917,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 603.0,
      "completions/max_terminated_length": 603.0,
      "completions/mean_length": 236.375,
      "completions/mean_terminated_length": 235.3199920654297,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 0.5349178984761238,
      "epoch": 0.01324,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1088210344314575,
      "kl": 0.32908192090690136,
      "learning_rate": 3.82624111189352e-06,
      "loss": -0.014,
      "num_tokens": 26813873.0,
      "reward": 0.8949999809265137,
      "reward_std": 0.23617109656333923,
      "rewards/rollout_reward_func/mean": 0.8949999809265137,
      "rewards/rollout_reward_func/std": 0.3333892226219177,
      "sampling/importance_sampling_ratio/max": 1.4703048467636108,
      "sampling/importance_sampling_ratio/mean": 0.9246768951416016,
      "sampling/importance_sampling_ratio/min": 3.026106730987161e-15,
      "sampling/sampling_logp_difference/max": 8.50521183013916,
      "sampling/sampling_logp_difference/mean": 0.10635322332382202,
      "step": 662,
      "step_time": 12.503199219965609
    },
    {
      "clip_ratio/high_max": 0.011067708488553762,
      "clip_ratio/high_mean": 0.005533854244276881,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005533854244276881,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 968.0,
      "completions/max_terminated_length": 968.0,
      "completions/mean_length": 279.75,
      "completions/mean_terminated_length": 287.9615478515625,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.49003213830292225,
      "epoch": 0.01326,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.394606351852417,
      "kl": 1.4356586579233408,
      "learning_rate": 3.81586462485757e-06,
      "loss": 0.0508,
      "num_tokens": 26856386.0,
      "reward": 0.608514666557312,
      "reward_std": 0.5997353196144104,
      "rewards/rollout_reward_func/mean": 0.608514666557312,
      "rewards/rollout_reward_func/std": 0.6206797361373901,
      "sampling/importance_sampling_ratio/max": 1.3352476358413696,
      "sampling/importance_sampling_ratio/mean": 0.8522337675094604,
      "sampling/importance_sampling_ratio/min": 2.0221571583104085e-18,
      "sampling/sampling_logp_difference/max": 23.6413516998291,
      "sampling/sampling_logp_difference/mean": 0.15707308053970337,
      "step": 663,
      "step_time": 14.418104309996124
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0011574074160307646,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031105324160307646,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 267.21875,
      "completions/mean_terminated_length": 240.58334350585938,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.45701839262619615,
      "epoch": 0.01328,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3754663467407227,
      "kl": 0.5415058629587293,
      "learning_rate": 3.8055202457127015e-06,
      "loss": -0.1567,
      "num_tokens": 26898345.0,
      "reward": 0.8112499713897705,
      "reward_std": 0.42353954911231995,
      "rewards/rollout_reward_func/mean": 0.8112499713897705,
      "rewards/rollout_reward_func/std": 0.49223482608795166,
      "sampling/importance_sampling_ratio/max": 1.572266936302185,
      "sampling/importance_sampling_ratio/mean": 0.937019944190979,
      "sampling/importance_sampling_ratio/min": 1.380649435158943e-18,
      "sampling/sampling_logp_difference/max": 21.05695152282715,
      "sampling/sampling_logp_difference/mean": 0.09237344563007355,
      "step": 664,
      "step_time": 12.731940999045037
    },
    {
      "clip_ratio/high_max": 0.008463541744276881,
      "clip_ratio/high_mean": 0.004231770872138441,
      "clip_ratio/low_mean": 0.0013020833721384406,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005533854244276881,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 650.0,
      "completions/max_terminated_length": 650.0,
      "completions/mean_length": 252.15625,
      "completions/mean_terminated_length": 203.0,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.5448182132095098,
      "epoch": 0.0133,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6955535411834717,
      "kl": 0.7063761493191123,
      "learning_rate": 3.795208110908265e-06,
      "loss": -0.0602,
      "num_tokens": 26938449.0,
      "reward": 0.6397280097007751,
      "reward_std": 0.579208254814148,
      "rewards/rollout_reward_func/mean": 0.6397280097007751,
      "rewards/rollout_reward_func/std": 0.6109738945960999,
      "sampling/importance_sampling_ratio/max": 2.1526055335998535,
      "sampling/importance_sampling_ratio/mean": 0.990337610244751,
      "sampling/importance_sampling_ratio/min": 1.12121688289335e-05,
      "sampling/sampling_logp_difference/max": 11.172160148620605,
      "sampling/sampling_logp_difference/mean": 0.052998725324869156,
      "step": 665,
      "step_time": 12.965288296996732
    },
    {
      "clip_ratio/high_max": 0.007378472248092294,
      "clip_ratio/high_mean": 0.003689236124046147,
      "clip_ratio/low_mean": 0.0010080644860863686,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004697300610132515,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 584.0,
      "completions/max_terminated_length": 584.0,
      "completions/mean_length": 202.84375,
      "completions/mean_terminated_length": 166.43478393554688,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.5389713225886226,
      "epoch": 0.01332,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5376530885696411,
      "kl": 0.45315183605998755,
      "learning_rate": 3.784928356468293e-06,
      "loss": 0.0332,
      "num_tokens": 26977973.0,
      "reward": 0.9206249713897705,
      "reward_std": 0.27660924196243286,
      "rewards/rollout_reward_func/mean": 0.9206249713897705,
      "rewards/rollout_reward_func/std": 0.29619017243385315,
      "sampling/importance_sampling_ratio/max": 1.596404790878296,
      "sampling/importance_sampling_ratio/mean": 0.8665210008621216,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 22.181262969970703,
      "sampling/sampling_logp_difference/mean": 0.1433517038822174,
      "step": 666,
      "step_time": 13.007520418002969
    },
    {
      "clip_ratio/high_max": 0.012834821827709675,
      "clip_ratio/high_mean": 0.006417410913854837,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006417410913854837,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 995.0,
      "completions/max_terminated_length": 995.0,
      "completions/mean_length": 206.59375,
      "completions/mean_terminated_length": 221.51998901367188,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47050856705754995,
      "epoch": 0.01334,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6101274490356445,
      "kl": 0.37388406228274107,
      "learning_rate": 3.7746811179896937e-06,
      "loss": -0.0019,
      "num_tokens": 27016266.0,
      "reward": 0.888124942779541,
      "reward_std": 0.3610377907752991,
      "rewards/rollout_reward_func/mean": 0.888124942779541,
      "rewards/rollout_reward_func/std": 0.41732972860336304,
      "sampling/importance_sampling_ratio/max": 1.8128730058670044,
      "sampling/importance_sampling_ratio/mean": 1.0128402709960938,
      "sampling/importance_sampling_ratio/min": 0.5188121795654297,
      "sampling/sampling_logp_difference/max": 0.482529878616333,
      "sampling/sampling_logp_difference/mean": 0.027551786974072456,
      "step": 667,
      "step_time": 13.508988971982035
    },
    {
      "clip_ratio/high_max": 0.007440476445481181,
      "clip_ratio/high_mean": 0.0037202382227405906,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0037202382227405906,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 612.0,
      "completions/mean_length": 250.59375,
      "completions/mean_terminated_length": 212.92308044433594,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47440165374428034,
      "epoch": 0.01336,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0594309568405151,
      "kl": 0.43300297390669584,
      "learning_rate": 3.7644665306404726e-06,
      "loss": -0.0195,
      "num_tokens": 27057923.0,
      "reward": 0.6728529930114746,
      "reward_std": 0.6757850646972656,
      "rewards/rollout_reward_func/mean": 0.6728529930114746,
      "rewards/rollout_reward_func/std": 0.6628318428993225,
      "sampling/importance_sampling_ratio/max": 1.2249152660369873,
      "sampling/importance_sampling_ratio/mean": 0.8100740909576416,
      "sampling/importance_sampling_ratio/min": 2.218621913241429e-23,
      "sampling/sampling_logp_difference/max": 20.988248825073242,
      "sampling/sampling_logp_difference/mean": 0.2012956291437149,
      "step": 668,
      "step_time": 13.022509737027576
    },
    {
      "clip_ratio/high_max": 0.00687210657633841,
      "clip_ratio/high_mean": 0.003436053288169205,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003436053288169205,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1314.0,
      "completions/max_terminated_length": 623.0,
      "completions/mean_length": 293.59375,
      "completions/mean_terminated_length": 223.73077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5297418422996998,
      "epoch": 0.01338,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9773735404014587,
      "kl": 0.3717000503093004,
      "learning_rate": 3.7542847291579375e-06,
      "loss": -0.1143,
      "num_tokens": 27099213.0,
      "reward": 0.9006249904632568,
      "reward_std": 0.29919499158859253,
      "rewards/rollout_reward_func/mean": 0.9006249904632568,
      "rewards/rollout_reward_func/std": 0.42324352264404297,
      "sampling/importance_sampling_ratio/max": 1.6695058345794678,
      "sampling/importance_sampling_ratio/mean": 0.9677022695541382,
      "sampling/importance_sampling_ratio/min": 9.010791423278207e-27,
      "sampling/sampling_logp_difference/max": 27.038387298583984,
      "sampling/sampling_logp_difference/mean": 0.11482158303260803,
      "step": 669,
      "step_time": 16.098595812974963
    },
    {
      "clip_ratio/high_max": 0.011488970601931214,
      "clip_ratio/high_mean": 0.005744485300965607,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005744485300965607,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 670.0,
      "completions/max_terminated_length": 670.0,
      "completions/mean_length": 247.46875,
      "completions/mean_terminated_length": 218.95455932617188,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.46804756484925747,
      "epoch": 0.0134,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.1982195377349854,
      "kl": 0.3483984163030982,
      "learning_rate": 3.744135847846937e-06,
      "loss": 0.2075,
      "num_tokens": 27139558.0,
      "reward": 0.7712499499320984,
      "reward_std": 0.4193088114261627,
      "rewards/rollout_reward_func/mean": 0.7712499499320984,
      "rewards/rollout_reward_func/std": 0.4326344132423401,
      "sampling/importance_sampling_ratio/max": 2.6154391765594482,
      "sampling/importance_sampling_ratio/mean": 1.1388994455337524,
      "sampling/importance_sampling_ratio/min": 0.5805791020393372,
      "sampling/sampling_logp_difference/max": 0.668877124786377,
      "sampling/sampling_logp_difference/mean": 0.032165974378585815,
      "step": 670,
      "step_time": 12.327312931010965
    },
    {
      "clip_ratio/high_max": 0.006414473755285144,
      "clip_ratio/high_mean": 0.003207236877642572,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004183799377642572,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 1094.0,
      "completions/max_terminated_length": 1094.0,
      "completions/mean_length": 331.71875,
      "completions/mean_terminated_length": 267.6521911621094,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5264268536120653,
      "epoch": 0.01342,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.1322505474090576,
      "kl": 0.42106579057872295,
      "learning_rate": 3.7340200205780724e-06,
      "loss": 0.1955,
      "num_tokens": 27183604.0,
      "reward": 0.6121875047683716,
      "reward_std": 0.5019437074661255,
      "rewards/rollout_reward_func/mean": 0.6121875047683716,
      "rewards/rollout_reward_func/std": 0.4949990510940552,
      "sampling/importance_sampling_ratio/max": 1.6578325033187866,
      "sampling/importance_sampling_ratio/mean": 1.0783987045288086,
      "sampling/importance_sampling_ratio/min": 0.1807776242494583,
      "sampling/sampling_logp_difference/max": 0.8654961585998535,
      "sampling/sampling_logp_difference/mean": 0.03394868224859238,
      "step": 671,
      "step_time": 14.36100667595747
    },
    {
      "clip_ratio/high_max": 0.0024999999441206455,
      "clip_ratio/high_mean": 0.0012499999720603228,
      "clip_ratio/low_mean": 0.00390625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005156249972060323,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 682.0,
      "completions/max_terminated_length": 593.0,
      "completions/mean_length": 225.09375,
      "completions/mean_terminated_length": 127.13636779785156,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4742811368778348,
      "epoch": 0.01344,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5733973979949951,
      "kl": 0.5537325600162148,
      "learning_rate": 3.723937380785948e-06,
      "loss": 0.0195,
      "num_tokens": 27221882.0,
      "reward": 0.7646874785423279,
      "reward_std": 0.41911113262176514,
      "rewards/rollout_reward_func/mean": 0.7646874785423279,
      "rewards/rollout_reward_func/std": 0.5204958915710449,
      "sampling/importance_sampling_ratio/max": 1.5292872190475464,
      "sampling/importance_sampling_ratio/mean": 0.9984804391860962,
      "sampling/importance_sampling_ratio/min": 0.421293705701828,
      "sampling/sampling_logp_difference/max": 0.9445359706878662,
      "sampling/sampling_logp_difference/mean": 0.031505245715379715,
      "step": 672,
      "step_time": 12.74985065999499
    },
    {
      "clip_ratio/high_max": 0.022135416977107525,
      "clip_ratio/high_mean": 0.011067708488553762,
      "clip_ratio/low_mean": 0.0026400862261652946,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013707794714719057,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 670.0,
      "completions/max_terminated_length": 670.0,
      "completions/mean_length": 269.0,
      "completions/mean_terminated_length": 265.2592468261719,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.518554849550128,
      "epoch": 0.01346,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4273245334625244,
      "kl": 0.4056283035315573,
      "learning_rate": 3.7138880614673957e-06,
      "loss": 0.0228,
      "num_tokens": 27263742.0,
      "reward": 0.7381249666213989,
      "reward_std": 0.467901349067688,
      "rewards/rollout_reward_func/mean": 0.7381249666213989,
      "rewards/rollout_reward_func/std": 0.4643235504627228,
      "sampling/importance_sampling_ratio/max": 1.901292085647583,
      "sampling/importance_sampling_ratio/mean": 0.9940473437309265,
      "sampling/importance_sampling_ratio/min": 0.31946611404418945,
      "sampling/sampling_logp_difference/max": 0.662750244140625,
      "sampling/sampling_logp_difference/mean": 0.031633876264095306,
      "step": 673,
      "step_time": 12.539973849037779
    },
    {
      "clip_ratio/high_max": 0.015217391541227698,
      "clip_ratio/high_mean": 0.007608695770613849,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007608695770613849,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 638.0,
      "completions/max_terminated_length": 624.0,
      "completions/mean_length": 189.28125,
      "completions/mean_terminated_length": 175.23809814453125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4549552481621504,
      "epoch": 0.01348,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0069897174835205,
      "kl": 0.4328292468562722,
      "learning_rate": 3.7038721951797318e-06,
      "loss": 0.0875,
      "num_tokens": 27302319.0,
      "reward": 0.7659375071525574,
      "reward_std": 0.5676172971725464,
      "rewards/rollout_reward_func/mean": 0.7659375071525574,
      "rewards/rollout_reward_func/std": 0.5657244920730591,
      "sampling/importance_sampling_ratio/max": 1.596789002418518,
      "sampling/importance_sampling_ratio/mean": 1.0523571968078613,
      "sampling/importance_sampling_ratio/min": 0.529807448387146,
      "sampling/sampling_logp_difference/max": 0.43332386016845703,
      "sampling/sampling_logp_difference/mean": 0.02978689596056938,
      "step": 674,
      "step_time": 13.21398041403154
    },
    {
      "clip_ratio/high_max": 0.007378472248092294,
      "clip_ratio/high_mean": 0.003689236124046147,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003689236124046147,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 655.0,
      "completions/max_terminated_length": 588.0,
      "completions/mean_length": 196.28125,
      "completions/mean_terminated_length": 191.46429443359375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4134611068293452,
      "epoch": 0.0135,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2168489694595337,
      "kl": 0.8481378932483494,
      "learning_rate": 3.6938899140390016e-06,
      "loss": -0.0396,
      "num_tokens": 27339436.0,
      "reward": 0.8653124570846558,
      "reward_std": 0.361545205116272,
      "rewards/rollout_reward_func/mean": 0.8653124570846558,
      "rewards/rollout_reward_func/std": 0.43961846828460693,
      "sampling/importance_sampling_ratio/max": 1.519090175628662,
      "sampling/importance_sampling_ratio/mean": 1.008110523223877,
      "sampling/importance_sampling_ratio/min": 0.6804192066192627,
      "sampling/sampling_logp_difference/max": 0.43203604221343994,
      "sampling/sampling_logp_difference/mean": 0.028362471610307693,
      "step": 675,
      "step_time": 11.934711166017223
    },
    {
      "clip_ratio/high_max": 0.014787946827709675,
      "clip_ratio/high_mean": 0.007393973413854837,
      "clip_ratio/low_mean": 0.0033808479784056544,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010774821392260492,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 653.0,
      "completions/max_terminated_length": 596.0,
      "completions/mean_length": 192.09375,
      "completions/mean_terminated_length": 163.45834350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6084843017160892,
      "epoch": 0.01352,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9219491481781006,
      "kl": 0.5252900756895542,
      "learning_rate": 3.683941349718244e-06,
      "loss": -0.1066,
      "num_tokens": 27376574.0,
      "reward": 0.7640625238418579,
      "reward_std": 0.5182199478149414,
      "rewards/rollout_reward_func/mean": 0.7640625238418579,
      "rewards/rollout_reward_func/std": 0.5047634840011597,
      "sampling/importance_sampling_ratio/max": 1.4722607135772705,
      "sampling/importance_sampling_ratio/mean": 0.8949025869369507,
      "sampling/importance_sampling_ratio/min": 2.3566862166184838e-14,
      "sampling/sampling_logp_difference/max": 20.843347549438477,
      "sampling/sampling_logp_difference/mean": 0.15443146228790283,
      "step": 676,
      "step_time": 12.453115561016602
    },
    {
      "clip_ratio/high_max": 0.008576766354963183,
      "clip_ratio/high_mean": 0.004288383177481592,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004288383177481592,
      "completions/clipped_ratio": 0.09375,
      "completions/max_length": 482.0,
      "completions/max_terminated_length": 482.0,
      "completions/mean_length": 137.6875,
      "completions/mean_terminated_length": 143.44827270507812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.43279262259602547,
      "epoch": 0.01354,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2699947357177734,
      "kl": 0.3922168184071779,
      "learning_rate": 3.674026633445747e-06,
      "loss": -0.2132,
      "num_tokens": 27413557.0,
      "reward": 0.7665624618530273,
      "reward_std": 0.533150315284729,
      "rewards/rollout_reward_func/mean": 0.7665624618530273,
      "rewards/rollout_reward_func/std": 0.6267355680465698,
      "sampling/importance_sampling_ratio/max": 1.8686803579330444,
      "sampling/importance_sampling_ratio/mean": 1.1104614734649658,
      "sampling/importance_sampling_ratio/min": 0.5654491186141968,
      "sampling/sampling_logp_difference/max": 0.5518884658813477,
      "sampling/sampling_logp_difference/mean": 0.02844529040157795,
      "step": 677,
      "step_time": 12.333497382991482
    },
    {
      "clip_ratio/high_max": 0.011424366151914,
      "clip_ratio/high_mean": 0.005712183075957,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006688745575957,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 660.0,
      "completions/mean_length": 277.96875,
      "completions/mean_terminated_length": 268.9599914550781,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5011369655840099,
      "epoch": 0.01356,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0158485174179077,
      "kl": 0.3377501736395061,
      "learning_rate": 3.6641458960033194e-06,
      "loss": -0.0814,
      "num_tokens": 27455648.0,
      "reward": 0.7056249976158142,
      "reward_std": 0.6168807744979858,
      "rewards/rollout_reward_func/mean": 0.7056249976158142,
      "rewards/rollout_reward_func/std": 0.6063318252563477,
      "sampling/importance_sampling_ratio/max": 1.3613700866699219,
      "sampling/importance_sampling_ratio/mean": 0.8345322608947754,
      "sampling/importance_sampling_ratio/min": 2.2628147727972447e-13,
      "sampling/sampling_logp_difference/max": 21.95026969909668,
      "sampling/sampling_logp_difference/mean": 0.17358440160751343,
      "step": 678,
      "step_time": 12.415569677003077
    },
    {
      "clip_ratio/high_max": 0.008938456187024713,
      "clip_ratio/high_mean": 0.004469228093512356,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005445790593512356,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 655.0,
      "completions/max_terminated_length": 572.0,
      "completions/mean_length": 191.84375,
      "completions/mean_terminated_length": 176.60870361328125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5371397398412228,
      "epoch": 0.01358,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.286537528038025,
      "kl": 0.6965162185952067,
      "learning_rate": 3.654299267724567e-06,
      "loss": -0.0056,
      "num_tokens": 27493088.0,
      "reward": 0.824999988079071,
      "reward_std": 0.3962734043598175,
      "rewards/rollout_reward_func/mean": 0.824999988079071,
      "rewards/rollout_reward_func/std": 0.462838351726532,
      "sampling/importance_sampling_ratio/max": 1.3642781972885132,
      "sampling/importance_sampling_ratio/mean": 0.8105850219726562,
      "sampling/importance_sampling_ratio/min": 9.958455575398448e-26,
      "sampling/sampling_logp_difference/max": 25.507827758789062,
      "sampling/sampling_logp_difference/mean": 0.42080429196357727,
      "step": 679,
      "step_time": 12.583426392971887
    },
    {
      "clip_ratio/high_max": 0.004464285913854837,
      "clip_ratio/high_mean": 0.0022321429569274187,
      "clip_ratio/low_mean": 0.017187500023283064,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019419642980210483,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 676.0,
      "completions/max_terminated_length": 676.0,
      "completions/mean_length": 246.0,
      "completions/mean_terminated_length": 260.23809814453125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49337851628661156,
      "epoch": 0.0136,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2520194053649902,
      "kl": 2.0818214919418097,
      "learning_rate": 3.6444868784931752e-06,
      "loss": -0.0291,
      "num_tokens": 27534332.0,
      "reward": 0.6485146284103394,
      "reward_std": 0.500841498374939,
      "rewards/rollout_reward_func/mean": 0.6485146284103394,
      "rewards/rollout_reward_func/std": 0.6688398122787476,
      "sampling/importance_sampling_ratio/max": 1.326898455619812,
      "sampling/importance_sampling_ratio/mean": 0.8623232841491699,
      "sampling/importance_sampling_ratio/min": 1.3697460524256146e-29,
      "sampling/sampling_logp_difference/max": 27.593505859375,
      "sampling/sampling_logp_difference/mean": 0.1749143898487091,
      "step": 680,
      "step_time": 13.733976393996272
    },
    {
      "clip_ratio/high_max": 0.016406250186264515,
      "clip_ratio/high_mean": 0.008203125093132257,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009179687593132257,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 667.0,
      "completions/mean_length": 188.15625,
      "completions/mean_terminated_length": 175.87998962402344,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.3818716360256076,
      "epoch": 0.01362,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0217868089675903,
      "kl": 0.5394688230007887,
      "learning_rate": 3.6347088577411906e-06,
      "loss": -0.0728,
      "num_tokens": 27572565.0,
      "reward": 0.3934375047683716,
      "reward_std": 0.7316640019416809,
      "rewards/rollout_reward_func/mean": 0.3934375047683716,
      "rewards/rollout_reward_func/std": 0.7503616809844971,
      "sampling/importance_sampling_ratio/max": 1.5750261545181274,
      "sampling/importance_sampling_ratio/mean": 0.9558899402618408,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.058936357498169,
      "sampling/sampling_logp_difference/mean": 0.03458136320114136,
      "step": 681,
      "step_time": 12.15077731301426
    },
    {
      "clip_ratio/high_max": 0.018071725266054273,
      "clip_ratio/high_mean": 0.009035862633027136,
      "clip_ratio/low_mean": 0.0019846269860863686,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011020489619113505,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 670.0,
      "completions/mean_length": 269.46875,
      "completions/mean_terminated_length": 206.09524536132812,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5746014304459095,
      "epoch": 0.01364,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0862904787063599,
      "kl": 0.7939167600125074,
      "learning_rate": 3.6249653344473162e-06,
      "loss": -0.0098,
      "num_tokens": 27614539.0,
      "reward": 0.6102237105369568,
      "reward_std": 0.6448776721954346,
      "rewards/rollout_reward_func/mean": 0.6102237105369568,
      "rewards/rollout_reward_func/std": 0.6788160800933838,
      "sampling/importance_sampling_ratio/max": 1.7163156270980835,
      "sampling/importance_sampling_ratio/mean": 0.8272701501846313,
      "sampling/importance_sampling_ratio/min": 2.205544127340497e-29,
      "sampling/sampling_logp_difference/max": 27.173799514770508,
      "sampling/sampling_logp_difference/mean": 0.24853375554084778,
      "step": 682,
      "step_time": 13.472708870001952
    },
    {
      "clip_ratio/high_max": 0.0166930858977139,
      "clip_ratio/high_mean": 0.00834654294885695,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00932310544885695,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 676.0,
      "completions/max_terminated_length": 676.0,
      "completions/mean_length": 292.03125,
      "completions/mean_terminated_length": 253.45834350585938,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5411045271903276,
      "epoch": 0.01366,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3332477807998657,
      "kl": 0.5921177798882127,
      "learning_rate": 3.61525643713521e-06,
      "loss": 0.0057,
      "num_tokens": 27656907.0,
      "reward": 0.7074999809265137,
      "reward_std": 0.5498466491699219,
      "rewards/rollout_reward_func/mean": 0.7074999809265137,
      "rewards/rollout_reward_func/std": 0.6064119338989258,
      "sampling/importance_sampling_ratio/max": 1.4456753730773926,
      "sampling/importance_sampling_ratio/mean": 0.9490648508071899,
      "sampling/importance_sampling_ratio/min": 0.46714961528778076,
      "sampling/sampling_logp_difference/max": 0.5098052024841309,
      "sampling/sampling_logp_difference/mean": 0.03455250710248947,
      "step": 683,
      "step_time": 13.472316951010725
    },
    {
      "clip_ratio/high_max": 0.012483016354963183,
      "clip_ratio/high_mean": 0.006241508177481592,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007218070677481592,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 611.0,
      "completions/max_terminated_length": 611.0,
      "completions/mean_length": 216.90625,
      "completions/mean_terminated_length": 228.86956787109375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5053418148308992,
      "epoch": 0.01368,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.63044273853302,
      "kl": 0.41368041560053825,
      "learning_rate": 3.6055822938717933e-06,
      "loss": 0.0066,
      "num_tokens": 27695883.0,
      "reward": 0.7667862772941589,
      "reward_std": 0.44930922985076904,
      "rewards/rollout_reward_func/mean": 0.7667862772941589,
      "rewards/rollout_reward_func/std": 0.5814555883407593,
      "sampling/importance_sampling_ratio/max": 1.6242555379867554,
      "sampling/importance_sampling_ratio/mean": 0.9994926452636719,
      "sampling/importance_sampling_ratio/min": 0.3740607500076294,
      "sampling/sampling_logp_difference/max": 0.9752464294433594,
      "sampling/sampling_logp_difference/mean": 0.03171507641673088,
      "step": 684,
      "step_time": 12.474892192039988
    },
    {
      "clip_ratio/high_max": 0.023153409361839294,
      "clip_ratio/high_mean": 0.011576704680919647,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011576704680919647,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 662.0,
      "completions/mean_length": 291.03125,
      "completions/mean_terminated_length": 281.60870361328125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5464327912777662,
      "epoch": 0.0137,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.237144112586975,
      "kl": 0.3830048032104969,
      "learning_rate": 3.595943032265554e-06,
      "loss": -0.0283,
      "num_tokens": 27738533.0,
      "reward": 0.6184779405593872,
      "reward_std": 0.6779525876045227,
      "rewards/rollout_reward_func/mean": 0.6184779405593872,
      "rewards/rollout_reward_func/std": 0.6715327501296997,
      "sampling/importance_sampling_ratio/max": 1.3606067895889282,
      "sampling/importance_sampling_ratio/mean": 0.8792416453361511,
      "sampling/importance_sampling_ratio/min": 5.471835762271837e-10,
      "sampling/sampling_logp_difference/max": 8.77779769897461,
      "sampling/sampling_logp_difference/mean": 0.0695377066731453,
      "step": 685,
      "step_time": 13.160346784003195
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 649.0,
      "completions/max_terminated_length": 649.0,
      "completions/mean_length": 226.0,
      "completions/mean_terminated_length": 212.30435180664062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5227221827954054,
      "epoch": 0.01372,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6400986909866333,
      "kl": 1.6521658673882484,
      "learning_rate": 3.5863387794648695e-06,
      "loss": -0.0013,
      "num_tokens": 27777977.0,
      "reward": 0.8971874713897705,
      "reward_std": 0.35660403966903687,
      "rewards/rollout_reward_func/mean": 0.8971874713897705,
      "rewards/rollout_reward_func/std": 0.4198865592479706,
      "sampling/importance_sampling_ratio/max": 2.262397050857544,
      "sampling/importance_sampling_ratio/mean": 1.0427626371383667,
      "sampling/importance_sampling_ratio/min": 0.3203648030757904,
      "sampling/sampling_logp_difference/max": 0.7993316650390625,
      "sampling/sampling_logp_difference/mean": 0.035057730972766876,
      "step": 686,
      "step_time": 12.638318994009751
    },
    {
      "clip_ratio/high_max": 0.0034722222480922937,
      "clip_ratio/high_mean": 0.0017361111240461469,
      "clip_ratio/low_mean": 0.005521616665646434,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007257727789692581,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1141.0,
      "completions/max_terminated_length": 1141.0,
      "completions/mean_length": 326.09375,
      "completions/mean_terminated_length": 308.04168701171875,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47202583122998476,
      "epoch": 0.01374,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2009005546569824,
      "kl": 0.5281374407932162,
      "learning_rate": 3.5767696621563247e-06,
      "loss": -0.1644,
      "num_tokens": 27821352.0,
      "reward": 0.6200023889541626,
      "reward_std": 0.7366254329681396,
      "rewards/rollout_reward_func/mean": 0.6200023889541626,
      "rewards/rollout_reward_func/std": 0.7124864459037781,
      "sampling/importance_sampling_ratio/max": 1.7755779027938843,
      "sampling/importance_sampling_ratio/mean": 1.001570463180542,
      "sampling/importance_sampling_ratio/min": 0.44619283080101013,
      "sampling/sampling_logp_difference/max": 0.8448967933654785,
      "sampling/sampling_logp_difference/mean": 0.030551016330718994,
      "step": 687,
      "step_time": 14.877704153012019
    },
    {
      "clip_ratio/high_max": 0.006310096243396401,
      "clip_ratio/high_mean": 0.0031550481216982007,
      "clip_ratio/low_mean": 0.0011160714784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00427111960016191,
      "completions/clipped_ratio": 0.40625,
      "completions/max_length": 670.0,
      "completions/max_terminated_length": 670.0,
      "completions/mean_length": 283.03125,
      "completions/mean_terminated_length": 227.26315307617188,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5604637553915381,
      "epoch": 0.01376,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.499671459197998,
      "kl": 0.34088035207241774,
      "learning_rate": 3.5672358065630486e-06,
      "loss": 0.2121,
      "num_tokens": 27864161.0,
      "reward": 0.6834374666213989,
      "reward_std": 0.548515796661377,
      "rewards/rollout_reward_func/mean": 0.6834374666213989,
      "rewards/rollout_reward_func/std": 0.5446786880493164,
      "sampling/importance_sampling_ratio/max": 2.093287944793701,
      "sampling/importance_sampling_ratio/mean": 1.0842962265014648,
      "sampling/importance_sampling_ratio/min": 2.9321458914079306e-20,
      "sampling/sampling_logp_difference/max": 17.879467010498047,
      "sampling/sampling_logp_difference/mean": 0.10572520643472672,
      "step": 688,
      "step_time": 13.264305329983472
    },
    {
      "clip_ratio/high_max": 0.030032483395189047,
      "clip_ratio/high_mean": 0.016969366697594523,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016969366697594523,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 997.0,
      "completions/max_terminated_length": 997.0,
      "completions/mean_length": 263.09375,
      "completions/mean_terminated_length": 220.1599884033203,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49489172268658876,
      "epoch": 0.01378,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9863717555999756,
      "kl": 0.7824133904650807,
      "learning_rate": 3.5577373384430397e-06,
      "loss": -0.2031,
      "num_tokens": 27906620.0,
      "reward": 0.6828124523162842,
      "reward_std": 0.6120439171791077,
      "rewards/rollout_reward_func/mean": 0.6828124523162842,
      "rewards/rollout_reward_func/std": 0.7017362713813782,
      "sampling/importance_sampling_ratio/max": 1.3548458814620972,
      "sampling/importance_sampling_ratio/mean": 0.8752695322036743,
      "sampling/importance_sampling_ratio/min": 1.0892919181921198e-18,
      "sampling/sampling_logp_difference/max": 23.777318954467773,
      "sampling/sampling_logp_difference/mean": 0.10476773977279663,
      "step": 689,
      "step_time": 14.176198363988078
    },
    {
      "clip_ratio/high_max": 0.004464285913854837,
      "clip_ratio/high_mean": 0.0029114907374605536,
      "clip_ratio/low_mean": 0.0011160714784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004027562215924263,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 834.0,
      "completions/max_terminated_length": 585.0,
      "completions/mean_length": 269.09375,
      "completions/mean_terminated_length": 191.6999969482422,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5498644672334194,
      "epoch": 0.0138,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8488774299621582,
      "kl": 0.40423559583723545,
      "learning_rate": 3.5482743830875137e-06,
      "loss": 0.0544,
      "num_tokens": 27948122.0,
      "reward": 0.5449999570846558,
      "reward_std": 0.672865629196167,
      "rewards/rollout_reward_func/mean": 0.5449999570846558,
      "rewards/rollout_reward_func/std": 0.6676294207572937,
      "sampling/importance_sampling_ratio/max": 2.7151613235473633,
      "sampling/importance_sampling_ratio/mean": 1.1335093975067139,
      "sampling/importance_sampling_ratio/min": 3.334491571749254e-09,
      "sampling/sampling_logp_difference/max": 19.416913986206055,
      "sampling/sampling_logp_difference/mean": 0.07013338059186935,
      "step": 690,
      "step_time": 12.909609567010193
    },
    {
      "clip_ratio/high_max": 0.047656250186264515,
      "clip_ratio/high_mean": 0.023828125093132257,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.025781250093132257,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 664.0,
      "completions/max_terminated_length": 664.0,
      "completions/mean_length": 217.40625,
      "completions/mean_terminated_length": 177.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49482215754687786,
      "epoch": 0.01382,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6679697036743164,
      "kl": 0.6092204544693232,
      "learning_rate": 3.538847065319248e-06,
      "loss": 0.125,
      "num_tokens": 27987110.0,
      "reward": 0.734375,
      "reward_std": 0.5412089824676514,
      "rewards/rollout_reward_func/mean": 0.734375,
      "rewards/rollout_reward_func/std": 0.5865422487258911,
      "sampling/importance_sampling_ratio/max": 1.9684768915176392,
      "sampling/importance_sampling_ratio/mean": 0.9473801851272583,
      "sampling/importance_sampling_ratio/min": 3.983630645406494e-13,
      "sampling/sampling_logp_difference/max": 20.460065841674805,
      "sampling/sampling_logp_difference/mean": 0.08441462367773056,
      "step": 691,
      "step_time": 12.90805984500912
    },
    {
      "clip_ratio/high_max": 0.027893775841221213,
      "clip_ratio/high_mean": 0.013946887920610607,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013946887920610607,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 622.0,
      "completions/mean_length": 224.34375,
      "completions/mean_terminated_length": 175.9091033935547,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5604528114199638,
      "epoch": 0.01384,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8761862516403198,
      "kl": 0.28787032049149275,
      "learning_rate": 3.5294555094909387e-06,
      "loss": 0.0603,
      "num_tokens": 28026180.0,
      "reward": 0.8249999284744263,
      "reward_std": 0.4496607780456543,
      "rewards/rollout_reward_func/mean": 0.8249999284744263,
      "rewards/rollout_reward_func/std": 0.5297534465789795,
      "sampling/importance_sampling_ratio/max": 1.2965080738067627,
      "sampling/importance_sampling_ratio/mean": 0.8629893064498901,
      "sampling/importance_sampling_ratio/min": 1.1015074198278598e-30,
      "sampling/sampling_logp_difference/max": 20.600650787353516,
      "sampling/sampling_logp_difference/mean": 0.25915560126304626,
      "step": 692,
      "step_time": 13.005226226014202
    },
    {
      "clip_ratio/high_max": 0.004036458441987634,
      "clip_ratio/high_mean": 0.002018229220993817,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004622395965270698,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 1152.0,
      "completions/max_terminated_length": 677.0,
      "completions/mean_length": 303.28125,
      "completions/mean_terminated_length": 267.6666564941406,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 0.5926210321485996,
      "epoch": 0.01386,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.75511634349823,
      "kl": 0.5918595511466265,
      "learning_rate": 3.520099839483554e-06,
      "loss": -0.1355,
      "num_tokens": 28069908.0,
      "reward": 0.7740625143051147,
      "reward_std": 0.4433564841747284,
      "rewards/rollout_reward_func/mean": 0.7740625143051147,
      "rewards/rollout_reward_func/std": 0.4351852238178253,
      "sampling/importance_sampling_ratio/max": 1.896841049194336,
      "sampling/importance_sampling_ratio/mean": 1.0099629163742065,
      "sampling/importance_sampling_ratio/min": 8.039853409263617e-24,
      "sampling/sampling_logp_difference/max": 24.214284896850586,
      "sampling/sampling_logp_difference/mean": 0.17542609572410583,
      "step": 693,
      "step_time": 14.684128053020686
    },
    {
      "clip_ratio/high_max": 0.009865900501608849,
      "clip_ratio/high_mean": 0.004932950250804424,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020557950250804424,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 675.0,
      "completions/max_terminated_length": 675.0,
      "completions/mean_length": 295.625,
      "completions/mean_terminated_length": 301.8800048828125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4781248045619577,
      "epoch": 0.01388,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2633920907974243,
      "kl": 0.3091589701361954,
      "learning_rate": 3.5107801787047088e-06,
      "loss": -0.1641,
      "num_tokens": 28114230.0,
      "reward": 0.5575000047683716,
      "reward_std": 0.758021891117096,
      "rewards/rollout_reward_func/mean": 0.5575000047683716,
      "rewards/rollout_reward_func/std": 0.7405838966369629,
      "sampling/importance_sampling_ratio/max": 1.678185224533081,
      "sampling/importance_sampling_ratio/mean": 1.081213355064392,
      "sampling/importance_sampling_ratio/min": 0.49496015906333923,
      "sampling/sampling_logp_difference/max": 0.443789005279541,
      "sampling/sampling_logp_difference/mean": 0.025725210085511208,
      "step": 694,
      "step_time": 12.904670552015887
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.004036458441987634,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005013020941987634,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 591.0,
      "completions/mean_length": 230.3125,
      "completions/mean_terminated_length": 188.1904754638672,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.5571547718718648,
      "epoch": 0.0139,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.3252267837524414,
      "kl": 0.864398236386478,
      "learning_rate": 3.5014966500870244e-06,
      "loss": 0.1819,
      "num_tokens": 28154228.0,
      "reward": 0.6699999570846558,
      "reward_std": 0.6348837018013,
      "rewards/rollout_reward_func/mean": 0.6699999570846558,
      "rewards/rollout_reward_func/std": 0.616142988204956,
      "sampling/importance_sampling_ratio/max": 1.6645225286483765,
      "sampling/importance_sampling_ratio/mean": 0.8970004916191101,
      "sampling/importance_sampling_ratio/min": 7.436758992030385e-23,
      "sampling/sampling_logp_difference/max": 25.37287139892578,
      "sampling/sampling_logp_difference/mean": 0.21976956725120544,
      "step": 695,
      "step_time": 12.950652930972865
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 1132.0,
      "completions/max_terminated_length": 1132.0,
      "completions/mean_length": 301.84375,
      "completions/mean_terminated_length": 266.5,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.6298695355653763,
      "epoch": 0.01392,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9571341276168823,
      "kl": 0.40394139662384987,
      "learning_rate": 3.492249376086523e-06,
      "loss": 0.1151,
      "num_tokens": 28196702.0,
      "reward": 0.8324999809265137,
      "reward_std": 0.3497332036495209,
      "rewards/rollout_reward_func/mean": 0.8324999809265137,
      "rewards/rollout_reward_func/std": 0.39456793665885925,
      "sampling/importance_sampling_ratio/max": 1.608758807182312,
      "sampling/importance_sampling_ratio/mean": 0.9666367769241333,
      "sampling/importance_sampling_ratio/min": 1.970451235593061e-19,
      "sampling/sampling_logp_difference/max": 24.35404396057129,
      "sampling/sampling_logp_difference/mean": 0.1074635460972786,
      "step": 696,
      "step_time": 14.08946482600004
    },
    {
      "clip_ratio/high_max": 0.00801454740576446,
      "clip_ratio/high_mean": 0.00400727370288223,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00400727370288223,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1157.0,
      "completions/max_terminated_length": 1157.0,
      "completions/mean_length": 293.0625,
      "completions/mean_terminated_length": 346.8846435546875,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47503141313791275,
      "epoch": 0.01394,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.190507173538208,
      "kl": 0.3305091978982091,
      "learning_rate": 3.483038478680996e-06,
      "loss": -0.1823,
      "num_tokens": 28239933.0,
      "reward": 0.8349999785423279,
      "reward_std": 0.4011285901069641,
      "rewards/rollout_reward_func/mean": 0.8349999785423279,
      "rewards/rollout_reward_func/std": 0.47750020027160645,
      "sampling/importance_sampling_ratio/max": 1.6839138269424438,
      "sampling/importance_sampling_ratio/mean": 1.0241940021514893,
      "sampling/importance_sampling_ratio/min": 9.126115939829244e-29,
      "sampling/sampling_logp_difference/max": 26.296951293945312,
      "sampling/sampling_logp_difference/mean": 0.24743101000785828,
      "step": 697,
      "step_time": 15.648987195992959
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 640.0,
      "completions/max_terminated_length": 640.0,
      "completions/mean_length": 253.59375,
      "completions/mean_terminated_length": 229.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5320828082039952,
      "epoch": 0.01396,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9302533268928528,
      "kl": 0.34506333712488413,
      "learning_rate": 3.4738640793684103e-06,
      "loss": -0.058,
      "num_tokens": 28280457.0,
      "reward": 0.7068749666213989,
      "reward_std": 0.457611620426178,
      "rewards/rollout_reward_func/mean": 0.7068749666213989,
      "rewards/rollout_reward_func/std": 0.46235328912734985,
      "sampling/importance_sampling_ratio/max": 1.2598010301589966,
      "sampling/importance_sampling_ratio/mean": 0.8252662420272827,
      "sampling/importance_sampling_ratio/min": 9.345584854270619e-25,
      "sampling/sampling_logp_difference/max": 27.40046501159668,
      "sampling/sampling_logp_difference/mean": 0.39684000611305237,
      "step": 698,
      "step_time": 12.43025702299201
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0062500000931322575,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0072265625931322575,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 633.0,
      "completions/mean_length": 286.40625,
      "completions/mean_terminated_length": 287.66668701171875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5043802186846733,
      "epoch": 0.01398,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.1052098274230957,
      "kl": 0.3346577435731888,
      "learning_rate": 3.464726299165293e-06,
      "loss": -0.2069,
      "num_tokens": 28323450.0,
      "reward": 0.9249999523162842,
      "reward_std": 0.2198353409767151,
      "rewards/rollout_reward_func/mean": 0.9249999523162842,
      "rewards/rollout_reward_func/std": 0.29243138432502747,
      "sampling/importance_sampling_ratio/max": 1.7296653985977173,
      "sampling/importance_sampling_ratio/mean": 0.9801712036132812,
      "sampling/importance_sampling_ratio/min": 1.460914229572862e-23,
      "sampling/sampling_logp_difference/max": 26.230802536010742,
      "sampling/sampling_logp_difference/mean": 0.18975161015987396,
      "step": 699,
      "step_time": 13.275816311972449
    },
    {
      "clip_ratio/high_max": 0.012369791744276881,
      "clip_ratio/high_mean": 0.006184895872138441,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006184895872138441,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 640.0,
      "completions/max_terminated_length": 640.0,
      "completions/mean_length": 269.1875,
      "completions/mean_terminated_length": 240.43478393554688,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5493016978725791,
      "epoch": 0.014,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2251770496368408,
      "kl": 0.3709989641793072,
      "learning_rate": 3.4556252586051436e-06,
      "loss": -0.1197,
      "num_tokens": 28366333.0,
      "reward": 0.8093749284744263,
      "reward_std": 0.4964429438114166,
      "rewards/rollout_reward_func/mean": 0.8093749284744263,
      "rewards/rollout_reward_func/std": 0.49013447761535645,
      "sampling/importance_sampling_ratio/max": 1.6920177936553955,
      "sampling/importance_sampling_ratio/mean": 0.9456868767738342,
      "sampling/importance_sampling_ratio/min": 1.2538762088108238e-27,
      "sampling/sampling_logp_difference/max": 27.085458755493164,
      "sampling/sampling_logp_difference/mean": 0.2695414423942566,
      "step": 700,
      "step_time": 13.596144855982857
    },
    {
      "clip_ratio/high_max": 0.0076772188767790794,
      "clip_ratio/high_mean": 0.0038386094383895397,
      "clip_ratio/low_mean": 0.00329455710016191,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00713316653855145,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 304.875,
      "completions/mean_terminated_length": 281.25,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5698796473443508,
      "epoch": 0.01402,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.185355305671692,
      "kl": 0.5797918774187565,
      "learning_rate": 3.4465610777368364e-06,
      "loss": 0.1352,
      "num_tokens": 28409990.0,
      "reward": 0.609375,
      "reward_std": 0.5079530477523804,
      "rewards/rollout_reward_func/mean": 0.609375,
      "rewards/rollout_reward_func/std": 0.5460233688354492,
      "sampling/importance_sampling_ratio/max": 2.211724042892456,
      "sampling/importance_sampling_ratio/mean": 0.9663984775543213,
      "sampling/importance_sampling_ratio/min": 4.36341711115361e-22,
      "sampling/sampling_logp_difference/max": 24.965057373046875,
      "sampling/sampling_logp_difference/mean": 0.1163058653473854,
      "step": 701,
      "step_time": 12.829013890994247
    },
    {
      "clip_ratio/high_max": 0.017561141401529312,
      "clip_ratio/high_mean": 0.008780570700764656,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008780570700764656,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1130.0,
      "completions/max_terminated_length": 1130.0,
      "completions/mean_length": 242.5625,
      "completions/mean_terminated_length": 246.5416717529297,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.4773689853027463,
      "epoch": 0.01404,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8900770545005798,
      "kl": 0.3306396808475256,
      "learning_rate": 3.4375338761230474e-06,
      "loss": -0.0306,
      "num_tokens": 28450493.0,
      "reward": 0.8062499761581421,
      "reward_std": 0.4181758165359497,
      "rewards/rollout_reward_func/mean": 0.8062499761581421,
      "rewards/rollout_reward_func/std": 0.4187365770339966,
      "sampling/importance_sampling_ratio/max": 1.8619780540466309,
      "sampling/importance_sampling_ratio/mean": 0.9250959157943726,
      "sampling/importance_sampling_ratio/min": 1.858753324597128e-14,
      "sampling/sampling_logp_difference/max": 23.936220169067383,
      "sampling/sampling_logp_difference/mean": 0.08511929214000702,
      "step": 702,
      "step_time": 14.430027512018569
    },
    {
      "clip_ratio/high_max": 0.019053819589316845,
      "clip_ratio/high_mean": 0.010503472061827779,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010503472061827779,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 585.0,
      "completions/max_terminated_length": 585.0,
      "completions/mean_length": 240.21875,
      "completions/mean_terminated_length": 200.33334350585938,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47737657744437456,
      "epoch": 0.01406,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4472237825393677,
      "kl": 0.29936269856989384,
      "learning_rate": 3.428543772838667e-06,
      "loss": -0.1427,
      "num_tokens": 28490882.0,
      "reward": 0.9018748998641968,
      "reward_std": 0.28255411982536316,
      "rewards/rollout_reward_func/mean": 0.9018748998641968,
      "rewards/rollout_reward_func/std": 0.417870432138443,
      "sampling/importance_sampling_ratio/max": 1.7584800720214844,
      "sampling/importance_sampling_ratio/mean": 1.007722020149231,
      "sampling/importance_sampling_ratio/min": 1.942769119406091e-18,
      "sampling/sampling_logp_difference/max": 24.963802337646484,
      "sampling/sampling_logp_difference/mean": 0.17740777134895325,
      "step": 703,
      "step_time": 13.246284633976757
    },
    {
      "clip_ratio/high_max": 0.0031250000465661287,
      "clip_ratio/high_mean": 0.0015625000232830644,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015625000232830644,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 590.0,
      "completions/max_terminated_length": 586.0,
      "completions/mean_length": 178.59375,
      "completions/mean_terminated_length": 166.16000366210938,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.47368170507252216,
      "epoch": 0.01408,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7823237180709839,
      "kl": 0.7438545096665621,
      "learning_rate": 3.4195908864692364e-06,
      "loss": -0.1466,
      "num_tokens": 28528213.0,
      "reward": 0.7634375095367432,
      "reward_std": 0.5683258771896362,
      "rewards/rollout_reward_func/mean": 0.7634375095367432,
      "rewards/rollout_reward_func/std": 0.5714758038520813,
      "sampling/importance_sampling_ratio/max": 1.6100714206695557,
      "sampling/importance_sampling_ratio/mean": 0.8542810678482056,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 21.11115837097168,
      "sampling/sampling_logp_difference/mean": 0.1800258904695511,
      "step": 704,
      "step_time": 12.29397098798654
    },
    {
      "clip_ratio/high_max": 0.0044531249441206455,
      "clip_ratio/high_mean": 0.0022265624720603228,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00406479777302593,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 648.0,
      "completions/max_terminated_length": 586.0,
      "completions/mean_length": 269.4375,
      "completions/mean_terminated_length": 223.87998962402344,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.46978864166885614,
      "epoch": 0.0141,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5469540357589722,
      "kl": 0.676227318122983,
      "learning_rate": 3.410675335109376e-06,
      "loss": 0.0283,
      "num_tokens": 28569536.0,
      "reward": 0.7412499785423279,
      "reward_std": 0.5235478281974792,
      "rewards/rollout_reward_func/mean": 0.7412499785423279,
      "rewards/rollout_reward_func/std": 0.5348424315452576,
      "sampling/importance_sampling_ratio/max": 1.581871747970581,
      "sampling/importance_sampling_ratio/mean": 0.9792987108230591,
      "sampling/importance_sampling_ratio/min": 0.4092930853366852,
      "sampling/sampling_logp_difference/max": 0.9078044891357422,
      "sampling/sampling_logp_difference/mean": 0.032250553369522095,
      "step": 705,
      "step_time": 13.228938123982516
    },
    {
      "clip_ratio/high_max": 0.02151864068582654,
      "clip_ratio/high_mean": 0.01075932034291327,
      "clip_ratio/low_mean": 0.011393229477107525,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022152549820020795,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 611.0,
      "completions/max_terminated_length": 611.0,
      "completions/mean_length": 226.90625,
      "completions/mean_terminated_length": 221.7391357421875,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.5264049842953682,
      "epoch": 0.01412,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1826257705688477,
      "kl": 0.6694169184193015,
      "learning_rate": 3.4017972363612374e-06,
      "loss": -0.1804,
      "num_tokens": 28609397.0,
      "reward": 0.8693749904632568,
      "reward_std": 0.3855917155742645,
      "rewards/rollout_reward_func/mean": 0.8693749904632568,
      "rewards/rollout_reward_func/std": 0.4489409327507019,
      "sampling/importance_sampling_ratio/max": 1.936043381690979,
      "sampling/importance_sampling_ratio/mean": 1.007035255432129,
      "sampling/importance_sampling_ratio/min": 1.7099869753027614e-26,
      "sampling/sampling_logp_difference/max": 19.480743408203125,
      "sampling/sampling_logp_difference/mean": 0.1274818331003189,
      "step": 706,
      "step_time": 12.943529020994902
    },
    {
      "clip_ratio/high_max": 0.0036764706019312143,
      "clip_ratio/high_mean": 0.0018382353009656072,
      "clip_ratio/low_mean": 0.0030691964784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0049074317794293165,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 643.0,
      "completions/max_terminated_length": 643.0,
      "completions/mean_length": 201.1875,
      "completions/mean_terminated_length": 167.7391357421875,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.4954536743462086,
      "epoch": 0.01414,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.041111469268799,
      "kl": 0.4324188716709614,
      "learning_rate": 3.392956707332945e-06,
      "loss": 0.1529,
      "num_tokens": 28647559.0,
      "reward": 0.6078125238418579,
      "reward_std": 0.5483777523040771,
      "rewards/rollout_reward_func/mean": 0.6078125238418579,
      "rewards/rollout_reward_func/std": 0.6063253879547119,
      "sampling/importance_sampling_ratio/max": 2.691195011138916,
      "sampling/importance_sampling_ratio/mean": 1.0759475231170654,
      "sampling/importance_sampling_ratio/min": 4.0700345316224684e-09,
      "sampling/sampling_logp_difference/max": 19.61903190612793,
      "sampling/sampling_logp_difference/mean": 0.10086800903081894,
      "step": 707,
      "step_time": 12.042882207984803
    },
    {
      "clip_ratio/high_max": 0.010512907756492496,
      "clip_ratio/high_mean": 0.005256453878246248,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005256453878246248,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 655.0,
      "completions/max_terminated_length": 655.0,
      "completions/mean_length": 282.28125,
      "completions/mean_terminated_length": 242.53846740722656,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4829971119761467,
      "epoch": 0.01416,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9921626448631287,
      "kl": 0.46329403202980757,
      "learning_rate": 3.384153864637052e-06,
      "loss": -0.1526,
      "num_tokens": 28690335.0,
      "reward": 0.7775000333786011,
      "reward_std": 0.4419880509376526,
      "rewards/rollout_reward_func/mean": 0.7775000333786011,
      "rewards/rollout_reward_func/std": 0.5122184157371521,
      "sampling/importance_sampling_ratio/max": 1.5431698560714722,
      "sampling/importance_sampling_ratio/mean": 0.9173480272293091,
      "sampling/importance_sampling_ratio/min": 1.6913253538585394e-23,
      "sampling/sampling_logp_difference/max": 24.932493209838867,
      "sampling/sampling_logp_difference/mean": 0.15442444384098053,
      "step": 708,
      "step_time": 13.045117838992155
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 670.0,
      "completions/max_terminated_length": 670.0,
      "completions/mean_length": 263.84375,
      "completions/mean_terminated_length": 249.6666717529297,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47990472987294197,
      "epoch": 0.01418,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.391416072845459,
      "kl": 0.45310464315116405,
      "learning_rate": 3.3753888243890024e-06,
      "loss": -0.1642,
      "num_tokens": 28732534.0,
      "reward": 0.7450000047683716,
      "reward_std": 0.4773208498954773,
      "rewards/rollout_reward_func/mean": 0.7450000047683716,
      "rewards/rollout_reward_func/std": 0.5206199288368225,
      "sampling/importance_sampling_ratio/max": 1.8249852657318115,
      "sampling/importance_sampling_ratio/mean": 1.0556124448776245,
      "sampling/importance_sampling_ratio/min": 0.48363590240478516,
      "sampling/sampling_logp_difference/max": 0.3546333312988281,
      "sampling/sampling_logp_difference/mean": 0.02800387144088745,
      "step": 709,
      "step_time": 13.002759124006843
    },
    {
      "clip_ratio/high_max": 0.007919034222140908,
      "clip_ratio/high_mean": 0.003959517111070454,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.019584517111070454,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 683.0,
      "completions/mean_length": 230.46875,
      "completions/mean_terminated_length": 217.07693481445312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.48967288061976433,
      "epoch": 0.0142,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1521424055099487,
      "kl": 0.5007813004776835,
      "learning_rate": 3.3666617022056063e-06,
      "loss": -0.1173,
      "num_tokens": 28772646.0,
      "reward": 0.5721874833106995,
      "reward_std": 0.5809985399246216,
      "rewards/rollout_reward_func/mean": 0.5721874833106995,
      "rewards/rollout_reward_func/std": 0.6119652986526489,
      "sampling/importance_sampling_ratio/max": 1.8371174335479736,
      "sampling/importance_sampling_ratio/mean": 0.9320015907287598,
      "sampling/importance_sampling_ratio/min": 1.62229851774498e-09,
      "sampling/sampling_logp_difference/max": 10.611817359924316,
      "sampling/sampling_logp_difference/mean": 0.08162572979927063,
      "step": 710,
      "step_time": 12.337245897986577
    },
    {
      "clip_ratio/high_max": 0.015156249981373549,
      "clip_ratio/high_mean": 0.007578124990686774,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009531249990686774,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 660.0,
      "completions/mean_length": 214.96875,
      "completions/mean_terminated_length": 222.13636779785156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5286127105355263,
      "epoch": 0.01422,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3111858367919922,
      "kl": 0.5056091984733939,
      "learning_rate": 3.357972613203503e-06,
      "loss": -0.17,
      "num_tokens": 28811082.0,
      "reward": 0.7406250238418579,
      "reward_std": 0.5652167797088623,
      "rewards/rollout_reward_func/mean": 0.7406250238418579,
      "rewards/rollout_reward_func/std": 0.5763339400291443,
      "sampling/importance_sampling_ratio/max": 1.9255945682525635,
      "sampling/importance_sampling_ratio/mean": 1.0035463571548462,
      "sampling/importance_sampling_ratio/min": 3.751408803509548e-05,
      "sampling/sampling_logp_difference/max": 10.17403507232666,
      "sampling/sampling_logp_difference/mean": 0.05978291854262352,
      "step": 711,
      "step_time": 12.889424158012844
    },
    {
      "clip_ratio/high_max": 0.01030667731538415,
      "clip_ratio/high_mean": 0.005153338657692075,
      "clip_ratio/low_mean": 0.005022321478463709,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010175660136155784,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 678.0,
      "completions/max_terminated_length": 568.0,
      "completions/mean_length": 247.75,
      "completions/mean_terminated_length": 218.0,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4610928390175104,
      "epoch": 0.01424,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0974819660186768,
      "kl": 0.3773178206756711,
      "learning_rate": 3.3493216719976483e-06,
      "loss": -0.0406,
      "num_tokens": 28852238.0,
      "reward": 0.7112499475479126,
      "reward_std": 0.610455334186554,
      "rewards/rollout_reward_func/mean": 0.7112499475479126,
      "rewards/rollout_reward_func/std": 0.5935608148574829,
      "sampling/importance_sampling_ratio/max": 1.58568274974823,
      "sampling/importance_sampling_ratio/mean": 1.0467911958694458,
      "sampling/importance_sampling_ratio/min": 0.4928528070449829,
      "sampling/sampling_logp_difference/max": 0.34273433685302734,
      "sampling/sampling_logp_difference/mean": 0.029337547719478607,
      "step": 712,
      "step_time": 13.039710854980513
    },
    {
      "clip_ratio/high_max": 0.005078125046566129,
      "clip_ratio/high_mean": 0.0025390625232830644,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004492187523283064,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 616.0,
      "completions/mean_length": 205.4375,
      "completions/mean_terminated_length": 201.6521759033203,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5389968231320381,
      "epoch": 0.01426,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4987709522247314,
      "kl": 0.3097542431205511,
      "learning_rate": 3.340708992699808e-06,
      "loss": -0.0564,
      "num_tokens": 28891545.0,
      "reward": 0.6128124594688416,
      "reward_std": 0.623967170715332,
      "rewards/rollout_reward_func/mean": 0.6128124594688416,
      "rewards/rollout_reward_func/std": 0.6200428009033203,
      "sampling/importance_sampling_ratio/max": 1.6250985860824585,
      "sampling/importance_sampling_ratio/mean": 0.9241743683815002,
      "sampling/importance_sampling_ratio/min": 3.8199394558154885e-19,
      "sampling/sampling_logp_difference/max": 22.292898178100586,
      "sampling/sampling_logp_difference/mean": 0.15461465716362,
      "step": 713,
      "step_time": 12.823461647058139
    },
    {
      "clip_ratio/high_max": 0.009765625,
      "clip_ratio/high_mean": 0.0048828125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0048828125,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 628.0,
      "completions/max_terminated_length": 628.0,
      "completions/mean_length": 192.21875,
      "completions/mean_terminated_length": 190.75999450683594,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4502426153048873,
      "epoch": 0.01428,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3396546840667725,
      "kl": 0.8268671901896596,
      "learning_rate": 3.3321346889170426e-06,
      "loss": 0.0103,
      "num_tokens": 28928982.0,
      "reward": 0.6728896498680115,
      "reward_std": 0.5914465188980103,
      "rewards/rollout_reward_func/mean": 0.6728896498680115,
      "rewards/rollout_reward_func/std": 0.614251434803009,
      "sampling/importance_sampling_ratio/max": 1.2740529775619507,
      "sampling/importance_sampling_ratio/mean": 0.9161950349807739,
      "sampling/importance_sampling_ratio/min": 5.8384633414213846e-33,
      "sampling/sampling_logp_difference/max": 24.564224243164062,
      "sampling/sampling_logp_difference/mean": 0.18355610966682434,
      "step": 714,
      "step_time": 12.490839308025897
    },
    {
      "clip_ratio/high_max": 0.0013020833721384406,
      "clip_ratio/high_mean": 0.0006510416860692203,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0026041666860692203,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 1062.0,
      "completions/max_terminated_length": 554.0,
      "completions/mean_length": 268.34375,
      "completions/mean_terminated_length": 202.88888549804688,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.5286011127755046,
      "epoch": 0.0143,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4554654359817505,
      "kl": 0.4121161261573434,
      "learning_rate": 3.323598873750216e-06,
      "loss": -0.0805,
      "num_tokens": 28970238.0,
      "reward": 0.6442862749099731,
      "reward_std": 0.5939133167266846,
      "rewards/rollout_reward_func/mean": 0.6442862749099731,
      "rewards/rollout_reward_func/std": 0.6188769936561584,
      "sampling/importance_sampling_ratio/max": 1.4448583126068115,
      "sampling/importance_sampling_ratio/mean": 0.9802855253219604,
      "sampling/importance_sampling_ratio/min": 0.4188537299633026,
      "sampling/sampling_logp_difference/max": 0.6501269340515137,
      "sampling/sampling_logp_difference/mean": 0.030388034880161285,
      "step": 715,
      "step_time": 14.354621340011363
    },
    {
      "clip_ratio/high_max": 0.026041667442768812,
      "clip_ratio/high_mean": 0.013020833721384406,
      "clip_ratio/low_mean": 0.0013586956774815917,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014379529398865998,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 959.0,
      "completions/max_terminated_length": 959.0,
      "completions/mean_length": 246.4375,
      "completions/mean_terminated_length": 218.13636779785156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4798473287373781,
      "epoch": 0.01432,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8744069337844849,
      "kl": 1.1700994856655598,
      "learning_rate": 3.315101659792499e-06,
      "loss": 0.2372,
      "num_tokens": 29009645.0,
      "reward": 0.562264621257782,
      "reward_std": 0.6745161414146423,
      "rewards/rollout_reward_func/mean": 0.562264621257782,
      "rewards/rollout_reward_func/std": 0.6889097690582275,
      "sampling/importance_sampling_ratio/max": 2.263731002807617,
      "sampling/importance_sampling_ratio/mean": 1.0201733112335205,
      "sampling/importance_sampling_ratio/min": 0.30909299850463867,
      "sampling/sampling_logp_difference/max": 0.9329099655151367,
      "sampling/sampling_logp_difference/mean": 0.032889820635318756,
      "step": 716,
      "step_time": 14.054363012968679
    },
    {
      "clip_ratio/high_max": 0.0029761905316263437,
      "clip_ratio/high_mean": 0.0014880952658131719,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002464657765813172,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 578.0,
      "completions/mean_length": 209.53125,
      "completions/mean_terminated_length": 194.59999084472656,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4332255842164159,
      "epoch": 0.01434,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0990835428237915,
      "kl": 0.9386600153520703,
      "learning_rate": 3.306643159127889e-06,
      "loss": -0.1753,
      "num_tokens": 29048398.0,
      "reward": 0.7019521594047546,
      "reward_std": 0.6581175327301025,
      "rewards/rollout_reward_func/mean": 0.7019521594047546,
      "rewards/rollout_reward_func/std": 0.6617401242256165,
      "sampling/importance_sampling_ratio/max": 1.7448502779006958,
      "sampling/importance_sampling_ratio/mean": 0.977921724319458,
      "sampling/importance_sampling_ratio/min": 0.37719613313674927,
      "sampling/sampling_logp_difference/max": 0.9861326217651367,
      "sampling/sampling_logp_difference/mean": 0.03098313882946968,
      "step": 717,
      "step_time": 12.501150178999524
    },
    {
      "clip_ratio/high_max": 0.01065340917557478,
      "clip_ratio/high_mean": 0.00532670458778739,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0075588475447148085,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1967.0,
      "completions/max_terminated_length": 1967.0,
      "completions/mean_length": 291.5,
      "completions/mean_terminated_length": 279.227294921875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5180753171443939,
      "epoch": 0.01436,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5371558666229248,
      "kl": 0.48448933102190495,
      "learning_rate": 3.2982234833297273e-06,
      "loss": -0.0159,
      "num_tokens": 29090197.0,
      "reward": 0.6623090505599976,
      "reward_std": 0.519455075263977,
      "rewards/rollout_reward_func/mean": 0.6623090505599976,
      "rewards/rollout_reward_func/std": 0.5482516884803772,
      "sampling/importance_sampling_ratio/max": 1.6159664392471313,
      "sampling/importance_sampling_ratio/mean": 0.8991981744766235,
      "sampling/importance_sampling_ratio/min": 4.929646643149488e-13,
      "sampling/sampling_logp_difference/max": 20.900863647460938,
      "sampling/sampling_logp_difference/mean": 0.13725176453590393,
      "step": 718,
      "step_time": 17.74648307201278
    },
    {
      "clip_ratio/high_max": 0.015873016323894262,
      "clip_ratio/high_mean": 0.007936508161947131,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007936508161947131,
      "completions/clipped_ratio": 0.15625,
      "completions/max_length": 633.0,
      "completions/max_terminated_length": 633.0,
      "completions/mean_length": 240.65625,
      "completions/mean_terminated_length": 245.59259033203125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.49224462546408176,
      "epoch": 0.01438,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1902836561203003,
      "kl": 0.37016783095896244,
      "learning_rate": 3.28984274345923e-06,
      "loss": -0.002,
      "num_tokens": 29130473.0,
      "reward": 0.704415500164032,
      "reward_std": 0.5518934726715088,
      "rewards/rollout_reward_func/mean": 0.704415500164032,
      "rewards/rollout_reward_func/std": 0.5420209169387817,
      "sampling/importance_sampling_ratio/max": 1.3148653507232666,
      "sampling/importance_sampling_ratio/mean": 1.0047491788864136,
      "sampling/importance_sampling_ratio/min": 0.5721502900123596,
      "sampling/sampling_logp_difference/max": 0.577038049697876,
      "sampling/sampling_logp_difference/mean": 0.028062354773283005,
      "step": 719,
      "step_time": 12.662389084027382
    },
    {
      "clip_ratio/high_max": 0.01750963064841926,
      "clip_ratio/high_mean": 0.00875481532420963,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00973137782420963,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 646.0,
      "completions/mean_length": 229.5,
      "completions/mean_terminated_length": 194.8636474609375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.507029989734292,
      "epoch": 0.0144,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8154157400131226,
      "kl": 0.40269656851887703,
      "learning_rate": 3.281501050064022e-06,
      "loss": 0.0054,
      "num_tokens": 29170442.0,
      "reward": 0.8968749642372131,
      "reward_std": 0.3562329113483429,
      "rewards/rollout_reward_func/mean": 0.8968749642372131,
      "rewards/rollout_reward_func/std": 0.4195576310157776,
      "sampling/importance_sampling_ratio/max": 1.5824490785598755,
      "sampling/importance_sampling_ratio/mean": 0.9493585824966431,
      "sampling/importance_sampling_ratio/min": 0.4340607821941376,
      "sampling/sampling_logp_difference/max": 0.5756795406341553,
      "sampling/sampling_logp_difference/mean": 0.03249254822731018,
      "step": 720,
      "step_time": 12.738819597987458
    },
    {
      "clip_ratio/high_max": 0.00987103208899498,
      "clip_ratio/high_mean": 0.00493551604449749,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00493551604449749,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 666.0,
      "completions/max_terminated_length": 666.0,
      "completions/mean_length": 252.34375,
      "completions/mean_terminated_length": 251.0,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.49298456963151693,
      "epoch": 0.01442,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4559191465377808,
      "kl": 0.32447809912264347,
      "learning_rate": 3.2731985131766765e-06,
      "loss": -0.0625,
      "num_tokens": 29212375.0,
      "reward": 0.5181249976158142,
      "reward_std": 0.675213098526001,
      "rewards/rollout_reward_func/mean": 0.5181249976158142,
      "rewards/rollout_reward_func/std": 0.7126616835594177,
      "sampling/importance_sampling_ratio/max": 1.428519606590271,
      "sampling/importance_sampling_ratio/mean": 0.9478368759155273,
      "sampling/importance_sampling_ratio/min": 1.8096818684023132e-15,
      "sampling/sampling_logp_difference/max": 19.823469161987305,
      "sampling/sampling_logp_difference/mean": 0.08846940100193024,
      "step": 721,
      "step_time": 12.470684099011123
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 608.0,
      "completions/max_terminated_length": 567.0,
      "completions/mean_length": 240.9375,
      "completions/mean_terminated_length": 232.20834350585938,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.4798616375774145,
      "epoch": 0.01444,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9289250373840332,
      "kl": 0.3316602557897568,
      "learning_rate": 3.2649352423132675e-06,
      "loss": -0.1151,
      "num_tokens": 29251796.0,
      "reward": 0.9306249618530273,
      "reward_std": 0.21165037155151367,
      "rewards/rollout_reward_func/mean": 0.9306249618530273,
      "rewards/rollout_reward_func/std": 0.2917681038379669,
      "sampling/importance_sampling_ratio/max": 1.4687185287475586,
      "sampling/importance_sampling_ratio/mean": 0.9999045729637146,
      "sampling/importance_sampling_ratio/min": 0.3683030903339386,
      "sampling/sampling_logp_difference/max": 0.4748101234436035,
      "sampling/sampling_logp_difference/mean": 0.027078866958618164,
      "step": 722,
      "step_time": 12.706094427019707
    },
    {
      "clip_ratio/high_max": 0.0021551724057644606,
      "clip_ratio/high_mean": 0.0010775862028822303,
      "clip_ratio/low_mean": 0.0013020833721384406,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002379669575020671,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 682.0,
      "completions/max_terminated_length": 661.0,
      "completions/mean_length": 318.6875,
      "completions/mean_terminated_length": 305.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4711466021835804,
      "epoch": 0.01446,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2201794385910034,
      "kl": 0.34455125499516726,
      "learning_rate": 3.256711346471925e-06,
      "loss": -0.0241,
      "num_tokens": 29295195.0,
      "reward": 0.5234375,
      "reward_std": 0.5743794441223145,
      "rewards/rollout_reward_func/mean": 0.5234375,
      "rewards/rollout_reward_func/std": 0.5595136284828186,
      "sampling/importance_sampling_ratio/max": 1.3420860767364502,
      "sampling/importance_sampling_ratio/mean": 0.947127103805542,
      "sampling/importance_sampling_ratio/min": 0.5043270587921143,
      "sampling/sampling_logp_difference/max": 0.48111653327941895,
      "sampling/sampling_logp_difference/mean": 0.03130500018596649,
      "step": 723,
      "step_time": 12.849461837016861
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0032087054569274187,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 653.0,
      "completions/max_terminated_length": 608.0,
      "completions/mean_length": 219.1875,
      "completions/mean_terminated_length": 161.45834350585938,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.43899640440940857,
      "epoch": 0.01448,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4075775146484375,
      "kl": 0.43648094683885574,
      "learning_rate": 3.2485269341313947e-06,
      "loss": -0.0219,
      "num_tokens": 29333671.0,
      "reward": 0.8012499809265137,
      "reward_std": 0.47985148429870605,
      "rewards/rollout_reward_func/mean": 0.8012499809265137,
      "rewards/rollout_reward_func/std": 0.48714935779571533,
      "sampling/importance_sampling_ratio/max": 1.3885093927383423,
      "sampling/importance_sampling_ratio/mean": 0.9858540296554565,
      "sampling/importance_sampling_ratio/min": 6.416943051590351e-06,
      "sampling/sampling_logp_difference/max": 11.350176811218262,
      "sampling/sampling_logp_difference/mean": 0.06304404884576797,
      "step": 724,
      "step_time": 12.679976604005788
    },
    {
      "clip_ratio/high_max": 0.0041082974057644606,
      "clip_ratio/high_mean": 0.0020541487028822303,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0020541487028822303,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 669.0,
      "completions/max_terminated_length": 657.0,
      "completions/mean_length": 322.15625,
      "completions/mean_terminated_length": 248.61904907226562,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.48942918702960014,
      "epoch": 0.0145,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4180080890655518,
      "kl": 0.3462750893086195,
      "learning_rate": 3.240382113249607e-06,
      "loss": -0.2733,
      "num_tokens": 29378263.0,
      "reward": 0.6465142965316772,
      "reward_std": 0.7209170460700989,
      "rewards/rollout_reward_func/mean": 0.6465142965316772,
      "rewards/rollout_reward_func/std": 0.6928356885910034,
      "sampling/importance_sampling_ratio/max": 2.159876823425293,
      "sampling/importance_sampling_ratio/mean": 0.9184368848800659,
      "sampling/importance_sampling_ratio/min": 1.679204605890027e-26,
      "sampling/sampling_logp_difference/max": 26.855131149291992,
      "sampling/sampling_logp_difference/mean": 0.135372593998909,
      "step": 725,
      "step_time": 13.09926985896891
    },
    {
      "clip_ratio/high_max": 0.0130106289871037,
      "clip_ratio/high_mean": 0.00650531449355185,
      "clip_ratio/low_mean": 0.005425347248092294,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011930661741644144,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 587.0,
      "completions/max_terminated_length": 587.0,
      "completions/mean_length": 186.96875,
      "completions/mean_terminated_length": 175.17391967773438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5872897654771805,
      "epoch": 0.01452,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2716169357299805,
      "kl": 0.3558140629902482,
      "learning_rate": 3.232276991262254e-06,
      "loss": -0.1662,
      "num_tokens": 29415628.0,
      "reward": 0.7921874523162842,
      "reward_std": 0.36637550592422485,
      "rewards/rollout_reward_func/mean": 0.7921874523162842,
      "rewards/rollout_reward_func/std": 0.5518925189971924,
      "sampling/importance_sampling_ratio/max": 1.4693760871887207,
      "sampling/importance_sampling_ratio/mean": 0.9149693250656128,
      "sampling/importance_sampling_ratio/min": 7.551504332721118e-24,
      "sampling/sampling_logp_difference/max": 25.8953857421875,
      "sampling/sampling_logp_difference/mean": 0.13250653445720673,
      "step": 726,
      "step_time": 12.434578606000287
    },
    {
      "clip_ratio/high_max": 0.009765625,
      "clip_ratio/high_mean": 0.0048828125,
      "clip_ratio/low_mean": 0.002802144270390272,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007684956770390272,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 670.0,
      "completions/max_terminated_length": 628.0,
      "completions/mean_length": 351.1875,
      "completions/mean_terminated_length": 388.0,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 0.50084831379354,
      "epoch": 0.01454,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3183870315551758,
      "kl": 0.4376723375171423,
      "learning_rate": 3.224211675081378e-06,
      "loss": -0.0894,
      "num_tokens": 29460914.0,
      "reward": 0.776562511920929,
      "reward_std": 0.4391661286354065,
      "rewards/rollout_reward_func/mean": 0.776562511920929,
      "rewards/rollout_reward_func/std": 0.4294077455997467,
      "sampling/importance_sampling_ratio/max": 1.6610665321350098,
      "sampling/importance_sampling_ratio/mean": 1.0094783306121826,
      "sampling/importance_sampling_ratio/min": 0.27699902653694153,
      "sampling/sampling_logp_difference/max": 0.5866823196411133,
      "sampling/sampling_logp_difference/mean": 0.028457988053560257,
      "step": 727,
      "step_time": 12.801520429973607
    },
    {
      "clip_ratio/high_max": 0.009765625,
      "clip_ratio/high_mean": 0.0048828125,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0068359375,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 660.0,
      "completions/mean_length": 282.75,
      "completions/mean_terminated_length": 242.9375,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5355227906256914,
      "epoch": 0.01456,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4240736961364746,
      "kl": 0.5311089279130101,
      "learning_rate": 3.216186271093948e-06,
      "loss": -0.2122,
      "num_tokens": 29503204.0,
      "reward": 0.6368749737739563,
      "reward_std": 0.6426747441291809,
      "rewards/rollout_reward_func/mean": 0.6368749737739563,
      "rewards/rollout_reward_func/std": 0.6929386854171753,
      "sampling/importance_sampling_ratio/max": 2.568697690963745,
      "sampling/importance_sampling_ratio/mean": 1.0425124168395996,
      "sampling/importance_sampling_ratio/min": 3.388442147183031e-17,
      "sampling/sampling_logp_difference/max": 19.89157485961914,
      "sampling/sampling_logp_difference/mean": 0.08810840547084808,
      "step": 728,
      "step_time": 13.377193659965997
    },
    {
      "clip_ratio/high_max": 0.01212177611887455,
      "clip_ratio/high_mean": 0.006060888059437275,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007037450559437275,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 860.0,
      "completions/max_terminated_length": 860.0,
      "completions/mean_length": 266.28125,
      "completions/mean_terminated_length": 263.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.583252590149641,
      "epoch": 0.01458,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.1073246002197266,
      "kl": 0.4767769118770957,
      "learning_rate": 3.2082008851604685e-06,
      "loss": 0.2574,
      "num_tokens": 29543860.0,
      "reward": 0.8216252326965332,
      "reward_std": 0.4002404808998108,
      "rewards/rollout_reward_func/mean": 0.8216252326965332,
      "rewards/rollout_reward_func/std": 0.4781603217124939,
      "sampling/importance_sampling_ratio/max": 2.0474109649658203,
      "sampling/importance_sampling_ratio/mean": 0.9720062613487244,
      "sampling/importance_sampling_ratio/min": 3.1371375040441057e-22,
      "sampling/sampling_logp_difference/max": 18.933488845825195,
      "sampling/sampling_logp_difference/mean": 0.11713943630456924,
      "step": 729,
      "step_time": 13.449768159989617
    },
    {
      "clip_ratio/high_max": 0.004670516354963183,
      "clip_ratio/high_mean": 0.0023352581774815917,
      "clip_ratio/low_mean": 0.00424479169305414,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006580049870535731,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 685.0,
      "completions/max_terminated_length": 685.0,
      "completions/mean_length": 215.75,
      "completions/mean_terminated_length": 208.85714721679688,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.567943811416626,
      "epoch": 0.0146,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.380834698677063,
      "kl": 0.3466300508007407,
      "learning_rate": 3.200255622613578e-06,
      "loss": -0.2314,
      "num_tokens": 29582894.0,
      "reward": 0.7009375095367432,
      "reward_std": 0.5542019009590149,
      "rewards/rollout_reward_func/mean": 0.7009375095367432,
      "rewards/rollout_reward_func/std": 0.6397799849510193,
      "sampling/importance_sampling_ratio/max": 1.2731469869613647,
      "sampling/importance_sampling_ratio/mean": 0.9326949715614319,
      "sampling/importance_sampling_ratio/min": 3.239857417541213e-23,
      "sampling/sampling_logp_difference/max": 21.554866790771484,
      "sampling/sampling_logp_difference/mean": 0.12790434062480927,
      "step": 730,
      "step_time": 12.256643322005402
    },
    {
      "clip_ratio/high_max": 0.006119791883975267,
      "clip_ratio/high_mean": 0.0030598959419876337,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0030598959419876337,
      "completions/clipped_ratio": 0.21875,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 610.0,
      "completions/mean_length": 287.15625,
      "completions/mean_terminated_length": 239.39999389648438,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.44000831455923617,
      "epoch": 0.01462,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9314566850662231,
      "kl": 0.5366800222545862,
      "learning_rate": 3.192350588256662e-06,
      "loss": 0.0356,
      "num_tokens": 29624689.0,
      "reward": 0.6484375,
      "reward_std": 0.5565017461776733,
      "rewards/rollout_reward_func/mean": 0.6484375,
      "rewards/rollout_reward_func/std": 0.6001457571983337,
      "sampling/importance_sampling_ratio/max": 1.4172297716140747,
      "sampling/importance_sampling_ratio/mean": 0.9292104244232178,
      "sampling/importance_sampling_ratio/min": 2.639477081345376e-08,
      "sampling/sampling_logp_difference/max": 17.244674682617188,
      "sampling/sampling_logp_difference/mean": 0.07010616362094879,
      "step": 731,
      "step_time": 13.012987769019674
    },
    {
      "clip_ratio/high_max": 0.013020833488553762,
      "clip_ratio/high_mean": 0.006510416744276881,
      "clip_ratio/low_mean": 0.0035908386344090104,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010101255378685892,
      "completions/clipped_ratio": 0.40625,
      "completions/max_length": 659.0,
      "completions/max_terminated_length": 659.0,
      "completions/mean_length": 287.65625,
      "completions/mean_terminated_length": 194.15789794921875,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5337341651320457,
      "epoch": 0.01464,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2560105323791504,
      "kl": 0.4467210881412029,
      "learning_rate": 3.1844858863624657e-06,
      "loss": 0.035,
      "num_tokens": 29666705.0,
      "reward": 0.77406245470047,
      "reward_std": 0.5044159889221191,
      "rewards/rollout_reward_func/mean": 0.77406245470047,
      "rewards/rollout_reward_func/std": 0.508094847202301,
      "sampling/importance_sampling_ratio/max": 1.5316784381866455,
      "sampling/importance_sampling_ratio/mean": 0.9965259432792664,
      "sampling/importance_sampling_ratio/min": 0.504578173160553,
      "sampling/sampling_logp_difference/max": 0.42350053787231445,
      "sampling/sampling_logp_difference/mean": 0.02890491858124733,
      "step": 732,
      "step_time": 12.481057238037465
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.005859375,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008463541744276881,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 1140.0,
      "completions/max_terminated_length": 1140.0,
      "completions/mean_length": 209.75,
      "completions/mean_terminated_length": 234.2857208251953,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5614222018048167,
      "epoch": 0.01466,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.174406051635742,
      "kl": 0.5525397658348083,
      "learning_rate": 3.1766616206717248e-06,
      "loss": 0.0986,
      "num_tokens": 29704685.0,
      "reward": 0.8713753819465637,
      "reward_std": 0.2849070131778717,
      "rewards/rollout_reward_func/mean": 0.8713753819465637,
      "rewards/rollout_reward_func/std": 0.3273763060569763,
      "sampling/importance_sampling_ratio/max": 1.9709469079971313,
      "sampling/importance_sampling_ratio/mean": 0.8749340772628784,
      "sampling/importance_sampling_ratio/min": 3.2160574510587745e-29,
      "sampling/sampling_logp_difference/max": 24.21875,
      "sampling/sampling_logp_difference/mean": 0.3078807294368744,
      "step": 733,
      "step_time": 14.1586909879843
    },
    {
      "clip_ratio/high_max": 0.01239583338610828,
      "clip_ratio/high_mean": 0.00619791669305414,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00619791669305414,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 677.0,
      "completions/max_terminated_length": 677.0,
      "completions/mean_length": 201.9375,
      "completions/mean_terminated_length": 195.63636779785156,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4076655497774482,
      "epoch": 0.01468,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.432253122329712,
      "kl": 0.31387206818908453,
      "learning_rate": 3.168877894391792e-06,
      "loss": -0.0848,
      "num_tokens": 29742938.0,
      "reward": 0.8618749380111694,
      "reward_std": 0.36544036865234375,
      "rewards/rollout_reward_func/mean": 0.8618749380111694,
      "rewards/rollout_reward_func/std": 0.44681984186172485,
      "sampling/importance_sampling_ratio/max": 1.5577398538589478,
      "sampling/importance_sampling_ratio/mean": 1.0004154443740845,
      "sampling/importance_sampling_ratio/min": 0.5313699841499329,
      "sampling/sampling_logp_difference/max": 0.5817354917526245,
      "sampling/sampling_logp_difference/mean": 0.030225668102502823,
      "step": 734,
      "step_time": 12.767145565027022
    },
    {
      "clip_ratio/high_max": 0.001953125,
      "clip_ratio/high_mean": 0.0009765625,
      "clip_ratio/low_mean": 0.0010416667209938169,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002018229220993817,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 664.0,
      "completions/max_terminated_length": 629.0,
      "completions/mean_length": 276.5625,
      "completions/mean_terminated_length": 278.04547119140625,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4516623103991151,
      "epoch": 0.0147,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3153918981552124,
      "kl": 0.3791872225701809,
      "learning_rate": 3.16113481019528e-06,
      "loss": -0.1129,
      "num_tokens": 29785651.0,
      "reward": 0.7803124785423279,
      "reward_std": 0.4110404849052429,
      "rewards/rollout_reward_func/mean": 0.7803124785423279,
      "rewards/rollout_reward_func/std": 0.50376957654953,
      "sampling/importance_sampling_ratio/max": 1.5252150297164917,
      "sampling/importance_sampling_ratio/mean": 0.9496152400970459,
      "sampling/importance_sampling_ratio/min": 1.0311159292057047e-26,
      "sampling/sampling_logp_difference/max": 20.596698760986328,
      "sampling/sampling_logp_difference/mean": 0.1711180955171585,
      "step": 735,
      "step_time": 12.83534394997696
    },
    {
      "clip_ratio/high_max": 0.014453125186264515,
      "clip_ratio/high_mean": 0.00830414891242981,
      "clip_ratio/low_mean": 0.0024999999441206455,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010804148856550455,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 630.0,
      "completions/max_terminated_length": 620.0,
      "completions/mean_length": 240.6875,
      "completions/mean_terminated_length": 193.69564819335938,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.43220276990905404,
      "epoch": 0.01472,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.094756603240967,
      "kl": 0.40736623480916023,
      "learning_rate": 3.1534324702187026e-06,
      "loss": -0.0678,
      "num_tokens": 29825414.0,
      "reward": 0.8265625238418579,
      "reward_std": 0.40395915508270264,
      "rewards/rollout_reward_func/mean": 0.8265625238418579,
      "rewards/rollout_reward_func/std": 0.48171475529670715,
      "sampling/importance_sampling_ratio/max": 1.6921881437301636,
      "sampling/importance_sampling_ratio/mean": 0.9668674468994141,
      "sampling/importance_sampling_ratio/min": 4.681599818923132e-08,
      "sampling/sampling_logp_difference/max": 16.562488555908203,
      "sampling/sampling_logp_difference/mean": 0.0660192146897316,
      "step": 736,
      "step_time": 12.658509466971736
    },
    {
      "clip_ratio/high_max": 0.015414663823321462,
      "clip_ratio/high_mean": 0.007707331911660731,
      "clip_ratio/low_mean": 0.005621693329885602,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013329025241546333,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 602.0,
      "completions/max_terminated_length": 602.0,
      "completions/mean_length": 233.03125,
      "completions/mean_terminated_length": 248.4499969482422,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.4949864475056529,
      "epoch": 0.01474,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0583629608154297,
      "kl": 1.4401422869414091,
      "learning_rate": 3.1457709760611314e-06,
      "loss": -0.15,
      "num_tokens": 29864131.0,
      "reward": 0.5484374761581421,
      "reward_std": 0.6239173412322998,
      "rewards/rollout_reward_func/mean": 0.5484374761581421,
      "rewards/rollout_reward_func/std": 0.7267563939094543,
      "sampling/importance_sampling_ratio/max": 1.7347853183746338,
      "sampling/importance_sampling_ratio/mean": 0.8741345405578613,
      "sampling/importance_sampling_ratio/min": 1.1145926227545802e-17,
      "sampling/sampling_logp_difference/max": 24.68350601196289,
      "sampling/sampling_logp_difference/mean": 0.15708062052726746,
      "step": 737,
      "step_time": 12.58249156497186
    },
    {
      "clip_ratio/high_max": 0.013585069682449102,
      "clip_ratio/high_mean": 0.006792534841224551,
      "clip_ratio/low_mean": 0.004266036208719015,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011058571166358888,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 628.0,
      "completions/max_terminated_length": 628.0,
      "completions/mean_length": 319.125,
      "completions/mean_terminated_length": 353.5,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5605905363336205,
      "epoch": 0.01476,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5093605518341064,
      "kl": 0.3197620464488864,
      "learning_rate": 3.1381504287828525e-06,
      "loss": -0.1335,
      "num_tokens": 29907681.0,
      "reward": 0.6415624618530273,
      "reward_std": 0.6676101088523865,
      "rewards/rollout_reward_func/mean": 0.6415624618530273,
      "rewards/rollout_reward_func/std": 0.6778148412704468,
      "sampling/importance_sampling_ratio/max": 1.4363117218017578,
      "sampling/importance_sampling_ratio/mean": 0.8834515810012817,
      "sampling/importance_sampling_ratio/min": 1.7498326908712517e-16,
      "sampling/sampling_logp_difference/max": 19.701305389404297,
      "sampling/sampling_logp_difference/mean": 0.12919928133487701,
      "step": 738,
      "step_time": 12.7497917219589
    },
    {
      "clip_ratio/high_max": 0.010044642956927419,
      "clip_ratio/high_mean": 0.005022321478463709,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005022321478463709,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 536.0,
      "completions/mean_length": 160.625,
      "completions/mean_terminated_length": 121.80952453613281,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.5116040417924523,
      "epoch": 0.01478,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9275041222572327,
      "kl": 1.0747242690995336,
      "learning_rate": 3.130570928904039e-06,
      "loss": -0.1476,
      "num_tokens": 29944603.0,
      "reward": 0.5974999666213989,
      "reward_std": 0.7301347255706787,
      "rewards/rollout_reward_func/mean": 0.5974999666213989,
      "rewards/rollout_reward_func/std": 0.7274523973464966,
      "sampling/importance_sampling_ratio/max": 2.172180652618408,
      "sampling/importance_sampling_ratio/mean": 0.800421953201294,
      "sampling/importance_sampling_ratio/min": 7.865823495408694e-29,
      "sampling/sampling_logp_difference/max": 25.35128402709961,
      "sampling/sampling_logp_difference/mean": 0.3813154101371765,
      "step": 739,
      "step_time": 12.627796524044243
    },
    {
      "clip_ratio/high_max": 0.004036458441987634,
      "clip_ratio/high_mean": 0.002018229220993817,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002018229220993817,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 670.0,
      "completions/max_terminated_length": 670.0,
      "completions/mean_length": 195.0,
      "completions/mean_terminated_length": 197.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.47076040506362915,
      "epoch": 0.0148,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8684266805648804,
      "kl": 0.7203430468216538,
      "learning_rate": 3.1230325764034134e-06,
      "loss": -0.2225,
      "num_tokens": 29981549.0,
      "reward": 0.6687905192375183,
      "reward_std": 0.5619752407073975,
      "rewards/rollout_reward_func/mean": 0.6687905192375183,
      "rewards/rollout_reward_func/std": 0.6543561816215515,
      "sampling/importance_sampling_ratio/max": 1.400680422782898,
      "sampling/importance_sampling_ratio/mean": 0.8738430738449097,
      "sampling/importance_sampling_ratio/min": 1.144596136620768e-11,
      "sampling/sampling_logp_difference/max": 25.203420639038086,
      "sampling/sampling_logp_difference/mean": 0.18555590510368347,
      "step": 740,
      "step_time": 12.368393273005495
    },
    {
      "clip_ratio/high_max": 0.007305195089429617,
      "clip_ratio/high_mean": 0.0036525975447148085,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036525975447148085,
      "completions/clipped_ratio": 0.40625,
      "completions/max_length": 1209.0,
      "completions/max_terminated_length": 641.0,
      "completions/mean_length": 310.65625,
      "completions/mean_terminated_length": 241.57894897460938,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5151430908590555,
      "epoch": 0.01482,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1576963663101196,
      "kl": 0.34719780925661325,
      "learning_rate": 3.1155354707169467e-06,
      "loss": -0.1433,
      "num_tokens": 30024309.0,
      "reward": 0.6484375,
      "reward_std": 0.5802971124649048,
      "rewards/rollout_reward_func/mean": 0.6484375,
      "rewards/rollout_reward_func/std": 0.5998069643974304,
      "sampling/importance_sampling_ratio/max": 1.2452459335327148,
      "sampling/importance_sampling_ratio/mean": 0.9086505174636841,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 20.0908145904541,
      "sampling/sampling_logp_difference/mean": 0.09213875234127045,
      "step": 741,
      "step_time": 15.547273068033974
    },
    {
      "clip_ratio/high_max": 0.0031250000465661287,
      "clip_ratio/high_mean": 0.0026785715017467737,
      "clip_ratio/low_mean": 0.0026041667442768812,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005282738246023655,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 968.0,
      "completions/max_terminated_length": 968.0,
      "completions/mean_length": 242.59375,
      "completions/mean_terminated_length": 245.0416717529297,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4731646485161036,
      "epoch": 0.01484,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.261169672012329,
      "kl": 0.31969987880438566,
      "learning_rate": 3.108079710736526e-06,
      "loss": -0.0594,
      "num_tokens": 30065623.0,
      "reward": 0.6746875047683716,
      "reward_std": 0.6152135729789734,
      "rewards/rollout_reward_func/mean": 0.6746875047683716,
      "rewards/rollout_reward_func/std": 0.6480938792228699,
      "sampling/importance_sampling_ratio/max": 2.0577616691589355,
      "sampling/importance_sampling_ratio/mean": 0.982590913772583,
      "sampling/importance_sampling_ratio/min": 3.10685548406708e-20,
      "sampling/sampling_logp_difference/max": 22.87569808959961,
      "sampling/sampling_logp_difference/mean": 0.10986822843551636,
      "step": 742,
      "step_time": 14.013582757979748
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.002474767155945301,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006381017155945301,
      "completions/clipped_ratio": 0.34375,
      "completions/max_length": 873.0,
      "completions/max_terminated_length": 873.0,
      "completions/mean_length": 328.625,
      "completions/mean_terminated_length": 280.1428527832031,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5073415283113718,
      "epoch": 0.01486,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5766180753707886,
      "kl": 0.8581191245466471,
      "learning_rate": 3.1006653948086677e-06,
      "loss": -0.0257,
      "num_tokens": 30110516.0,
      "reward": 0.8081274032592773,
      "reward_std": 0.41531452536582947,
      "rewards/rollout_reward_func/mean": 0.8081274032592773,
      "rewards/rollout_reward_func/std": 0.5054205060005188,
      "sampling/importance_sampling_ratio/max": 1.6091597080230713,
      "sampling/importance_sampling_ratio/mean": 0.9421054720878601,
      "sampling/importance_sampling_ratio/min": 0.21989549696445465,
      "sampling/sampling_logp_difference/max": 1.2113580703735352,
      "sampling/sampling_logp_difference/mean": 0.03692500293254852,
      "step": 743,
      "step_time": 13.615862716003903
    },
    {
      "clip_ratio/high_max": 0.005859375,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.007921006996184587,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010850694496184587,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 683.0,
      "completions/max_terminated_length": 563.0,
      "completions/mean_length": 245.90625,
      "completions/mean_terminated_length": 235.4499969482422,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.40744494879618287,
      "epoch": 0.01488,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1440675258636475,
      "kl": 0.37208239547908306,
      "learning_rate": 3.0932926207332104e-06,
      "loss": -0.027,
      "num_tokens": 30151715.0,
      "reward": 0.7106249332427979,
      "reward_std": 0.5804787874221802,
      "rewards/rollout_reward_func/mean": 0.7106249332427979,
      "rewards/rollout_reward_func/std": 0.5967813730239868,
      "sampling/importance_sampling_ratio/max": 1.8259047269821167,
      "sampling/importance_sampling_ratio/mean": 1.0135408639907837,
      "sampling/importance_sampling_ratio/min": 0.3458947539329529,
      "sampling/sampling_logp_difference/max": 0.5492846965789795,
      "sampling/sampling_logp_difference/mean": 0.03012843430042267,
      "step": 744,
      "step_time": 12.523846199997934
    },
    {
      "clip_ratio/high_max": 0.008721064776182175,
      "clip_ratio/high_mean": 0.004360532388091087,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004360532388091087,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 599.0,
      "completions/max_terminated_length": 599.0,
      "completions/mean_length": 225.9375,
      "completions/mean_terminated_length": 232.78260803222656,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4925082065165043,
      "epoch": 0.0149,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2509959936141968,
      "kl": 0.5574252661317587,
      "learning_rate": 3.0859614857620308e-06,
      "loss": -0.1247,
      "num_tokens": 30191552.0,
      "reward": 0.7696874737739563,
      "reward_std": 0.4131181836128235,
      "rewards/rollout_reward_func/mean": 0.7696874737739563,
      "rewards/rollout_reward_func/std": 0.5028273463249207,
      "sampling/importance_sampling_ratio/max": 1.9727065563201904,
      "sampling/importance_sampling_ratio/mean": 1.0254004001617432,
      "sampling/importance_sampling_ratio/min": 0.45614567399024963,
      "sampling/sampling_logp_difference/max": 0.36458849906921387,
      "sampling/sampling_logp_difference/mean": 0.03064616024494171,
      "step": 745,
      "step_time": 12.713932881015353
    },
    {
      "clip_ratio/high_max": 0.01228248723782599,
      "clip_ratio/high_mean": 0.006141243618912995,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006141243618912995,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 656.0,
      "completions/max_terminated_length": 560.0,
      "completions/mean_length": 213.25,
      "completions/mean_terminated_length": 156.25,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47700708732008934,
      "epoch": 0.01492,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.070786952972412,
      "kl": 0.3700340464711189,
      "learning_rate": 3.0786720865977503e-06,
      "loss": -0.076,
      "num_tokens": 30230068.0,
      "reward": 0.7359374761581421,
      "reward_std": 0.49488821625709534,
      "rewards/rollout_reward_func/mean": 0.7359374761581421,
      "rewards/rollout_reward_func/std": 0.5735450387001038,
      "sampling/importance_sampling_ratio/max": 1.933611273765564,
      "sampling/importance_sampling_ratio/mean": 1.0596773624420166,
      "sampling/importance_sampling_ratio/min": 0.5527346134185791,
      "sampling/sampling_logp_difference/max": 0.34058642387390137,
      "sampling/sampling_logp_difference/mean": 0.028428714722394943,
      "step": 746,
      "step_time": 12.736391898011789
    },
    {
      "clip_ratio/high_max": 0.011848958441987634,
      "clip_ratio/high_mean": 0.005924479220993817,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007877604220993817,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1155.0,
      "completions/max_terminated_length": 1155.0,
      "completions/mean_length": 235.0625,
      "completions/mean_terminated_length": 245.4499969482422,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.530717272311449,
      "epoch": 0.01494,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8401225805282593,
      "kl": 0.39797286316752434,
      "learning_rate": 3.0714245193924757e-06,
      "loss": 0.0093,
      "num_tokens": 30270737.0,
      "reward": 0.6165624856948853,
      "reward_std": 0.5694639086723328,
      "rewards/rollout_reward_func/mean": 0.6165624856948853,
      "rewards/rollout_reward_func/std": 0.658740758895874,
      "sampling/importance_sampling_ratio/max": 1.8916919231414795,
      "sampling/importance_sampling_ratio/mean": 0.9606941938400269,
      "sampling/importance_sampling_ratio/min": 2.850814973265788e-20,
      "sampling/sampling_logp_difference/max": 19.03331756591797,
      "sampling/sampling_logp_difference/mean": 0.10720722377300262,
      "step": 747,
      "step_time": 15.294303995993687
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.40625,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 177.5,
      "completions/mean_terminated_length": 184.84210205078125,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 0.44781807251274586,
      "epoch": 0.01496,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.422370433807373,
      "kl": 0.36365021485835314,
      "learning_rate": 3.064218879746515e-06,
      "loss": -0.0016,
      "num_tokens": 30307252.0,
      "reward": 0.8590624332427979,
      "reward_std": 0.3247721195220947,
      "rewards/rollout_reward_func/mean": 0.8590624332427979,
      "rewards/rollout_reward_func/std": 0.36588698625564575,
      "sampling/importance_sampling_ratio/max": 1.5050479173660278,
      "sampling/importance_sampling_ratio/mean": 1.0213823318481445,
      "sampling/importance_sampling_ratio/min": 0.45186078548431396,
      "sampling/sampling_logp_difference/max": 0.7137298583984375,
      "sampling/sampling_logp_difference/mean": 0.02835715189576149,
      "step": 748,
      "step_time": 12.551455967026413
    },
    {
      "clip_ratio/high_max": 0.009027487598359585,
      "clip_ratio/high_mean": 0.005490306182764471,
      "clip_ratio/low_mean": 0.0030691964784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00855950266122818,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 652.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 337.78125,
      "completions/mean_terminated_length": 307.0500183105469,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4640009421855211,
      "epoch": 0.01498,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0922178030014038,
      "kl": 0.3394578015431762,
      "learning_rate": 3.057055262707128e-06,
      "loss": 0.0389,
      "num_tokens": 30353029.0,
      "reward": 0.5903124809265137,
      "reward_std": 0.6159409880638123,
      "rewards/rollout_reward_func/mean": 0.5903124809265137,
      "rewards/rollout_reward_func/std": 0.6179961562156677,
      "sampling/importance_sampling_ratio/max": 1.8813104629516602,
      "sampling/importance_sampling_ratio/mean": 0.9517285823822021,
      "sampling/importance_sampling_ratio/min": 1.0224828665616845e-19,
      "sampling/sampling_logp_difference/max": 22.191104888916016,
      "sampling/sampling_logp_difference/mean": 0.10394315421581268,
      "step": 749,
      "step_time": 12.944917745044222
    },
    {
      "clip_ratio/high_max": 0.020818366901949048,
      "clip_ratio/high_mean": 0.010409183450974524,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010409183450974524,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 1117.0,
      "completions/max_terminated_length": 1117.0,
      "completions/mean_length": 238.46875,
      "completions/mean_terminated_length": 245.69566345214844,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 0.5325307454913855,
      "epoch": 0.015,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8294697999954224,
      "kl": 0.4676121240481734,
      "learning_rate": 3.0499337627672667e-06,
      "loss": 0.2092,
      "num_tokens": 30392857.0,
      "reward": 0.9118750095367432,
      "reward_std": 0.26337453722953796,
      "rewards/rollout_reward_func/mean": 0.9118750095367432,
      "rewards/rollout_reward_func/std": 0.2873172461986542,
      "sampling/importance_sampling_ratio/max": 2.3363659381866455,
      "sampling/importance_sampling_ratio/mean": 0.9679088592529297,
      "sampling/importance_sampling_ratio/min": 3.748330068295255e-18,
      "sampling/sampling_logp_difference/max": 21.06137466430664,
      "sampling/sampling_logp_difference/mean": 0.10137906670570374,
      "step": 750,
      "step_time": 14.68338212594972
    },
    {
      "clip_ratio/high_max": 0.008246527868323028,
      "clip_ratio/high_mean": 0.004123263934161514,
      "clip_ratio/low_mean": 0.005923763965256512,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010047027899418026,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1098.0,
      "completions/max_terminated_length": 665.0,
      "completions/mean_length": 290.25,
      "completions/mean_terminated_length": 225.3181915283203,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4267449760809541,
      "epoch": 0.01502,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.528073787689209,
      "kl": 0.3735023671761155,
      "learning_rate": 3.042854473864327e-06,
      "loss": 0.0079,
      "num_tokens": 30434413.0,
      "reward": 0.6456249952316284,
      "reward_std": 0.5231055021286011,
      "rewards/rollout_reward_func/mean": 0.6456249952316284,
      "rewards/rollout_reward_func/std": 0.5524603724479675,
      "sampling/importance_sampling_ratio/max": 1.9067716598510742,
      "sampling/importance_sampling_ratio/mean": 0.9585108757019043,
      "sampling/importance_sampling_ratio/min": 0.5079618692398071,
      "sampling/sampling_logp_difference/max": 0.6830422878265381,
      "sampling/sampling_logp_difference/mean": 0.0315118208527565,
      "step": 751,
      "step_time": 14.608383285056334
    },
    {
      "clip_ratio/high_max": 0.07281249994412065,
      "clip_ratio/high_mean": 0.03640624997206032,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03640624997206032,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 598.0,
      "completions/max_terminated_length": 581.0,
      "completions/mean_length": 237.9375,
      "completions/mean_terminated_length": 233.68182373046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.46793496888130903,
      "epoch": 0.01504,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8446381092071533,
      "kl": 0.5279836757108569,
      "learning_rate": 3.0358174893789156e-06,
      "loss": -0.1043,
      "num_tokens": 30474984.0,
      "reward": 0.7331655025482178,
      "reward_std": 0.5341328382492065,
      "rewards/rollout_reward_func/mean": 0.7331655025482178,
      "rewards/rollout_reward_func/std": 0.5840622186660767,
      "sampling/importance_sampling_ratio/max": 2.3520376682281494,
      "sampling/importance_sampling_ratio/mean": 0.9722007513046265,
      "sampling/importance_sampling_ratio/min": 8.686988353113527e-24,
      "sampling/sampling_logp_difference/max": 23.497533798217773,
      "sampling/sampling_logp_difference/mean": 0.13999278843402863,
      "step": 752,
      "step_time": 12.36513399798423
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.0010080644860863686,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0049143144860863686,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 690.0,
      "completions/max_terminated_length": 643.0,
      "completions/mean_length": 248.15625,
      "completions/mean_terminated_length": 185.6875,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 0.5192950367927551,
      "epoch": 0.01506,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.006845474243164,
      "kl": 0.33689256478101015,
      "learning_rate": 3.0288229021336165e-06,
      "loss": 0.1465,
      "num_tokens": 30514199.0,
      "reward": 0.8268749713897705,
      "reward_std": 0.40236496925354004,
      "rewards/rollout_reward_func/mean": 0.8268749713897705,
      "rewards/rollout_reward_func/std": 0.3918725550174713,
      "sampling/importance_sampling_ratio/max": 1.5867793560028076,
      "sampling/importance_sampling_ratio/mean": 1.0655133724212646,
      "sampling/importance_sampling_ratio/min": 0.5540239810943604,
      "sampling/sampling_logp_difference/max": 0.6779935359954834,
      "sampling/sampling_logp_difference/mean": 0.033394549041986465,
      "step": 753,
      "step_time": 13.253685446004965
    },
    {
      "clip_ratio/high_max": 0.008221726398915052,
      "clip_ratio/high_mean": 0.004110863199457526,
      "clip_ratio/low_mean": 0.0011160714784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0052269346779212356,
      "completions/clipped_ratio": 0.40625,
      "completions/max_length": 660.0,
      "completions/max_terminated_length": 498.0,
      "completions/mean_length": 258.15625,
      "completions/mean_terminated_length": 181.4736785888672,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5178470863029361,
      "epoch": 0.01508,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3089908361434937,
      "kl": 0.44722865521907806,
      "learning_rate": 3.021870804391763e-06,
      "loss": -0.0353,
      "num_tokens": 30556180.0,
      "reward": 0.6746875047683716,
      "reward_std": 0.6635714769363403,
      "rewards/rollout_reward_func/mean": 0.6746875047683716,
      "rewards/rollout_reward_func/std": 0.6451755166053772,
      "sampling/importance_sampling_ratio/max": 1.5245122909545898,
      "sampling/importance_sampling_ratio/mean": 1.0140845775604248,
      "sampling/importance_sampling_ratio/min": 0.6000978946685791,
      "sampling/sampling_logp_difference/max": 0.46904706954956055,
      "sampling/sampling_logp_difference/mean": 0.03113793581724167,
      "step": 754,
      "step_time": 13.215881446012645
    },
    {
      "clip_ratio/high_max": 0.005859375,
      "clip_ratio/high_mean": 0.0029296875,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00390625,
      "completions/clipped_ratio": 0.28125,
      "completions/max_length": 905.0,
      "completions/max_terminated_length": 905.0,
      "completions/mean_length": 248.40625,
      "completions/mean_terminated_length": 239.86956787109375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.532974174246192,
      "epoch": 0.0151,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2935746908187866,
      "kl": 0.46843178663402796,
      "learning_rate": 3.014961287856224e-06,
      "loss": -0.1697,
      "num_tokens": 30596704.0,
      "reward": 0.7025405168533325,
      "reward_std": 0.606001615524292,
      "rewards/rollout_reward_func/mean": 0.7025405168533325,
      "rewards/rollout_reward_func/std": 0.5970246195793152,
      "sampling/importance_sampling_ratio/max": 1.5356931686401367,
      "sampling/importance_sampling_ratio/mean": 0.9699382185935974,
      "sampling/importance_sampling_ratio/min": 0.5148270726203918,
      "sampling/sampling_logp_difference/max": 0.40901756286621094,
      "sampling/sampling_logp_difference/mean": 0.03312190622091293,
      "step": 755,
      "step_time": 13.093787843958125
    },
    {
      "clip_ratio/high_max": 0.017299107741564512,
      "clip_ratio/high_mean": 0.008649553870782256,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010602678870782256,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 676.0,
      "completions/max_terminated_length": 599.0,
      "completions/mean_length": 214.53125,
      "completions/mean_terminated_length": 153.0,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4583741696551442,
      "epoch": 0.01512,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7622438669204712,
      "kl": 0.3892849422991276,
      "learning_rate": 3.0080944436681924e-06,
      "loss": -0.2615,
      "num_tokens": 30636544.0,
      "reward": 0.7743749618530273,
      "reward_std": 0.4703516364097595,
      "rewards/rollout_reward_func/mean": 0.7743749618530273,
      "rewards/rollout_reward_func/std": 0.5692436099052429,
      "sampling/importance_sampling_ratio/max": 2.37758469581604,
      "sampling/importance_sampling_ratio/mean": 1.0148932933807373,
      "sampling/importance_sampling_ratio/min": 3.5949983963873923e-25,
      "sampling/sampling_logp_difference/max": 22.242101669311523,
      "sampling/sampling_logp_difference/mean": 0.14147859811782837,
      "step": 756,
      "step_time": 13.202697605986032
    },
    {
      "clip_ratio/high_max": 0.008091517956927419,
      "clip_ratio/high_mean": 0.004045758978463709,
      "clip_ratio/low_mean": 0.005022321478463709,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009068080456927419,
      "completions/clipped_ratio": 0.46875,
      "completions/max_length": 692.0,
      "completions/max_terminated_length": 530.0,
      "completions/mean_length": 235.25,
      "completions/mean_terminated_length": 202.58824157714844,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5299225766211748,
      "epoch": 0.01514,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7332797050476074,
      "kl": 0.3640566896647215,
      "learning_rate": 3.001270362405987e-06,
      "loss": 0.0509,
      "num_tokens": 30676391.0,
      "reward": 0.8600000143051147,
      "reward_std": 0.2393033653497696,
      "rewards/rollout_reward_func/mean": 0.8600000143051147,
      "rewards/rollout_reward_func/std": 0.37205618619918823,
      "sampling/importance_sampling_ratio/max": 1.5478515625,
      "sampling/importance_sampling_ratio/mean": 0.9886972904205322,
      "sampling/importance_sampling_ratio/min": 2.1798790609198524e-24,
      "sampling/sampling_logp_difference/max": 23.553796768188477,
      "sampling/sampling_logp_difference/mean": 0.10773153603076935,
      "step": 757,
      "step_time": 13.433438447958906
    },
    {
      "clip_ratio/high_max": 0.017163826152682304,
      "clip_ratio/high_mean": 0.009558475576341152,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009558475576341152,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 665.0,
      "completions/max_terminated_length": 651.0,
      "completions/mean_length": 305.65625,
      "completions/mean_terminated_length": 283.1000061035156,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5083888196386397,
      "epoch": 0.01516,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.314605951309204,
      "kl": 0.36627142038196325,
      "learning_rate": 2.9944891340838517e-06,
      "loss": 0.0014,
      "num_tokens": 30720546.0,
      "reward": 0.6781249642372131,
      "reward_std": 0.5317622423171997,
      "rewards/rollout_reward_func/mean": 0.6781249642372131,
      "rewards/rollout_reward_func/std": 0.5390878319740295,
      "sampling/importance_sampling_ratio/max": 1.4580367803573608,
      "sampling/importance_sampling_ratio/mean": 0.8877754211425781,
      "sampling/importance_sampling_ratio/min": 0.24966230988502502,
      "sampling/sampling_logp_difference/max": 0.6414527893066406,
      "sampling/sampling_logp_difference/mean": 0.03596293926239014,
      "step": 758,
      "step_time": 12.833835147990612
    },
    {
      "clip_ratio/high_max": 0.0048363097012043,
      "clip_ratio/high_mean": 0.00241815485060215,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00241815485060215,
      "completions/clipped_ratio": 0.40625,
      "completions/max_length": 678.0,
      "completions/max_terminated_length": 502.0,
      "completions/mean_length": 237.25,
      "completions/mean_terminated_length": 207.1052703857422,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.5549900718033314,
      "epoch": 0.01518,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0609318017959595,
      "kl": 0.6132746785879135,
      "learning_rate": 2.9877508481507723e-06,
      "loss": -0.1321,
      "num_tokens": 30760826.0,
      "reward": 0.8350000381469727,
      "reward_std": 0.2913733422756195,
      "rewards/rollout_reward_func/mean": 0.8350000381469727,
      "rewards/rollout_reward_func/std": 0.4647302031517029,
      "sampling/importance_sampling_ratio/max": 1.4914497137069702,
      "sampling/importance_sampling_ratio/mean": 0.9391607046127319,
      "sampling/importance_sampling_ratio/min": 0.4164001941680908,
      "sampling/sampling_logp_difference/max": 0.4849429130554199,
      "sampling/sampling_logp_difference/mean": 0.030831169337034225,
      "step": 759,
      "step_time": 13.004410063003888
    },
    {
      "clip_ratio/high_max": 0.0060614224057644606,
      "clip_ratio/high_mean": 0.0030307112028822303,
      "clip_ratio/low_mean": 0.0009765625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00400727370288223,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1197.0,
      "completions/max_terminated_length": 554.0,
      "completions/mean_length": 303.84375,
      "completions/mean_terminated_length": 228.90000915527344,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.4507888052612543,
      "epoch": 0.0152,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1247159242630005,
      "kl": 0.37034228071570396,
      "learning_rate": 2.981055593489295e-06,
      "loss": -0.1284,
      "num_tokens": 30803732.0,
      "reward": 0.5896874666213989,
      "reward_std": 0.6882156133651733,
      "rewards/rollout_reward_func/mean": 0.5896874666213989,
      "rewards/rollout_reward_func/std": 0.6784064769744873,
      "sampling/importance_sampling_ratio/max": 1.9723132848739624,
      "sampling/importance_sampling_ratio/mean": 1.047371506690979,
      "sampling/importance_sampling_ratio/min": 0.5286672711372375,
      "sampling/sampling_logp_difference/max": 0.3673262596130371,
      "sampling/sampling_logp_difference/mean": 0.029805844649672508,
      "step": 760,
      "step_time": 15.53590162305045
    },
    {
      "clip_ratio/high_max": 0.008091517956927419,
      "clip_ratio/high_mean": 0.004045758978463709,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004045758978463709,
      "completions/clipped_ratio": 0.46875,
      "completions/max_length": 653.0,
      "completions/max_terminated_length": 631.0,
      "completions/mean_length": 275.5,
      "completions/mean_terminated_length": 210.47059631347656,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47283880645409226,
      "epoch": 0.01522,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6662938594818115,
      "kl": 0.4737983960658312,
      "learning_rate": 2.974403458414355e-06,
      "loss": 0.0175,
      "num_tokens": 30845529.0,
      "reward": 0.5810002088546753,
      "reward_std": 0.604245662689209,
      "rewards/rollout_reward_func/mean": 0.5810002088546753,
      "rewards/rollout_reward_func/std": 0.6258972883224487,
      "sampling/importance_sampling_ratio/max": 2.135319232940674,
      "sampling/importance_sampling_ratio/mean": 1.030308485031128,
      "sampling/importance_sampling_ratio/min": 0.33862438797950745,
      "sampling/sampling_logp_difference/max": 1.09218168258667,
      "sampling/sampling_logp_difference/mean": 0.03349260613322258,
      "step": 761,
      "step_time": 12.698352941006306
    },
    {
      "clip_ratio/high_max": 0.01518506370484829,
      "clip_ratio/high_mean": 0.007592531852424145,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007592531852424145,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 941.0,
      "completions/max_terminated_length": 941.0,
      "completions/mean_length": 279.71875,
      "completions/mean_terminated_length": 315.20001220703125,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 0.47340640611946583,
      "epoch": 0.01524,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1173207759857178,
      "kl": 0.3223028965294361,
      "learning_rate": 2.9677945306721114e-06,
      "loss": -0.1653,
      "num_tokens": 30888078.0,
      "reward": 0.9084374904632568,
      "reward_std": 0.24815446138381958,
      "rewards/rollout_reward_func/mean": 0.9084374904632568,
      "rewards/rollout_reward_func/std": 0.48936501145362854,
      "sampling/importance_sampling_ratio/max": 1.593660593032837,
      "sampling/importance_sampling_ratio/mean": 0.9522698521614075,
      "sampling/importance_sampling_ratio/min": 2.386536523957261e-10,
      "sampling/sampling_logp_difference/max": 15.61818790435791,
      "sampling/sampling_logp_difference/mean": 0.06465025246143341,
      "step": 762,
      "step_time": 14.505000113029382
    },
    {
      "clip_ratio/high_max": 0.04581634979695082,
      "clip_ratio/high_mean": 0.02290817489847541,
      "clip_ratio/low_mean": 0.0011160714784637094,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.024024246376939118,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 599.0,
      "completions/max_terminated_length": 599.0,
      "completions/mean_length": 225.46875,
      "completions/mean_terminated_length": 193.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49248040467500687,
      "epoch": 0.01526,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.744982123374939,
      "kl": 0.502603062428534,
      "learning_rate": 2.9612288974387883e-06,
      "loss": -0.1559,
      "num_tokens": 30928011.0,
      "reward": 0.6393749713897705,
      "reward_std": 0.5950148701667786,
      "rewards/rollout_reward_func/mean": 0.6393749713897705,
      "rewards/rollout_reward_func/std": 0.6706084609031677,
      "sampling/importance_sampling_ratio/max": 1.9489963054656982,
      "sampling/importance_sampling_ratio/mean": 0.9916138648986816,
      "sampling/importance_sampling_ratio/min": 1.1219682614044705e-17,
      "sampling/sampling_logp_difference/max": 21.909500122070312,
      "sampling/sampling_logp_difference/mean": 0.1564740240573883,
      "step": 763,
      "step_time": 12.527172476999112
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 900,
  "num_input_tokens_seen": 30928011,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}