{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.0095,
  "eval_steps": 500,
  "global_step": 950,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1665.0,
      "completions/max_terminated_length": 1665.0,
      "completions/mean_length": 598.0,
      "completions/mean_terminated_length": 598.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3080205172300339,
      "epoch": 1e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.038385272026062,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": -0.0325,
      "num_tokens": 24563.0,
      "reward": -0.05861438810825348,
      "reward_std": 1.0091829299926758,
      "rewards/rollout_reward_func/mean": -0.05861438810825348,
      "rewards/rollout_reward_func/std": 0.9944182634353638,
      "sampling/importance_sampling_ratio/max": 1.2905676364898682,
      "sampling/importance_sampling_ratio/mean": 1.0094048976898193,
      "sampling/importance_sampling_ratio/min": 0.8202549815177917,
      "sampling/sampling_logp_difference/max": 0.22703981399536133,
      "sampling/sampling_logp_difference/mean": 0.04825877398252487,
      "step": 1,
      "step_time": 10.86158763100002
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 840.0,
      "completions/max_terminated_length": 840.0,
      "completions/mean_length": 449.4375,
      "completions/mean_terminated_length": 449.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.298059806227684,
      "epoch": 2e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4646263122558594,
      "kl": 0.0,
      "learning_rate": 1.225e-06,
      "loss": -0.0362,
      "num_tokens": 46805.0,
      "reward": -0.1876218169927597,
      "reward_std": 0.986554741859436,
      "rewards/rollout_reward_func/mean": -0.1876218169927597,
      "rewards/rollout_reward_func/std": 0.976506233215332,
      "sampling/importance_sampling_ratio/max": 1.453641414642334,
      "sampling/importance_sampling_ratio/mean": 0.9910303950309753,
      "sampling/importance_sampling_ratio/min": 0.566702663898468,
      "sampling/sampling_logp_difference/max": 0.4780527353286743,
      "sampling/sampling_logp_difference/mean": 0.05092613026499748,
      "step": 2,
      "step_time": 7.115287888000239
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2260.0,
      "completions/max_terminated_length": 2260.0,
      "completions/mean_length": 451.0,
      "completions/mean_terminated_length": 451.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.371096596121788,
      "epoch": 3e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8392657041549683,
      "kl": 0.001978808009880595,
      "learning_rate": 2.45e-06,
      "loss": -0.0236,
      "num_tokens": 68931.0,
      "reward": -0.13188211619853973,
      "reward_std": 0.9706485271453857,
      "rewards/rollout_reward_func/mean": -0.13188211619853973,
      "rewards/rollout_reward_func/std": 0.9525101184844971,
      "sampling/importance_sampling_ratio/max": 1.3011143207550049,
      "sampling/importance_sampling_ratio/mean": 1.030664086341858,
      "sampling/importance_sampling_ratio/min": 0.8483749628067017,
      "sampling/sampling_logp_difference/max": 0.1789489984512329,
      "sampling/sampling_logp_difference/mean": 0.046712249517440796,
      "step": 3,
      "step_time": 11.049060311999938
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1693.0,
      "completions/max_terminated_length": 1693.0,
      "completions/mean_length": 475.4375,
      "completions/mean_terminated_length": 475.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.392275646328926,
      "epoch": 4e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2899916172027588,
      "kl": 0.004987280830391683,
      "learning_rate": 3.674999999999999e-06,
      "loss": 0.008,
      "num_tokens": 91511.0,
      "reward": -0.11510545015335083,
      "reward_std": 1.0417895317077637,
      "rewards/rollout_reward_func/mean": -0.11510545015335083,
      "rewards/rollout_reward_func/std": 1.0159754753112793,
      "sampling/importance_sampling_ratio/max": 1.1650457382202148,
      "sampling/importance_sampling_ratio/mean": 0.9780612587928772,
      "sampling/importance_sampling_ratio/min": 0.772137463092804,
      "sampling/sampling_logp_difference/max": 0.2176060676574707,
      "sampling/sampling_logp_difference/mean": 0.03813418745994568,
      "step": 4,
      "step_time": 9.984487735999664
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1675.0,
      "completions/max_terminated_length": 1675.0,
      "completions/mean_length": 485.625,
      "completions/mean_terminated_length": 485.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2597919180989265,
      "epoch": 5e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9500104784965515,
      "kl": 0.0022987909987932653,
      "learning_rate": 4.9e-06,
      "loss": 0.0187,
      "num_tokens": 113858.0,
      "reward": -0.6242519617080688,
      "reward_std": 0.7263472080230713,
      "rewards/rollout_reward_func/mean": -0.6242519617080688,
      "rewards/rollout_reward_func/std": 0.7017263770103455,
      "sampling/importance_sampling_ratio/max": 1.2941312789916992,
      "sampling/importance_sampling_ratio/mean": 0.9646577835083008,
      "sampling/importance_sampling_ratio/min": 0.4677954614162445,
      "sampling/sampling_logp_difference/max": 0.8021426200866699,
      "sampling/sampling_logp_difference/mean": 0.05170299857854843,
      "step": 5,
      "step_time": 9.44691723699998
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1708.0,
      "completions/max_terminated_length": 1708.0,
      "completions/mean_length": 579.9375,
      "completions/mean_terminated_length": 579.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3976583629846573,
      "epoch": 6e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1756339073181152,
      "kl": 0.0033139032020699233,
      "learning_rate": 6.124999999999999e-06,
      "loss": -0.0288,
      "num_tokens": 138133.0,
      "reward": -0.2394159883260727,
      "reward_std": 0.9882635474205017,
      "rewards/rollout_reward_func/mean": -0.2394159883260727,
      "rewards/rollout_reward_func/std": 0.9917852282524109,
      "sampling/importance_sampling_ratio/max": 1.3044286966323853,
      "sampling/importance_sampling_ratio/mean": 0.9652995467185974,
      "sampling/importance_sampling_ratio/min": 0.5838958621025085,
      "sampling/sampling_logp_difference/max": 0.3397185802459717,
      "sampling/sampling_logp_difference/mean": 0.04887489601969719,
      "step": 6,
      "step_time": 9.796152859999893
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1718.0,
      "completions/max_terminated_length": 1718.0,
      "completions/mean_length": 479.0,
      "completions/mean_terminated_length": 479.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3436996340751648,
      "epoch": 7e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3011540174484253,
      "kl": 0.009219749772455543,
      "learning_rate": 7.349999999999998e-06,
      "loss": 0.0036,
      "num_tokens": 160608.0,
      "reward": -0.2618994116783142,
      "reward_std": 0.9248126745223999,
      "rewards/rollout_reward_func/mean": -0.2618994116783142,
      "rewards/rollout_reward_func/std": 0.9250544905662537,
      "sampling/importance_sampling_ratio/max": 1.3645485639572144,
      "sampling/importance_sampling_ratio/mean": 1.0156559944152832,
      "sampling/importance_sampling_ratio/min": 0.4044071435928345,
      "sampling/sampling_logp_difference/max": 0.9214320182800293,
      "sampling/sampling_logp_difference/mean": 0.06430624425411224,
      "step": 7,
      "step_time": 9.47085824900023
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2572.0,
      "completions/max_terminated_length": 2572.0,
      "completions/mean_length": 725.25,
      "completions/mean_terminated_length": 725.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3191779032349586,
      "epoch": 8e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0471835136413574,
      "kl": 0.007690569691476412,
      "learning_rate": 8.574999999999998e-06,
      "loss": -0.0131,
      "num_tokens": 186929.0,
      "reward": -0.18908730149269104,
      "reward_std": 0.8609955906867981,
      "rewards/rollout_reward_func/mean": -0.18908730149269104,
      "rewards/rollout_reward_func/std": 0.9001888036727905,
      "sampling/importance_sampling_ratio/max": 1.212273120880127,
      "sampling/importance_sampling_ratio/mean": 0.929000735282898,
      "sampling/importance_sampling_ratio/min": 0.7371073961257935,
      "sampling/sampling_logp_difference/max": 0.46265602111816406,
      "sampling/sampling_logp_difference/mean": 0.04625009000301361,
      "step": 8,
      "step_time": 12.224600155999951
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3266.0,
      "completions/max_terminated_length": 3266.0,
      "completions/mean_length": 803.25,
      "completions/mean_terminated_length": 803.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2979115545749664,
      "epoch": 9e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.045641541481018,
      "kl": 0.009142367780441418,
      "learning_rate": 9.8e-06,
      "loss": -0.0075,
      "num_tokens": 214545.0,
      "reward": 0.15107432007789612,
      "reward_std": 1.023350715637207,
      "rewards/rollout_reward_func/mean": 0.15107432007789612,
      "rewards/rollout_reward_func/std": 0.9954668283462524,
      "sampling/importance_sampling_ratio/max": 1.6534229516983032,
      "sampling/importance_sampling_ratio/mean": 1.0594488382339478,
      "sampling/importance_sampling_ratio/min": 0.7688047289848328,
      "sampling/sampling_logp_difference/max": 0.4624967575073242,
      "sampling/sampling_logp_difference/mean": 0.050298962742090225,
      "step": 9,
      "step_time": 14.244561664999992
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1710.0,
      "completions/max_terminated_length": 1710.0,
      "completions/mean_length": 481.875,
      "completions/mean_terminated_length": 481.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1939708292484283,
      "epoch": 0.0001,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0899871587753296,
      "kl": 0.06292384451626276,
      "learning_rate": 1.1024999999999999e-05,
      "loss": -0.0182,
      "num_tokens": 236584.0,
      "reward": -0.483462393283844,
      "reward_std": 0.8630062341690063,
      "rewards/rollout_reward_func/mean": -0.483462393283844,
      "rewards/rollout_reward_func/std": 0.8850297927856445,
      "sampling/importance_sampling_ratio/max": 1.1202483177185059,
      "sampling/importance_sampling_ratio/mean": 0.9383817911148071,
      "sampling/importance_sampling_ratio/min": 0.661373496055603,
      "sampling/sampling_logp_difference/max": 0.5146188735961914,
      "sampling/sampling_logp_difference/mean": 0.056227728724479675,
      "step": 10,
      "step_time": 9.494891244999735
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1512.0,
      "completions/max_terminated_length": 1512.0,
      "completions/mean_length": 462.0625,
      "completions/mean_terminated_length": 462.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1955695375800133,
      "epoch": 0.00011,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8720059394836426,
      "kl": 0.09627947781700641,
      "learning_rate": 1.2249999999999998e-05,
      "loss": -0.0175,
      "num_tokens": 258420.0,
      "reward": -0.23854297399520874,
      "reward_std": 0.8606164455413818,
      "rewards/rollout_reward_func/mean": -0.23854297399520874,
      "rewards/rollout_reward_func/std": 0.9910469055175781,
      "sampling/importance_sampling_ratio/max": 1.4608699083328247,
      "sampling/importance_sampling_ratio/mean": 0.9987151622772217,
      "sampling/importance_sampling_ratio/min": 0.41053181886672974,
      "sampling/sampling_logp_difference/max": 0.8121306896209717,
      "sampling/sampling_logp_difference/mean": 0.0706997737288475,
      "step": 11,
      "step_time": 9.633399043999589
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1586.0,
      "completions/max_terminated_length": 1586.0,
      "completions/mean_length": 379.1875,
      "completions/mean_terminated_length": 379.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9605632424354553,
      "epoch": 0.00012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7697734832763672,
      "kl": 0.23501893621869385,
      "learning_rate": 1.3474999999999999e-05,
      "loss": -0.0214,
      "num_tokens": 278798.0,
      "reward": 0.03554739058017731,
      "reward_std": 0.8809672594070435,
      "rewards/rollout_reward_func/mean": 0.03554739058017731,
      "rewards/rollout_reward_func/std": 0.8541486263275146,
      "sampling/importance_sampling_ratio/max": 1.5664547681808472,
      "sampling/importance_sampling_ratio/mean": 0.9929382801055908,
      "sampling/importance_sampling_ratio/min": 0.2364688366651535,
      "sampling/sampling_logp_difference/max": 1.2145824432373047,
      "sampling/sampling_logp_difference/mean": 0.14878182113170624,
      "step": 12,
      "step_time": 9.16907877799963
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1819.0,
      "completions/max_terminated_length": 1819.0,
      "completions/mean_length": 466.8125,
      "completions/mean_terminated_length": 466.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9423728659749031,
      "epoch": 0.00013,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9766358733177185,
      "kl": 0.24916769471019506,
      "learning_rate": 1.4699999999999996e-05,
      "loss": -0.0354,
      "num_tokens": 301078.0,
      "reward": 0.0312364399433136,
      "reward_std": 0.6963824033737183,
      "rewards/rollout_reward_func/mean": 0.0312364399433136,
      "rewards/rollout_reward_func/std": 0.8579856753349304,
      "sampling/importance_sampling_ratio/max": 1.9092484712600708,
      "sampling/importance_sampling_ratio/mean": 1.0199568271636963,
      "sampling/importance_sampling_ratio/min": 0.27713489532470703,
      "sampling/sampling_logp_difference/max": 1.2832708358764648,
      "sampling/sampling_logp_difference/mean": 0.15194204449653625,
      "step": 13,
      "step_time": 9.937238432000186
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1713.0,
      "completions/max_terminated_length": 1713.0,
      "completions/mean_length": 303.875,
      "completions/mean_terminated_length": 303.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6607532240450382,
      "epoch": 0.00014,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4452351331710815,
      "kl": 1.9250693074427545,
      "learning_rate": 1.5924999999999997e-05,
      "loss": -0.0287,
      "num_tokens": 320741.0,
      "reward": 0.030335605144500732,
      "reward_std": 0.8609468936920166,
      "rewards/rollout_reward_func/mean": 0.030335605144500732,
      "rewards/rollout_reward_func/std": 0.8553087711334229,
      "sampling/importance_sampling_ratio/max": 1.5381555557250977,
      "sampling/importance_sampling_ratio/mean": 0.8666203022003174,
      "sampling/importance_sampling_ratio/min": 0.0496089793741703,
      "sampling/sampling_logp_difference/max": 3.0035367012023926,
      "sampling/sampling_logp_difference/mean": 0.16876575350761414,
      "step": 14,
      "step_time": 9.658226060000288
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 795.0,
      "completions/max_terminated_length": 795.0,
      "completions/mean_length": 256.5625,
      "completions/mean_terminated_length": 256.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5569913145154715,
      "epoch": 0.00015,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8759051561355591,
      "kl": 1.1759431147947907,
      "learning_rate": 1.7149999999999997e-05,
      "loss": -0.0225,
      "num_tokens": 339252.0,
      "reward": 0.4302932024002075,
      "reward_std": 0.9245354533195496,
      "rewards/rollout_reward_func/mean": 0.4302932024002075,
      "rewards/rollout_reward_func/std": 0.8961155414581299,
      "sampling/importance_sampling_ratio/max": 2.173036813735962,
      "sampling/importance_sampling_ratio/mean": 0.9290937185287476,
      "sampling/importance_sampling_ratio/min": 0.11926465481519699,
      "sampling/sampling_logp_difference/max": 2.126399278640747,
      "sampling/sampling_logp_difference/mean": 0.16680708527565002,
      "step": 15,
      "step_time": 6.719806952999761
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1701.0,
      "completions/max_terminated_length": 1701.0,
      "completions/mean_length": 437.0,
      "completions/mean_terminated_length": 437.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5592086017131805,
      "epoch": 0.00016,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1694447994232178,
      "kl": 2.243621489033103,
      "learning_rate": 1.8374999999999996e-05,
      "loss": -0.0597,
      "num_tokens": 360906.0,
      "reward": 0.3387101888656616,
      "reward_std": 0.7649751901626587,
      "rewards/rollout_reward_func/mean": 0.3387101888656616,
      "rewards/rollout_reward_func/std": 0.7390375733375549,
      "sampling/importance_sampling_ratio/max": 2.071281671524048,
      "sampling/importance_sampling_ratio/mean": 0.8357038497924805,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.3848016262054443,
      "sampling/sampling_logp_difference/mean": 0.22039607167243958,
      "step": 16,
      "step_time": 9.501191937999693
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 906.0,
      "completions/max_terminated_length": 906.0,
      "completions/mean_length": 478.8125,
      "completions/mean_terminated_length": 478.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4535759277641773,
      "epoch": 0.00017,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3703463077545166,
      "kl": 2.9010237026959658,
      "learning_rate": 1.96e-05,
      "loss": -0.0561,
      "num_tokens": 383365.0,
      "reward": 0.4087403416633606,
      "reward_std": 0.7456482648849487,
      "rewards/rollout_reward_func/mean": 0.4087403416633606,
      "rewards/rollout_reward_func/std": 0.7470653057098389,
      "sampling/importance_sampling_ratio/max": 2.2482383251190186,
      "sampling/importance_sampling_ratio/mean": 1.06211519241333,
      "sampling/importance_sampling_ratio/min": 0.04496994987130165,
      "sampling/sampling_logp_difference/max": 3.3672637939453125,
      "sampling/sampling_logp_difference/mean": 0.1919589638710022,
      "step": 17,
      "step_time": 7.4769104889996925
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 777.0,
      "completions/max_terminated_length": 777.0,
      "completions/mean_length": 292.25,
      "completions/mean_terminated_length": 292.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2912004040554166,
      "epoch": 0.00018,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3508421778678894,
      "kl": 0.5862631350755692,
      "learning_rate": 2.0824999999999995e-05,
      "loss": -0.0568,
      "num_tokens": 402915.0,
      "reward": 0.5995998382568359,
      "reward_std": 0.5810286998748779,
      "rewards/rollout_reward_func/mean": 0.5995998382568359,
      "rewards/rollout_reward_func/std": 0.6475933194160461,
      "sampling/importance_sampling_ratio/max": 1.656830072402954,
      "sampling/importance_sampling_ratio/mean": 1.0145221948623657,
      "sampling/importance_sampling_ratio/min": 0.285581111907959,
      "sampling/sampling_logp_difference/max": 1.2426962852478027,
      "sampling/sampling_logp_difference/mean": 0.09370909631252289,
      "step": 18,
      "step_time": 7.272644155999615
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 940.0,
      "completions/max_terminated_length": 940.0,
      "completions/mean_length": 554.5,
      "completions/mean_terminated_length": 554.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4010298140347004,
      "epoch": 0.00019,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 25.799972534179688,
      "kl": 44.40031658858061,
      "learning_rate": 2.2049999999999997e-05,
      "loss": 0.0968,
      "num_tokens": 426923.0,
      "reward": 0.5431992411613464,
      "reward_std": 0.6521517038345337,
      "rewards/rollout_reward_func/mean": 0.5431992411613464,
      "rewards/rollout_reward_func/std": 0.6403537392616272,
      "sampling/importance_sampling_ratio/max": 1.4492627382278442,
      "sampling/importance_sampling_ratio/mean": 0.6575343608856201,
      "sampling/importance_sampling_ratio/min": 0.04778499901294708,
      "sampling/sampling_logp_difference/max": 2.8215723037719727,
      "sampling/sampling_logp_difference/mean": 0.20963677763938904,
      "step": 19,
      "step_time": 7.815221583999801
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 843.0,
      "completions/max_terminated_length": 843.0,
      "completions/mean_length": 302.0625,
      "completions/mean_terminated_length": 302.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3009457290172577,
      "epoch": 0.0002,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.806941986083984,
      "kl": 14.886500734835863,
      "learning_rate": 2.3274999999999997e-05,
      "loss": -0.0372,
      "num_tokens": 446823.0,
      "reward": 0.2684488296508789,
      "reward_std": 0.744430661201477,
      "rewards/rollout_reward_func/mean": 0.2684488296508789,
      "rewards/rollout_reward_func/std": 0.7227689027786255,
      "sampling/importance_sampling_ratio/max": 1.4680366516113281,
      "sampling/importance_sampling_ratio/mean": 0.8125742673873901,
      "sampling/importance_sampling_ratio/min": 0.022621002048254013,
      "sampling/sampling_logp_difference/max": 3.088869094848633,
      "sampling/sampling_logp_difference/mean": 0.21233156323432922,
      "step": 20,
      "step_time": 6.96117387199979
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1595.0,
      "completions/max_terminated_length": 1595.0,
      "completions/mean_length": 349.5,
      "completions/mean_terminated_length": 349.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.30600353982299566,
      "epoch": 0.00021,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4448719024658203,
      "kl": 3.468205627053976,
      "learning_rate": 2.4499999999999996e-05,
      "loss": 0.017,
      "num_tokens": 466415.0,
      "reward": 0.46918609738349915,
      "reward_std": 0.6053762435913086,
      "rewards/rollout_reward_func/mean": 0.46918609738349915,
      "rewards/rollout_reward_func/std": 0.6526917815208435,
      "sampling/importance_sampling_ratio/max": 1.1725571155548096,
      "sampling/importance_sampling_ratio/mean": 0.8670682311058044,
      "sampling/importance_sampling_ratio/min": 0.015594758093357086,
      "sampling/sampling_logp_difference/max": 2.889758586883545,
      "sampling/sampling_logp_difference/mean": 0.15047253668308258,
      "step": 21,
      "step_time": 9.158445935000145
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2457.0,
      "completions/max_terminated_length": 2457.0,
      "completions/mean_length": 449.0625,
      "completions/mean_terminated_length": 449.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.33277699910104275,
      "epoch": 0.00022,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6454811096191406,
      "kl": 0.7181409858167171,
      "learning_rate": 2.5724999999999995e-05,
      "loss": 0.0168,
      "num_tokens": 488073.0,
      "reward": 0.5311313271522522,
      "reward_std": 0.6715220212936401,
      "rewards/rollout_reward_func/mean": 0.5311313271522522,
      "rewards/rollout_reward_func/std": 0.6575800776481628,
      "sampling/importance_sampling_ratio/max": 1.400679111480713,
      "sampling/importance_sampling_ratio/mean": 0.8541733026504517,
      "sampling/importance_sampling_ratio/min": 0.03655528649687767,
      "sampling/sampling_logp_difference/max": 1.8843355178833008,
      "sampling/sampling_logp_difference/mean": 0.15945179760456085,
      "step": 22,
      "step_time": 11.456809925000016
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 870.0,
      "completions/max_terminated_length": 870.0,
      "completions/mean_length": 467.125,
      "completions/mean_terminated_length": 467.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3949324041604996,
      "epoch": 0.00023,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5914931297302246,
      "kl": 0.9177547171711922,
      "learning_rate": 2.6949999999999998e-05,
      "loss": -0.0025,
      "num_tokens": 510102.0,
      "reward": 0.5344709157943726,
      "reward_std": 0.6339346170425415,
      "rewards/rollout_reward_func/mean": 0.5344709157943726,
      "rewards/rollout_reward_func/std": 0.6568873524665833,
      "sampling/importance_sampling_ratio/max": 1.5991919040679932,
      "sampling/importance_sampling_ratio/mean": 0.9060454368591309,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9394950866699219,
      "sampling/sampling_logp_difference/mean": 0.08896666765213013,
      "step": 23,
      "step_time": 7.334103751000157
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 789.0,
      "completions/max_terminated_length": 789.0,
      "completions/mean_length": 302.8125,
      "completions/mean_terminated_length": 302.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3776713814586401,
      "epoch": 0.00024,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7633317708969116,
      "kl": 0.4430163484066725,
      "learning_rate": 2.8174999999999994e-05,
      "loss": 0.0746,
      "num_tokens": 528303.0,
      "reward": 0.7979209423065186,
      "reward_std": 0.4378731846809387,
      "rewards/rollout_reward_func/mean": 0.7979209423065186,
      "rewards/rollout_reward_func/std": 0.4346418082714081,
      "sampling/importance_sampling_ratio/max": 2.7240328788757324,
      "sampling/importance_sampling_ratio/mean": 1.127419352531433,
      "sampling/importance_sampling_ratio/min": 0.6075409054756165,
      "sampling/sampling_logp_difference/max": 0.4775906801223755,
      "sampling/sampling_logp_difference/mean": 0.07646773010492325,
      "step": 24,
      "step_time": 7.320146713000213
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 904.0,
      "completions/max_terminated_length": 904.0,
      "completions/mean_length": 598.125,
      "completions/mean_terminated_length": 598.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4804343320429325,
      "epoch": 0.00025,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.645679235458374,
      "kl": 0.5101686120033264,
      "learning_rate": 2.9399999999999993e-05,
      "loss": 0.0102,
      "num_tokens": 553400.0,
      "reward": 0.4887513518333435,
      "reward_std": 0.7357117533683777,
      "rewards/rollout_reward_func/mean": 0.4887513518333435,
      "rewards/rollout_reward_func/std": 0.7241907715797424,
      "sampling/importance_sampling_ratio/max": 1.506259560585022,
      "sampling/importance_sampling_ratio/mean": 0.8302236795425415,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.0239880084991455,
      "sampling/sampling_logp_difference/mean": 0.10860125720500946,
      "step": 25,
      "step_time": 7.859662122999907
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 865.0,
      "completions/max_terminated_length": 865.0,
      "completions/mean_length": 553.6875,
      "completions/mean_terminated_length": 553.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4896836206316948,
      "epoch": 0.00026,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6009646058082581,
      "kl": 1.4081623312085867,
      "learning_rate": 3.062499999999999e-05,
      "loss": 0.0071,
      "num_tokens": 576934.0,
      "reward": 0.6779468655586243,
      "reward_std": 0.5660915970802307,
      "rewards/rollout_reward_func/mean": 0.6779468655586243,
      "rewards/rollout_reward_func/std": 0.6121453046798706,
      "sampling/importance_sampling_ratio/max": 1.5880839824676514,
      "sampling/importance_sampling_ratio/mean": 0.940824031829834,
      "sampling/importance_sampling_ratio/min": 0.3553817570209503,
      "sampling/sampling_logp_difference/max": 0.9277915954589844,
      "sampling/sampling_logp_difference/mean": 0.10550596565008163,
      "step": 26,
      "step_time": 7.488610232999918
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1917.0,
      "completions/max_terminated_length": 1917.0,
      "completions/mean_length": 390.3125,
      "completions/mean_terminated_length": 390.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.621343556791544,
      "epoch": 0.00027,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5723604559898376,
      "kl": 0.5664136484265327,
      "learning_rate": 3.1849999999999995e-05,
      "loss": 0.0086,
      "num_tokens": 597269.0,
      "reward": 0.3470344841480255,
      "reward_std": 0.7020057439804077,
      "rewards/rollout_reward_func/mean": 0.3470344841480255,
      "rewards/rollout_reward_func/std": 0.7327490448951721,
      "sampling/importance_sampling_ratio/max": 1.8454619646072388,
      "sampling/importance_sampling_ratio/mean": 0.9344164133071899,
      "sampling/importance_sampling_ratio/min": 0.31967291235923767,
      "sampling/sampling_logp_difference/max": 1.2733738422393799,
      "sampling/sampling_logp_difference/mean": 0.08395013213157654,
      "step": 27,
      "step_time": 9.942282175999935
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 876.0,
      "completions/max_terminated_length": 876.0,
      "completions/mean_length": 342.5625,
      "completions/mean_terminated_length": 342.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5975158773362637,
      "epoch": 0.00028,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6731982827186584,
      "kl": 0.4925971031188965,
      "learning_rate": 3.3075e-05,
      "loss": -0.0114,
      "num_tokens": 616475.0,
      "reward": 0.6804916262626648,
      "reward_std": 0.73275226354599,
      "rewards/rollout_reward_func/mean": 0.6804916262626648,
      "rewards/rollout_reward_func/std": 0.7119678258895874,
      "sampling/importance_sampling_ratio/max": 2.1998181343078613,
      "sampling/importance_sampling_ratio/mean": 0.9889349937438965,
      "sampling/importance_sampling_ratio/min": 0.48963552713394165,
      "sampling/sampling_logp_difference/max": 0.7888109683990479,
      "sampling/sampling_logp_difference/mean": 0.09927807748317719,
      "step": 28,
      "step_time": 7.0954170029997385
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1612.0,
      "completions/max_terminated_length": 1612.0,
      "completions/mean_length": 621.8125,
      "completions/mean_terminated_length": 621.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6134510859847069,
      "epoch": 0.00029,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.762920618057251,
      "kl": 0.5710562765598297,
      "learning_rate": 3.4299999999999993e-05,
      "loss": 0.0466,
      "num_tokens": 641281.0,
      "reward": 0.33794325590133667,
      "reward_std": 0.6528863906860352,
      "rewards/rollout_reward_func/mean": 0.33794325590133667,
      "rewards/rollout_reward_func/std": 0.6448003053665161,
      "sampling/importance_sampling_ratio/max": 1.5846138000488281,
      "sampling/importance_sampling_ratio/mean": 0.884196400642395,
      "sampling/importance_sampling_ratio/min": 0.4572957456111908,
      "sampling/sampling_logp_difference/max": 0.7598071098327637,
      "sampling/sampling_logp_difference/mean": 0.06531920284032822,
      "step": 29,
      "step_time": 9.987479473000349
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1781.0,
      "completions/max_terminated_length": 1781.0,
      "completions/mean_length": 499.9375,
      "completions/mean_terminated_length": 499.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6033477820456028,
      "epoch": 0.0003,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4928399324417114,
      "kl": 0.40737907588481903,
      "learning_rate": 3.5524999999999996e-05,
      "loss": 0.0074,
      "num_tokens": 663403.0,
      "reward": 0.49684274196624756,
      "reward_std": 0.8246954083442688,
      "rewards/rollout_reward_func/mean": 0.49684274196624756,
      "rewards/rollout_reward_func/std": 0.8102647066116333,
      "sampling/importance_sampling_ratio/max": 1.3956857919692993,
      "sampling/importance_sampling_ratio/mean": 0.9653814435005188,
      "sampling/importance_sampling_ratio/min": 0.5582724213600159,
      "sampling/sampling_logp_difference/max": 0.597130298614502,
      "sampling/sampling_logp_difference/mean": 0.06495120376348495,
      "step": 30,
      "step_time": 9.897962877000055
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 933.0,
      "completions/max_terminated_length": 933.0,
      "completions/mean_length": 375.4375,
      "completions/mean_terminated_length": 375.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5221531689167023,
      "epoch": 0.00031,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3775498867034912,
      "kl": 1.1993778832256794,
      "learning_rate": 3.674999999999999e-05,
      "loss": -0.0048,
      "num_tokens": 683577.0,
      "reward": 0.4816943109035492,
      "reward_std": 0.7497761845588684,
      "rewards/rollout_reward_func/mean": 0.4816943109035492,
      "rewards/rollout_reward_func/std": 0.7373611927032471,
      "sampling/importance_sampling_ratio/max": 1.1061304807662964,
      "sampling/importance_sampling_ratio/mean": 0.9316405057907104,
      "sampling/importance_sampling_ratio/min": 0.4876435101032257,
      "sampling/sampling_logp_difference/max": 0.7178101539611816,
      "sampling/sampling_logp_difference/mean": 0.05630192160606384,
      "step": 31,
      "step_time": 7.860200859000088
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1657.0,
      "completions/max_terminated_length": 1657.0,
      "completions/mean_length": 521.875,
      "completions/mean_terminated_length": 521.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5511312112212181,
      "epoch": 0.00032,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4919484257698059,
      "kl": 1.0328901298344135,
      "learning_rate": 3.7974999999999995e-05,
      "loss": -0.0228,
      "num_tokens": 706936.0,
      "reward": 0.39660215377807617,
      "reward_std": 0.6477394700050354,
      "rewards/rollout_reward_func/mean": 0.39660215377807617,
      "rewards/rollout_reward_func/std": 0.6607232689857483,
      "sampling/importance_sampling_ratio/max": 1.4525734186172485,
      "sampling/importance_sampling_ratio/mean": 0.9441134929656982,
      "sampling/importance_sampling_ratio/min": 0.4576497972011566,
      "sampling/sampling_logp_difference/max": 0.8632080554962158,
      "sampling/sampling_logp_difference/mean": 0.0775725319981575,
      "step": 32,
      "step_time": 9.492894405999778
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 718.0,
      "completions/max_terminated_length": 718.0,
      "completions/mean_length": 314.6875,
      "completions/mean_terminated_length": 314.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4711787812411785,
      "epoch": 0.00033,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.23085135221481323,
      "kl": 0.3134110625833273,
      "learning_rate": 3.92e-05,
      "loss": -0.0226,
      "num_tokens": 726479.0,
      "reward": 0.7384822368621826,
      "reward_std": 0.5699539184570312,
      "rewards/rollout_reward_func/mean": 0.7384822368621826,
      "rewards/rollout_reward_func/std": 0.5892293453216553,
      "sampling/importance_sampling_ratio/max": 1.3099726438522339,
      "sampling/importance_sampling_ratio/mean": 0.9683486819267273,
      "sampling/importance_sampling_ratio/min": 0.524747371673584,
      "sampling/sampling_logp_difference/max": 0.5525143146514893,
      "sampling/sampling_logp_difference/mean": 0.04434860870242119,
      "step": 33,
      "step_time": 6.692682208000406
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 972.0,
      "completions/max_terminated_length": 972.0,
      "completions/mean_length": 527.125,
      "completions/mean_terminated_length": 527.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6313586793839931,
      "epoch": 0.00034,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.159066915512085,
      "kl": 0.6591682471334934,
      "learning_rate": 4.042499999999999e-05,
      "loss": 0.1176,
      "num_tokens": 749513.0,
      "reward": 0.6087590456008911,
      "reward_std": 0.5673061609268188,
      "rewards/rollout_reward_func/mean": 0.6087590456008911,
      "rewards/rollout_reward_func/std": 0.6344466805458069,
      "sampling/importance_sampling_ratio/max": 1.1417256593704224,
      "sampling/importance_sampling_ratio/mean": 0.9860949516296387,
      "sampling/importance_sampling_ratio/min": 0.7021384835243225,
      "sampling/sampling_logp_difference/max": 0.26265788078308105,
      "sampling/sampling_logp_difference/mean": 0.03283798694610596,
      "step": 34,
      "step_time": 8.009743990999823
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1891.0,
      "completions/max_terminated_length": 826.0,
      "completions/mean_length": 558.625,
      "completions/mean_terminated_length": 501.0714416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7118250764906406,
      "epoch": 0.00035,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1274045705795288,
      "kl": 0.3033441980369389,
      "learning_rate": 4.164999999999999e-05,
      "loss": -0.0977,
      "num_tokens": 773294.0,
      "reward": 0.6734850406646729,
      "reward_std": 0.5780217051506042,
      "rewards/rollout_reward_func/mean": 0.6734850406646729,
      "rewards/rollout_reward_func/std": 0.6196912527084351,
      "sampling/importance_sampling_ratio/max": 1.6351478099822998,
      "sampling/importance_sampling_ratio/mean": 1.0265944004058838,
      "sampling/importance_sampling_ratio/min": 0.6987342238426208,
      "sampling/sampling_logp_difference/max": 0.4965474605560303,
      "sampling/sampling_logp_difference/mean": 0.038381777703762054,
      "step": 35,
      "step_time": 10.923112602999481
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 942.0,
      "completions/max_terminated_length": 942.0,
      "completions/mean_length": 449.875,
      "completions/mean_terminated_length": 449.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5922442898154259,
      "epoch": 0.00036,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.515069007873535,
      "kl": 1.1636196691542864,
      "learning_rate": 4.287499999999999e-05,
      "loss": 0.1336,
      "num_tokens": 794457.0,
      "reward": 0.5969337224960327,
      "reward_std": 0.5355220437049866,
      "rewards/rollout_reward_func/mean": 0.5969337224960327,
      "rewards/rollout_reward_func/std": 0.5376538634300232,
      "sampling/importance_sampling_ratio/max": 2.7776551246643066,
      "sampling/importance_sampling_ratio/mean": 1.0985993146896362,
      "sampling/importance_sampling_ratio/min": 0.8103693127632141,
      "sampling/sampling_logp_difference/max": 0.5289599895477295,
      "sampling/sampling_logp_difference/mean": 0.03776140138506889,
      "step": 36,
      "step_time": 7.478194834999385
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 928.0,
      "completions/max_terminated_length": 923.0,
      "completions/mean_length": 706.6875,
      "completions/mean_terminated_length": 691.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6826450601220131,
      "epoch": 0.00037,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.269562005996704,
      "kl": 0.5718726636841893,
      "learning_rate": 4.4099999999999995e-05,
      "loss": 0.1026,
      "num_tokens": 821328.0,
      "reward": 0.34161823987960815,
      "reward_std": 0.6536663770675659,
      "rewards/rollout_reward_func/mean": 0.34161823987960815,
      "rewards/rollout_reward_func/std": 0.6386427283287048,
      "sampling/importance_sampling_ratio/max": 2.9449622631073,
      "sampling/importance_sampling_ratio/mean": 1.0585622787475586,
      "sampling/importance_sampling_ratio/min": 0.5020780563354492,
      "sampling/sampling_logp_difference/max": 0.6694705486297607,
      "sampling/sampling_logp_difference/mean": 0.03489416092634201,
      "step": 37,
      "step_time": 8.73645642300039
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 933.0,
      "completions/max_terminated_length": 933.0,
      "completions/mean_length": 441.0625,
      "completions/mean_terminated_length": 426.8000183105469,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7703386694192886,
      "epoch": 0.00038,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5401936769485474,
      "kl": 0.6008347198367119,
      "learning_rate": 4.5325e-05,
      "loss": -0.0861,
      "num_tokens": 842481.0,
      "reward": 0.7979676723480225,
      "reward_std": 0.43857914209365845,
      "rewards/rollout_reward_func/mean": 0.7979676723480225,
      "rewards/rollout_reward_func/std": 0.43441838026046753,
      "sampling/importance_sampling_ratio/max": 1.2106308937072754,
      "sampling/importance_sampling_ratio/mean": 0.9422169327735901,
      "sampling/importance_sampling_ratio/min": 0.7520672678947449,
      "sampling/sampling_logp_difference/max": 0.30022406578063965,
      "sampling/sampling_logp_difference/mean": 0.03377055004239082,
      "step": 38,
      "step_time": 8.26117679999993
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2814.0,
      "completions/max_terminated_length": 926.0,
      "completions/mean_length": 577.6875,
      "completions/mean_terminated_length": 428.60003662109375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8957392647862434,
      "epoch": 0.00039,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.590280294418335,
      "kl": 0.40511392802000046,
      "learning_rate": 4.654999999999999e-05,
      "loss": -0.0182,
      "num_tokens": 867143.0,
      "reward": 0.3674936890602112,
      "reward_std": 0.8284348249435425,
      "rewards/rollout_reward_func/mean": 0.3674936890602112,
      "rewards/rollout_reward_func/std": 0.8045759201049805,
      "sampling/importance_sampling_ratio/max": 1.2228741645812988,
      "sampling/importance_sampling_ratio/mean": 0.8287155628204346,
      "sampling/importance_sampling_ratio/min": 4.647326336194014e-17,
      "sampling/sampling_logp_difference/max": 24.3472957611084,
      "sampling/sampling_logp_difference/mean": 0.1857939064502716,
      "step": 39,
      "step_time": 13.717644317999884
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3627.0,
      "completions/max_terminated_length": 3627.0,
      "completions/mean_length": 557.1875,
      "completions/mean_terminated_length": 557.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5529447235167027,
      "epoch": 0.0004,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3544224500656128,
      "kl": 0.3302403874695301,
      "learning_rate": 4.777499999999999e-05,
      "loss": -0.0213,
      "num_tokens": 889726.0,
      "reward": 0.616402268409729,
      "reward_std": 0.6316455006599426,
      "rewards/rollout_reward_func/mean": 0.616402268409729,
      "rewards/rollout_reward_func/std": 0.6298655867576599,
      "sampling/importance_sampling_ratio/max": 2.4867372512817383,
      "sampling/importance_sampling_ratio/mean": 1.0334075689315796,
      "sampling/importance_sampling_ratio/min": 0.4860857427120209,
      "sampling/sampling_logp_difference/max": 0.6889362335205078,
      "sampling/sampling_logp_difference/mean": 0.047486551105976105,
      "step": 40,
      "step_time": 15.371656973999507
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 970.0,
      "completions/max_terminated_length": 970.0,
      "completions/mean_length": 501.875,
      "completions/mean_terminated_length": 470.5833435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9360513836145401,
      "epoch": 0.00041,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3271543979644775,
      "kl": 0.23892609681934118,
      "learning_rate": 4.899999999999999e-05,
      "loss": 0.0179,
      "num_tokens": 912576.0,
      "reward": 0.5625379681587219,
      "reward_std": 0.7994391918182373,
      "rewards/rollout_reward_func/mean": 0.5625379681587219,
      "rewards/rollout_reward_func/std": 0.8049833178520203,
      "sampling/importance_sampling_ratio/max": 2.58750581741333,
      "sampling/importance_sampling_ratio/mean": 0.9478577375411987,
      "sampling/importance_sampling_ratio/min": 2.8898001613181235e-12,
      "sampling/sampling_logp_difference/max": 23.162309646606445,
      "sampling/sampling_logp_difference/mean": 0.11302236467599869,
      "step": 41,
      "step_time": 9.409655995000094
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 979.0,
      "completions/max_terminated_length": 824.0,
      "completions/mean_length": 399.4375,
      "completions/mean_terminated_length": 337.21429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.47894011810421944,
      "epoch": 0.00042,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6225797533988953,
      "kl": 0.5315571632236242,
      "learning_rate": 4.8999875900067906e-05,
      "loss": 0.0942,
      "num_tokens": 933183.0,
      "reward": 0.5274806022644043,
      "reward_std": 0.5687415599822998,
      "rewards/rollout_reward_func/mean": 0.5274806022644043,
      "rewards/rollout_reward_func/std": 0.5537037253379822,
      "sampling/importance_sampling_ratio/max": 1.0640567541122437,
      "sampling/importance_sampling_ratio/mean": 0.9238606691360474,
      "sampling/importance_sampling_ratio/min": 0.6170793175697327,
      "sampling/sampling_logp_difference/max": 0.24500274658203125,
      "sampling/sampling_logp_difference/mean": 0.01961543597280979,
      "step": 42,
      "step_time": 8.153204159000097
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 941.0,
      "completions/max_terminated_length": 941.0,
      "completions/mean_length": 535.125,
      "completions/mean_terminated_length": 568.6666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7527408394962549,
      "epoch": 0.00043,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2309379577636719,
      "kl": 0.5777555350214243,
      "learning_rate": 4.899950360175073e-05,
      "loss": -0.1131,
      "num_tokens": 956605.0,
      "reward": 0.3368086814880371,
      "reward_std": 0.6458407640457153,
      "rewards/rollout_reward_func/mean": 0.3368086814880371,
      "rewards/rollout_reward_func/std": 0.6419474482536316,
      "sampling/importance_sampling_ratio/max": 1.9890867471694946,
      "sampling/importance_sampling_ratio/mean": 0.9896240234375,
      "sampling/importance_sampling_ratio/min": 0.597228467464447,
      "sampling/sampling_logp_difference/max": 0.5041422843933105,
      "sampling/sampling_logp_difference/mean": 0.03504706174135208,
      "step": 43,
      "step_time": 8.167885456999784
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1651.0,
      "completions/max_terminated_length": 1651.0,
      "completions/mean_length": 648.6875,
      "completions/mean_terminated_length": 610.7692260742188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6924648731946945,
      "epoch": 0.00044,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.880660057067871,
      "kl": 0.2628351836465299,
      "learning_rate": 4.899888310948563e-05,
      "loss": 0.0073,
      "num_tokens": 982056.0,
      "reward": 0.5512905716896057,
      "reward_std": 0.7537702322006226,
      "rewards/rollout_reward_func/mean": 0.5512905716896057,
      "rewards/rollout_reward_func/std": 0.7312742471694946,
      "sampling/importance_sampling_ratio/max": 1.3531559705734253,
      "sampling/importance_sampling_ratio/mean": 0.9645261764526367,
      "sampling/importance_sampling_ratio/min": 0.717686653137207,
      "sampling/sampling_logp_difference/max": 0.3798220157623291,
      "sampling/sampling_logp_difference/mean": 0.031422290951013565,
      "step": 44,
      "step_time": 11.120789200999525
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2551.0,
      "completions/max_terminated_length": 887.0,
      "completions/mean_length": 535.8125,
      "completions/mean_terminated_length": 338.0833435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8317118249833584,
      "epoch": 0.00045,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5386226177215576,
      "kl": 1.2103287670761347,
      "learning_rate": 4.899801443066788e-05,
      "loss": 0.1976,
      "num_tokens": 1005400.0,
      "reward": 0.34260016679763794,
      "reward_std": 0.7474086880683899,
      "rewards/rollout_reward_func/mean": 0.34260016679763794,
      "rewards/rollout_reward_func/std": 0.7355738878250122,
      "sampling/importance_sampling_ratio/max": 2.5894763469696045,
      "sampling/importance_sampling_ratio/mean": 0.968580424785614,
      "sampling/importance_sampling_ratio/min": 7.410675373660113e-16,
      "sampling/sampling_logp_difference/max": 11.054435729980469,
      "sampling/sampling_logp_difference/mean": 0.15481340885162354,
      "step": 45,
      "step_time": 13.708368986999858
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1015.0,
      "completions/max_terminated_length": 887.0,
      "completions/mean_length": 346.0625,
      "completions/mean_terminated_length": 310.16668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6836047247052193,
      "epoch": 0.00046,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6661243438720703,
      "kl": 1.0819770283997059,
      "learning_rate": 4.899689757565068e-05,
      "loss": 0.2467,
      "num_tokens": 1024919.0,
      "reward": 0.725571870803833,
      "reward_std": 0.4764586389064789,
      "rewards/rollout_reward_func/mean": 0.725571870803833,
      "rewards/rollout_reward_func/std": 0.4910126030445099,
      "sampling/importance_sampling_ratio/max": 1.8568933010101318,
      "sampling/importance_sampling_ratio/mean": 1.1580030918121338,
      "sampling/importance_sampling_ratio/min": 0.7384299039840698,
      "sampling/sampling_logp_difference/max": 0.6984987258911133,
      "sampling/sampling_logp_difference/mean": 0.03216823562979698,
      "step": 46,
      "step_time": 8.744630648000111
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1820.0,
      "completions/max_terminated_length": 1820.0,
      "completions/mean_length": 334.25,
      "completions/mean_terminated_length": 404.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6961706299334764,
      "epoch": 0.00047,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.074855089187622,
      "kl": 0.6668054722249508,
      "learning_rate": 4.899553255774513e-05,
      "loss": 0.106,
      "num_tokens": 1044596.0,
      "reward": 0.11337269842624664,
      "reward_std": 0.8003221750259399,
      "rewards/rollout_reward_func/mean": 0.11337269842624664,
      "rewards/rollout_reward_func/std": 0.8761861324310303,
      "sampling/importance_sampling_ratio/max": 1.0713557004928589,
      "sampling/importance_sampling_ratio/mean": 0.8411636352539062,
      "sampling/importance_sampling_ratio/min": 0.5242186188697815,
      "sampling/sampling_logp_difference/max": 0.5514333248138428,
      "sampling/sampling_logp_difference/mean": 0.04574333503842354,
      "step": 47,
      "step_time": 10.327997164000635
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 959.0,
      "completions/max_terminated_length": 959.0,
      "completions/mean_length": 443.1875,
      "completions/mean_terminated_length": 470.60003662109375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3509393408894539,
      "epoch": 0.00048,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6886265277862549,
      "kl": 0.6169053837656975,
      "learning_rate": 4.899391939322001e-05,
      "loss": 0.0398,
      "num_tokens": 1066624.0,
      "reward": 0.45137301087379456,
      "reward_std": 0.5108827948570251,
      "rewards/rollout_reward_func/mean": 0.45137301087379456,
      "rewards/rollout_reward_func/std": 0.5668125152587891,
      "sampling/importance_sampling_ratio/max": 1.2540909051895142,
      "sampling/importance_sampling_ratio/mean": 1.0035295486450195,
      "sampling/importance_sampling_ratio/min": 0.5070871710777283,
      "sampling/sampling_logp_difference/max": 0.6772793531417847,
      "sampling/sampling_logp_difference/mean": 0.018441010266542435,
      "step": 48,
      "step_time": 8.325998215000254
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 843.0,
      "completions/max_terminated_length": 843.0,
      "completions/mean_length": 457.5,
      "completions/mean_terminated_length": 457.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3180043399333954,
      "epoch": 0.00049,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8612408638000488,
      "kl": 1.393201183527708,
      "learning_rate": 4.899205810130158e-05,
      "loss": 0.0067,
      "num_tokens": 1089017.0,
      "reward": 0.6563845872879028,
      "reward_std": 0.5398322343826294,
      "rewards/rollout_reward_func/mean": 0.6563845872879028,
      "rewards/rollout_reward_func/std": 0.5264623761177063,
      "sampling/importance_sampling_ratio/max": 1.1634191274642944,
      "sampling/importance_sampling_ratio/mean": 0.9531686305999756,
      "sampling/importance_sampling_ratio/min": 0.589726448059082,
      "sampling/sampling_logp_difference/max": 0.55267333984375,
      "sampling/sampling_logp_difference/mean": 0.017122317105531693,
      "step": 49,
      "step_time": 7.616170871999884
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 915.0,
      "completions/max_terminated_length": 915.0,
      "completions/mean_length": 496.75,
      "completions/mean_terminated_length": 475.5333557128906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4309030883014202,
      "epoch": 0.0005,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.44352805614471436,
      "kl": 0.4947665147483349,
      "learning_rate": 4.898994870417341e-05,
      "loss": -0.0589,
      "num_tokens": 1111700.0,
      "reward": 0.4130287766456604,
      "reward_std": 0.7640472054481506,
      "rewards/rollout_reward_func/mean": 0.4130287766456604,
      "rewards/rollout_reward_func/std": 0.7414007782936096,
      "sampling/importance_sampling_ratio/max": 1.1533663272857666,
      "sampling/importance_sampling_ratio/mean": 1.005097508430481,
      "sampling/importance_sampling_ratio/min": 0.8511068820953369,
      "sampling/sampling_logp_difference/max": 0.1596064567565918,
      "sampling/sampling_logp_difference/mean": 0.01745297573506832,
      "step": 50,
      "step_time": 7.920196226999906
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 937.0,
      "completions/max_terminated_length": 937.0,
      "completions/mean_length": 435.0625,
      "completions/mean_terminated_length": 435.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3957009054720402,
      "epoch": 0.00051,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.588508665561676,
      "kl": 0.41865204460918903,
      "learning_rate": 4.898759122697604e-05,
      "loss": 0.0866,
      "num_tokens": 1133291.0,
      "reward": 0.4571281671524048,
      "reward_std": 0.5621116161346436,
      "rewards/rollout_reward_func/mean": 0.4571281671524048,
      "rewards/rollout_reward_func/std": 0.5608876347541809,
      "sampling/importance_sampling_ratio/max": 1.5039056539535522,
      "sampling/importance_sampling_ratio/mean": 1.0103577375411987,
      "sampling/importance_sampling_ratio/min": 0.7756198644638062,
      "sampling/sampling_logp_difference/max": 0.2501624822616577,
      "sampling/sampling_logp_difference/mean": 0.018241815268993378,
      "step": 51,
      "step_time": 8.385023154999999
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 888.0,
      "completions/max_terminated_length": 888.0,
      "completions/mean_length": 414.1875,
      "completions/mean_terminated_length": 414.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.33888778276741505,
      "epoch": 0.00052,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4953216314315796,
      "kl": 0.4071905091404915,
      "learning_rate": 4.8984985697806726e-05,
      "loss": 0.0468,
      "num_tokens": 1154257.0,
      "reward": 0.6723202466964722,
      "reward_std": 0.574486255645752,
      "rewards/rollout_reward_func/mean": 0.6723202466964722,
      "rewards/rollout_reward_func/std": 0.6185075044631958,
      "sampling/importance_sampling_ratio/max": 1.1309895515441895,
      "sampling/importance_sampling_ratio/mean": 0.978479266166687,
      "sampling/importance_sampling_ratio/min": 0.5926486253738403,
      "sampling/sampling_logp_difference/max": 0.4208402633666992,
      "sampling/sampling_logp_difference/mean": 0.016002990305423737,
      "step": 52,
      "step_time": 7.4931090730001415
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 967.0,
      "completions/max_terminated_length": 914.0,
      "completions/mean_length": 349.1875,
      "completions/mean_terminated_length": 308.0000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.569396274164319,
      "epoch": 0.00053,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3563882112503052,
      "kl": 0.35854968428611755,
      "learning_rate": 4.8982132147719125e-05,
      "loss": 0.1588,
      "num_tokens": 1173448.0,
      "reward": 0.6568812131881714,
      "reward_std": 0.48520809412002563,
      "rewards/rollout_reward_func/mean": 0.6568812131881714,
      "rewards/rollout_reward_func/std": 0.5257081389427185,
      "sampling/importance_sampling_ratio/max": 1.196919322013855,
      "sampling/importance_sampling_ratio/mean": 0.9578896760940552,
      "sampling/importance_sampling_ratio/min": 3.540314569283609e-11,
      "sampling/sampling_logp_difference/max": 4.7215166091918945,
      "sampling/sampling_logp_difference/mean": 0.16964340209960938,
      "step": 53,
      "step_time": 8.206854371999725
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 835.0,
      "completions/max_terminated_length": 835.0,
      "completions/mean_length": 414.875,
      "completions/mean_terminated_length": 414.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3172105494886637,
      "epoch": 0.00054,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.31334999203681946,
      "kl": 0.4554547965526581,
      "learning_rate": 4.8979030610722866e-05,
      "loss": 0.0138,
      "num_tokens": 1194374.0,
      "reward": 0.7956867814064026,
      "reward_std": 0.4465956687927246,
      "rewards/rollout_reward_func/mean": 0.7956867814064026,
      "rewards/rollout_reward_func/std": 0.4393559396266937,
      "sampling/importance_sampling_ratio/max": 1.058880090713501,
      "sampling/importance_sampling_ratio/mean": 0.9801302552223206,
      "sampling/importance_sampling_ratio/min": 0.8037052154541016,
      "sampling/sampling_logp_difference/max": 0.11076879501342773,
      "sampling/sampling_logp_difference/mean": 0.011640859767794609,
      "step": 54,
      "step_time": 7.100645255000245
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 960.0,
      "completions/max_terminated_length": 960.0,
      "completions/mean_length": 640.0625,
      "completions/mean_terminated_length": 640.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4563128836452961,
      "epoch": 0.00055,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2884346544742584,
      "kl": 0.8046899549663067,
      "learning_rate": 4.897568112378319e-05,
      "loss": -0.0079,
      "num_tokens": 1219927.0,
      "reward": 0.45939934253692627,
      "reward_std": 0.38820499181747437,
      "rewards/rollout_reward_func/mean": 0.45939934253692627,
      "rewards/rollout_reward_func/std": 0.5584731698036194,
      "sampling/importance_sampling_ratio/max": 1.0916351079940796,
      "sampling/importance_sampling_ratio/mean": 0.9879904985427856,
      "sampling/importance_sampling_ratio/min": 0.8365259766578674,
      "sampling/sampling_logp_difference/max": 0.1770343780517578,
      "sampling/sampling_logp_difference/mean": 0.01600312627851963,
      "step": 55,
      "step_time": 7.906471364999561
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 933.0,
      "completions/max_terminated_length": 933.0,
      "completions/mean_length": 568.8125,
      "completions/mean_terminated_length": 604.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4983320403844118,
      "epoch": 0.00056,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.2193748503923416,
      "kl": 0.9177752295508981,
      "learning_rate": 4.8972083726820485e-05,
      "loss": 0.0109,
      "num_tokens": 1243865.0,
      "reward": 0.6804192066192627,
      "reward_std": 0.368791401386261,
      "rewards/rollout_reward_func/mean": 0.6804192066192627,
      "rewards/rollout_reward_func/std": 0.6023459434509277,
      "sampling/importance_sampling_ratio/max": 1.1070553064346313,
      "sampling/importance_sampling_ratio/mean": 0.9835435152053833,
      "sampling/importance_sampling_ratio/min": 0.814237117767334,
      "sampling/sampling_logp_difference/max": 0.2054814100265503,
      "sampling/sampling_logp_difference/mean": 0.020568570122122765,
      "step": 56,
      "step_time": 7.910705786000108
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1776.0,
      "completions/max_terminated_length": 1776.0,
      "completions/mean_length": 716.0625,
      "completions/mean_terminated_length": 716.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.520364124327898,
      "epoch": 0.00057,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9891803860664368,
      "kl": 0.9971761293709278,
      "learning_rate": 4.896823846270984e-05,
      "loss": 0.0136,
      "num_tokens": 1270571.0,
      "reward": 0.30243828892707825,
      "reward_std": 0.7455288171768188,
      "rewards/rollout_reward_func/mean": 0.30243828892707825,
      "rewards/rollout_reward_func/std": 0.7901991009712219,
      "sampling/importance_sampling_ratio/max": 1.2357555627822876,
      "sampling/importance_sampling_ratio/mean": 1.0140416622161865,
      "sampling/importance_sampling_ratio/min": 0.8033018112182617,
      "sampling/sampling_logp_difference/max": 0.22592806816101074,
      "sampling/sampling_logp_difference/mean": 0.02692149020731449,
      "step": 57,
      "step_time": 10.254281136000372
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 906.0,
      "completions/max_terminated_length": 906.0,
      "completions/mean_length": 623.5,
      "completions/mean_terminated_length": 623.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.35761274583637714,
      "epoch": 0.00058,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5162756443023682,
      "kl": 0.5397632606327534,
      "learning_rate": 4.8964145377280486e-05,
      "loss": 0.0299,
      "num_tokens": 1295642.0,
      "reward": 0.7454532980918884,
      "reward_std": 0.553511381149292,
      "rewards/rollout_reward_func/mean": 0.7454532980918884,
      "rewards/rollout_reward_func/std": 0.579538106918335,
      "sampling/importance_sampling_ratio/max": 1.1008728742599487,
      "sampling/importance_sampling_ratio/mean": 0.9978313446044922,
      "sampling/importance_sampling_ratio/min": 0.8696094751358032,
      "sampling/sampling_logp_difference/max": 0.1079874038696289,
      "sampling/sampling_logp_difference/mean": 0.01360712107270956,
      "step": 58,
      "step_time": 8.2727058520004
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1725.0,
      "completions/max_terminated_length": 1725.0,
      "completions/mean_length": 730.75,
      "completions/mean_terminated_length": 730.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3370282053947449,
      "epoch": 0.00059,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3925839960575104,
      "kl": 0.7174907010048628,
      "learning_rate": 4.895980451931529e-05,
      "loss": 0.0026,
      "num_tokens": 1322862.0,
      "reward": 0.8048437833786011,
      "reward_std": 0.42270803451538086,
      "rewards/rollout_reward_func/mean": 0.8048437833786011,
      "rewards/rollout_reward_func/std": 0.4198833405971527,
      "sampling/importance_sampling_ratio/max": 1.2124475240707397,
      "sampling/importance_sampling_ratio/mean": 0.9828190207481384,
      "sampling/importance_sampling_ratio/min": 0.7277776598930359,
      "sampling/sampling_logp_difference/max": 0.31936049461364746,
      "sampling/sampling_logp_difference/mean": 0.02720661647617817,
      "step": 59,
      "step_time": 9.982905115000221
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 951.0,
      "completions/max_terminated_length": 951.0,
      "completions/mean_length": 411.375,
      "completions/mean_terminated_length": 411.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.30831781029701233,
      "epoch": 0.0006,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2151029258966446,
      "kl": 0.9290981516242027,
      "learning_rate": 4.895521594055018e-05,
      "loss": 0.012,
      "num_tokens": 1343408.0,
      "reward": 0.4744306802749634,
      "reward_std": 0.6574569940567017,
      "rewards/rollout_reward_func/mean": 0.4744306802749634,
      "rewards/rollout_reward_func/std": 0.6545820832252502,
      "sampling/importance_sampling_ratio/max": 1.2075121402740479,
      "sampling/importance_sampling_ratio/mean": 1.00216805934906,
      "sampling/importance_sampling_ratio/min": 0.6956810355186462,
      "sampling/sampling_logp_difference/max": 0.2833765745162964,
      "sampling/sampling_logp_difference/mean": 0.03828718513250351,
      "step": 60,
      "step_time": 7.4483262790001845
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1780.0,
      "completions/max_terminated_length": 1780.0,
      "completions/mean_length": 740.25,
      "completions/mean_terminated_length": 740.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.25919711869210005,
      "epoch": 0.00061,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.48723870515823364,
      "kl": 1.6172149330377579,
      "learning_rate": 4.895037969567348e-05,
      "loss": 0.0142,
      "num_tokens": 1370555.0,
      "reward": 0.21409361064434052,
      "reward_std": 0.7155660390853882,
      "rewards/rollout_reward_func/mean": 0.21409361064434052,
      "rewards/rollout_reward_func/std": 0.6913310289382935,
      "sampling/importance_sampling_ratio/max": 1.2322887182235718,
      "sampling/importance_sampling_ratio/mean": 0.9904075860977173,
      "sampling/importance_sampling_ratio/min": 0.7125310301780701,
      "sampling/sampling_logp_difference/max": 0.33855485916137695,
      "sampling/sampling_logp_difference/mean": 0.022703073918819427,
      "step": 61,
      "step_time": 10.0587223360003
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 933.0,
      "completions/max_terminated_length": 933.0,
      "completions/mean_length": 484.9375,
      "completions/mean_terminated_length": 484.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.19125733710825443,
      "epoch": 0.00062,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2451031506061554,
      "kl": 1.227780293673277,
      "learning_rate": 4.89452958423253e-05,
      "loss": 0.0022,
      "num_tokens": 1392219.0,
      "reward": 0.6772332191467285,
      "reward_std": 0.5682100057601929,
      "rewards/rollout_reward_func/mean": 0.6772332191467285,
      "rewards/rollout_reward_func/std": 0.614905595779419,
      "sampling/importance_sampling_ratio/max": 1.21300208568573,
      "sampling/importance_sampling_ratio/mean": 0.9946990609169006,
      "sampling/importance_sampling_ratio/min": 0.6662668585777283,
      "sampling/sampling_logp_difference/max": 0.40597641468048096,
      "sampling/sampling_logp_difference/mean": 0.020531460642814636,
      "step": 62,
      "step_time": 7.424367028000233
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 923.0,
      "completions/max_terminated_length": 923.0,
      "completions/mean_length": 480.5625,
      "completions/mean_terminated_length": 480.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.17029015254229307,
      "epoch": 0.00063,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21118757128715515,
      "kl": 1.990612629801035,
      "learning_rate": 4.8939964441096836e-05,
      "loss": -0.0033,
      "num_tokens": 1414525.0,
      "reward": 0.4881027638912201,
      "reward_std": 0.7444739937782288,
      "rewards/rollout_reward_func/mean": 0.4881027638912201,
      "rewards/rollout_reward_func/std": 0.7328922748565674,
      "sampling/importance_sampling_ratio/max": 1.1125811338424683,
      "sampling/importance_sampling_ratio/mean": 0.984230637550354,
      "sampling/importance_sampling_ratio/min": 0.7811985611915588,
      "sampling/sampling_logp_difference/max": 0.2460927963256836,
      "sampling/sampling_logp_difference/mean": 0.017771562561392784,
      "step": 63,
      "step_time": 7.538278758000388
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1808.0,
      "completions/max_terminated_length": 1808.0,
      "completions/mean_length": 672.1875,
      "completions/mean_terminated_length": 672.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.19442320242524147,
      "epoch": 0.00064,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5721246600151062,
      "kl": 1.0051763765513897,
      "learning_rate": 4.893438555552967e-05,
      "loss": -0.005,
      "num_tokens": 1440041.0,
      "reward": 0.18524138629436493,
      "reward_std": 0.8226193189620972,
      "rewards/rollout_reward_func/mean": 0.18524138629436493,
      "rewards/rollout_reward_func/std": 0.8185408711433411,
      "sampling/importance_sampling_ratio/max": 1.1936362981796265,
      "sampling/importance_sampling_ratio/mean": 0.9861190915107727,
      "sampling/importance_sampling_ratio/min": 0.7553765177726746,
      "sampling/sampling_logp_difference/max": 0.295917272567749,
      "sampling/sampling_logp_difference/mean": 0.021850578486919403,
      "step": 64,
      "step_time": 10.102949832999911
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1585.0,
      "completions/max_terminated_length": 1585.0,
      "completions/mean_length": 723.0625,
      "completions/mean_terminated_length": 723.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.1570829013362527,
      "epoch": 0.00065,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.20437276363372803,
      "kl": 0.8459445647895336,
      "learning_rate": 4.892855925211496e-05,
      "loss": 0.0071,
      "num_tokens": 1466850.0,
      "reward": 0.7331885099411011,
      "reward_std": 0.46535658836364746,
      "rewards/rollout_reward_func/mean": 0.7331885099411011,
      "rewards/rollout_reward_func/std": 0.47736868262290955,
      "sampling/importance_sampling_ratio/max": 1.2122944593429565,
      "sampling/importance_sampling_ratio/mean": 0.9961503744125366,
      "sampling/importance_sampling_ratio/min": 0.7855719327926636,
      "sampling/sampling_logp_difference/max": 0.2377941608428955,
      "sampling/sampling_logp_difference/mean": 0.013996700756251812,
      "step": 65,
      "step_time": 10.157848146999413
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1718.0,
      "completions/max_terminated_length": 1718.0,
      "completions/mean_length": 657.875,
      "completions/mean_terminated_length": 657.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.17827294394373894,
      "epoch": 0.00066,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6156491041183472,
      "kl": 3.059620290994644,
      "learning_rate": 4.892248560029269e-05,
      "loss": 0.0042,
      "num_tokens": 1491998.0,
      "reward": 0.4184495806694031,
      "reward_std": 0.7619459629058838,
      "rewards/rollout_reward_func/mean": 0.4184495806694031,
      "rewards/rollout_reward_func/std": 0.7396833896636963,
      "sampling/importance_sampling_ratio/max": 1.085866093635559,
      "sampling/importance_sampling_ratio/mean": 0.9641371369361877,
      "sampling/importance_sampling_ratio/min": 0.6417354345321655,
      "sampling/sampling_logp_difference/max": 0.4434812068939209,
      "sampling/sampling_logp_difference/mean": 0.028886843472719193,
      "step": 66,
      "step_time": 9.829651840999759
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1689.0,
      "completions/max_terminated_length": 1689.0,
      "completions/mean_length": 559.625,
      "completions/mean_terminated_length": 559.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2952111102640629,
      "epoch": 0.00067,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3560001850128174,
      "kl": 0.7502390965819359,
      "learning_rate": 4.891616467245085e-05,
      "loss": 0.0045,
      "num_tokens": 1515502.0,
      "reward": 0.3644614517688751,
      "reward_std": 0.7647616863250732,
      "rewards/rollout_reward_func/mean": 0.3644614517688751,
      "rewards/rollout_reward_func/std": 0.8034436702728271,
      "sampling/importance_sampling_ratio/max": 1.1505366563796997,
      "sampling/importance_sampling_ratio/mean": 0.9907373189926147,
      "sampling/importance_sampling_ratio/min": 0.6978281736373901,
      "sampling/sampling_logp_difference/max": 0.3597443103790283,
      "sampling/sampling_logp_difference/mean": 0.022191070020198822,
      "step": 67,
      "step_time": 9.546588644999929
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2440.0,
      "completions/max_terminated_length": 2440.0,
      "completions/mean_length": 641.0625,
      "completions/mean_terminated_length": 641.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3412857875227928,
      "epoch": 0.00068,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4198976457118988,
      "kl": 0.8183774054050446,
      "learning_rate": 4.890959654392451e-05,
      "loss": -0.0159,
      "num_tokens": 1539966.0,
      "reward": 0.3715824484825134,
      "reward_std": 0.8930363655090332,
      "rewards/rollout_reward_func/mean": 0.3715824484825134,
      "rewards/rollout_reward_func/std": 0.8793548345565796,
      "sampling/importance_sampling_ratio/max": 1.2876261472702026,
      "sampling/importance_sampling_ratio/mean": 0.9591962099075317,
      "sampling/importance_sampling_ratio/min": 0.47812482714653015,
      "sampling/sampling_logp_difference/max": 0.44975757598876953,
      "sampling/sampling_logp_difference/mean": 0.03494518622756004,
      "step": 68,
      "step_time": 11.654221397000128
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1862.0,
      "completions/max_terminated_length": 1862.0,
      "completions/mean_length": 568.25,
      "completions/mean_terminated_length": 568.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5051987059414387,
      "epoch": 0.00069,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6733705401420593,
      "kl": 0.7016708664596081,
      "learning_rate": 4.890278129299502e-05,
      "loss": 0.0093,
      "num_tokens": 1563400.0,
      "reward": 0.6826345920562744,
      "reward_std": 0.6071673035621643,
      "rewards/rollout_reward_func/mean": 0.6826345920562744,
      "rewards/rollout_reward_func/std": 0.6006214618682861,
      "sampling/importance_sampling_ratio/max": 1.0955854654312134,
      "sampling/importance_sampling_ratio/mean": 0.9324354529380798,
      "sampling/importance_sampling_ratio/min": 0.6467282176017761,
      "sampling/sampling_logp_difference/max": 0.242919921875,
      "sampling/sampling_logp_difference/mean": 0.03436674922704697,
      "step": 69,
      "step_time": 10.02910807000012
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1728.0,
      "completions/max_terminated_length": 1728.0,
      "completions/mean_length": 525.8125,
      "completions/mean_terminated_length": 525.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5868957936763763,
      "epoch": 0.0007,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4819300174713135,
      "kl": 0.8681499697268009,
      "learning_rate": 4.8895719000889e-05,
      "loss": 0.0155,
      "num_tokens": 1585945.0,
      "reward": 0.2719266414642334,
      "reward_std": 0.8359231948852539,
      "rewards/rollout_reward_func/mean": 0.2719266414642334,
      "rewards/rollout_reward_func/std": 0.971158504486084,
      "sampling/importance_sampling_ratio/max": 1.3254462480545044,
      "sampling/importance_sampling_ratio/mean": 1.0544968843460083,
      "sampling/importance_sampling_ratio/min": 0.7963511943817139,
      "sampling/sampling_logp_difference/max": 0.2653179168701172,
      "sampling/sampling_logp_difference/mean": 0.042104896157979965,
      "step": 70,
      "step_time": 9.562392960999887
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1585.0,
      "completions/max_terminated_length": 1585.0,
      "completions/mean_length": 557.75,
      "completions/mean_terminated_length": 557.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5970004573464394,
      "epoch": 0.00071,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0883138179779053,
      "kl": 2.6468969769775867,
      "learning_rate": 4.888840975177738e-05,
      "loss": 0.0258,
      "num_tokens": 1609311.0,
      "reward": 0.49205559492111206,
      "reward_std": 0.7647160291671753,
      "rewards/rollout_reward_func/mean": 0.49205559492111206,
      "rewards/rollout_reward_func/std": 0.8195031881332397,
      "sampling/importance_sampling_ratio/max": 1.3790841102600098,
      "sampling/importance_sampling_ratio/mean": 1.0071711540222168,
      "sampling/importance_sampling_ratio/min": 0.42650559544563293,
      "sampling/sampling_logp_difference/max": 0.8578081130981445,
      "sampling/sampling_logp_difference/mean": 0.037521205842494965,
      "step": 71,
      "step_time": 10.038415755999722
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1701.0,
      "completions/max_terminated_length": 1701.0,
      "completions/mean_length": 505.375,
      "completions/mean_terminated_length": 505.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5414082705974579,
      "epoch": 0.00072,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5125952363014221,
      "kl": 0.695716549642384,
      "learning_rate": 4.8880853632774445e-05,
      "loss": 0.0051,
      "num_tokens": 1631658.0,
      "reward": 0.14395099878311157,
      "reward_std": 1.0298528671264648,
      "rewards/rollout_reward_func/mean": 0.14395099878311157,
      "rewards/rollout_reward_func/std": 1.0027309656143188,
      "sampling/importance_sampling_ratio/max": 1.329268217086792,
      "sampling/importance_sampling_ratio/mean": 1.0028799772262573,
      "sampling/importance_sampling_ratio/min": 0.8056287169456482,
      "sampling/sampling_logp_difference/max": 0.2691664695739746,
      "sampling/sampling_logp_difference/mean": 0.028703732416033745,
      "step": 72,
      "step_time": 9.488091585000348
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2456.0,
      "completions/max_terminated_length": 2456.0,
      "completions/mean_length": 937.5,
      "completions/mean_terminated_length": 937.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6696975007653236,
      "epoch": 0.00073,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.44106316566467285,
      "kl": 0.5452143084257841,
      "learning_rate": 4.887305073393672e-05,
      "loss": 0.0059,
      "num_tokens": 1661807.0,
      "reward": -0.03709357976913452,
      "reward_std": 0.9848629832267761,
      "rewards/rollout_reward_func/mean": -0.03709357976913452,
      "rewards/rollout_reward_func/std": 0.969811201095581,
      "sampling/importance_sampling_ratio/max": 1.1651084423065186,
      "sampling/importance_sampling_ratio/mean": 0.9971095323562622,
      "sampling/importance_sampling_ratio/min": 0.658902645111084,
      "sampling/sampling_logp_difference/max": 0.41978931427001953,
      "sampling/sampling_logp_difference/mean": 0.0338299423456192,
      "step": 73,
      "step_time": 12.1631746999999
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2550.0,
      "completions/max_terminated_length": 2550.0,
      "completions/mean_length": 884.25,
      "completions/mean_terminated_length": 884.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9279335886240005,
      "epoch": 0.00074,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.63059401512146,
      "kl": 0.4721297826617956,
      "learning_rate": 4.8865001148262e-05,
      "loss": -0.0554,
      "num_tokens": 1691406.0,
      "reward": 0.26807546615600586,
      "reward_std": 1.010190725326538,
      "rewards/rollout_reward_func/mean": 0.26807546615600586,
      "rewards/rollout_reward_func/std": 0.97596275806427,
      "sampling/importance_sampling_ratio/max": 1.5100480318069458,
      "sampling/importance_sampling_ratio/mean": 1.076258659362793,
      "sampling/importance_sampling_ratio/min": 0.887296736240387,
      "sampling/sampling_logp_difference/max": 0.40724611282348633,
      "sampling/sampling_logp_difference/mean": 0.03572637587785721,
      "step": 74,
      "step_time": 12.777590664000172
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1808.0,
      "completions/max_terminated_length": 1808.0,
      "completions/mean_length": 424.375,
      "completions/mean_terminated_length": 424.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6856178939342499,
      "epoch": 0.00075,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.45729050040245056,
      "kl": 4.011728692799807,
      "learning_rate": 4.8856704971688144e-05,
      "loss": -0.0049,
      "num_tokens": 1712620.0,
      "reward": 0.13459989428520203,
      "reward_std": 1.0404752492904663,
      "rewards/rollout_reward_func/mean": 0.13459989428520203,
      "rewards/rollout_reward_func/std": 1.0135445594787598,
      "sampling/importance_sampling_ratio/max": 1.310341715812683,
      "sampling/importance_sampling_ratio/mean": 1.0461218357086182,
      "sampling/importance_sampling_ratio/min": 0.8002749681472778,
      "sampling/sampling_logp_difference/max": 0.21268367767333984,
      "sampling/sampling_logp_difference/mean": 0.032558832317590714,
      "step": 75,
      "step_time": 9.692363602000114
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1573.0,
      "completions/max_terminated_length": 1573.0,
      "completions/mean_length": 605.4375,
      "completions/mean_terminated_length": 605.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7337356954813004,
      "epoch": 0.00076,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3206181228160858,
      "kl": 0.6717826426029205,
      "learning_rate": 4.8848162303091964e-05,
      "loss": -0.0075,
      "num_tokens": 1737190.0,
      "reward": 0.7556591629981995,
      "reward_std": 0.6911003589630127,
      "rewards/rollout_reward_func/mean": 0.7556591629981995,
      "rewards/rollout_reward_func/std": 0.6676918268203735,
      "sampling/importance_sampling_ratio/max": 1.2337191104888916,
      "sampling/importance_sampling_ratio/mean": 1.032653570175171,
      "sampling/importance_sampling_ratio/min": 0.8844587206840515,
      "sampling/sampling_logp_difference/max": 0.21206188201904297,
      "sampling/sampling_logp_difference/mean": 0.02903384529054165,
      "step": 76,
      "step_time": 9.538897219999626
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1674.0,
      "completions/max_terminated_length": 1674.0,
      "completions/mean_length": 710.75,
      "completions/mean_terminated_length": 710.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8344346210360527,
      "epoch": 0.00077,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.37589070200920105,
      "kl": 0.5597859025001526,
      "learning_rate": 4.883937324428809e-05,
      "loss": -0.0071,
      "num_tokens": 1763085.0,
      "reward": 0.2681085467338562,
      "reward_std": 1.0103193521499634,
      "rewards/rollout_reward_func/mean": 0.2681085467338562,
      "rewards/rollout_reward_func/std": 0.9760621190071106,
      "sampling/importance_sampling_ratio/max": 1.159041166305542,
      "sampling/importance_sampling_ratio/mean": 1.013990044593811,
      "sampling/importance_sampling_ratio/min": 0.8382033109664917,
      "sampling/sampling_logp_difference/max": 0.13455283641815186,
      "sampling/sampling_logp_difference/mean": 0.027372974902391434,
      "step": 77,
      "step_time": 9.820363725999641
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2612.0,
      "completions/max_terminated_length": 2612.0,
      "completions/mean_length": 750.1875,
      "completions/mean_terminated_length": 750.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8561322838068008,
      "epoch": 0.00078,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7622040510177612,
      "kl": 0.7444981001317501,
      "learning_rate": 4.8830337900027665e-05,
      "loss": -0.0231,
      "num_tokens": 1790215.0,
      "reward": -0.1066448986530304,
      "reward_std": 1.0347999334335327,
      "rewards/rollout_reward_func/mean": -0.1066448986530304,
      "rewards/rollout_reward_func/std": 1.0082353353500366,
      "sampling/importance_sampling_ratio/max": 1.1878044605255127,
      "sampling/importance_sampling_ratio/mean": 1.017825722694397,
      "sampling/importance_sampling_ratio/min": 0.8638620972633362,
      "sampling/sampling_logp_difference/max": 0.12212133407592773,
      "sampling/sampling_logp_difference/mean": 0.019169704988598824,
      "step": 78,
      "step_time": 12.300321268000062
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2450.0,
      "completions/max_terminated_length": 2450.0,
      "completions/mean_length": 847.5625,
      "completions/mean_terminated_length": 847.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8678246513009071,
      "epoch": 0.00079,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.39388349652290344,
      "kl": 0.8048326596617699,
      "learning_rate": 4.8821056377997206e-05,
      "loss": 0.0027,
      "num_tokens": 1818728.0,
      "reward": 0.38694435358047485,
      "reward_std": 0.9612793326377869,
      "rewards/rollout_reward_func/mean": 0.38694435358047485,
      "rewards/rollout_reward_func/std": 0.9392731785774231,
      "sampling/importance_sampling_ratio/max": 1.208518385887146,
      "sampling/importance_sampling_ratio/mean": 0.9855331778526306,
      "sampling/importance_sampling_ratio/min": 0.7240620255470276,
      "sampling/sampling_logp_difference/max": 0.32259416580200195,
      "sampling/sampling_logp_difference/mean": 0.028120391070842743,
      "step": 79,
      "step_time": 12.079894115000343
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1625.0,
      "completions/max_terminated_length": 1625.0,
      "completions/mean_length": 570.125,
      "completions/mean_terminated_length": 570.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0609731152653694,
      "epoch": 0.0008,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7451897263526917,
      "kl": 0.7536248341202736,
      "learning_rate": 4.881152878881723e-05,
      "loss": -0.0607,
      "num_tokens": 1843180.0,
      "reward": 0.01627398654818535,
      "reward_std": 1.0519883632659912,
      "rewards/rollout_reward_func/mean": 0.01627398654818535,
      "rewards/rollout_reward_func/std": 1.016318678855896,
      "sampling/importance_sampling_ratio/max": 1.1141277551651,
      "sampling/importance_sampling_ratio/mean": 0.9944953918457031,
      "sampling/importance_sampling_ratio/min": 0.7617278695106506,
      "sampling/sampling_logp_difference/max": 0.2707061767578125,
      "sampling/sampling_logp_difference/mean": 0.03098263405263424,
      "step": 80,
      "step_time": 9.599547194999786
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2439.0,
      "completions/max_terminated_length": 2439.0,
      "completions/mean_length": 903.0625,
      "completions/mean_terminated_length": 903.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9262722060084343,
      "epoch": 0.00081,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6183764338493347,
      "kl": 0.7306044884026051,
      "learning_rate": 4.8801755246040995e-05,
      "loss": 0.0013,
      "num_tokens": 1871949.0,
      "reward": 0.16598086059093475,
      "reward_std": 0.9397992491722107,
      "rewards/rollout_reward_func/mean": 0.16598086059093475,
      "rewards/rollout_reward_func/std": 0.9776033163070679,
      "sampling/importance_sampling_ratio/max": 1.2227150201797485,
      "sampling/importance_sampling_ratio/mean": 1.0114750862121582,
      "sampling/importance_sampling_ratio/min": 0.8833626508712769,
      "sampling/sampling_logp_difference/max": 0.12622690200805664,
      "sampling/sampling_logp_difference/mean": 0.025883793830871582,
      "step": 81,
      "step_time": 12.367132293000168
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1798.0,
      "completions/max_terminated_length": 1798.0,
      "completions/mean_length": 592.125,
      "completions/mean_terminated_length": 592.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9533793926239014,
      "epoch": 0.00082,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6649211645126343,
      "kl": 0.5920517183840275,
      "learning_rate": 4.87917358661531e-05,
      "loss": 0.0437,
      "num_tokens": 1895627.0,
      "reward": 0.2106788456439972,
      "reward_std": 0.9240612387657166,
      "rewards/rollout_reward_func/mean": 0.2106788456439972,
      "rewards/rollout_reward_func/std": 0.9447680711746216,
      "sampling/importance_sampling_ratio/max": 1.3248405456542969,
      "sampling/importance_sampling_ratio/mean": 1.033621072769165,
      "sampling/importance_sampling_ratio/min": 0.6627046465873718,
      "sampling/sampling_logp_difference/max": 0.19269180297851562,
      "sampling/sampling_logp_difference/mean": 0.0319562628865242,
      "step": 82,
      "step_time": 10.231523553999978
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1600.0,
      "completions/max_terminated_length": 1600.0,
      "completions/mean_length": 548.3125,
      "completions/mean_terminated_length": 525.7333374023438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9453123509883881,
      "epoch": 0.00083,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7771857976913452,
      "kl": 0.737138332799077,
      "learning_rate": 4.878147076856813e-05,
      "loss": -0.0807,
      "num_tokens": 1919134.0,
      "reward": 0.13246560096740723,
      "reward_std": 0.9555523991584778,
      "rewards/rollout_reward_func/mean": 0.13246560096740723,
      "rewards/rollout_reward_func/std": 0.9356718063354492,
      "sampling/importance_sampling_ratio/max": 1.2050647735595703,
      "sampling/importance_sampling_ratio/mean": 1.0169217586517334,
      "sampling/importance_sampling_ratio/min": 0.8763161897659302,
      "sampling/sampling_logp_difference/max": 0.19452190399169922,
      "sampling/sampling_logp_difference/mean": 0.034966859966516495,
      "step": 83,
      "step_time": 9.952704616999654
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3438.0,
      "completions/max_terminated_length": 3438.0,
      "completions/mean_length": 937.25,
      "completions/mean_terminated_length": 937.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0913000330328941,
      "epoch": 0.00084,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9031299948692322,
      "kl": 1.0373839884996414,
      "learning_rate": 4.877096007562923e-05,
      "loss": 0.0592,
      "num_tokens": 1948910.0,
      "reward": 0.1662713587284088,
      "reward_std": 1.004563808441162,
      "rewards/rollout_reward_func/mean": 0.1662713587284088,
      "rewards/rollout_reward_func/std": 0.9774200320243835,
      "sampling/importance_sampling_ratio/max": 1.2400377988815308,
      "sampling/importance_sampling_ratio/mean": 1.0008478164672852,
      "sampling/importance_sampling_ratio/min": 0.764829158782959,
      "sampling/sampling_logp_difference/max": 0.263547420501709,
      "sampling/sampling_logp_difference/mean": 0.03778738155961037,
      "step": 84,
      "step_time": 15.913168080999867
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1860.0,
      "completions/max_terminated_length": 1613.0,
      "completions/mean_length": 682.75,
      "completions/mean_terminated_length": 604.2667236328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2304420247673988,
      "epoch": 0.00085,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0858598947525024,
      "kl": 1.1724870838224888,
      "learning_rate": 4.876020391260662e-05,
      "loss": 0.0318,
      "num_tokens": 1974942.0,
      "reward": 0.21416312456130981,
      "reward_std": 0.9206823110580444,
      "rewards/rollout_reward_func/mean": 0.21416312456130981,
      "rewards/rollout_reward_func/std": 0.9451598525047302,
      "sampling/importance_sampling_ratio/max": 1.7841544151306152,
      "sampling/importance_sampling_ratio/mean": 1.0966160297393799,
      "sampling/importance_sampling_ratio/min": 0.797315239906311,
      "sampling/sampling_logp_difference/max": 0.35060548782348633,
      "sampling/sampling_logp_difference/mean": 0.04261929169297218,
      "step": 85,
      "step_time": 11.190730362000068
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2570.0,
      "completions/max_terminated_length": 2570.0,
      "completions/mean_length": 593.1875,
      "completions/mean_terminated_length": 404.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5899821668863297,
      "epoch": 0.00086,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3809760808944702,
      "kl": 1.261812973767519,
      "learning_rate": 4.874920240769614e-05,
      "loss": 0.1907,
      "num_tokens": 1998460.0,
      "reward": 0.5607922673225403,
      "reward_std": 0.8308257460594177,
      "rewards/rollout_reward_func/mean": 0.5607922673225403,
      "rewards/rollout_reward_func/std": 0.8084061145782471,
      "sampling/importance_sampling_ratio/max": 1.4531539678573608,
      "sampling/importance_sampling_ratio/mean": 0.9302196502685547,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.2891359329223633,
      "sampling/sampling_logp_difference/mean": 0.046063344925642014,
      "step": 86,
      "step_time": 13.297108432000186
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2459.0,
      "completions/max_terminated_length": 1626.0,
      "completions/mean_length": 503.875,
      "completions/mean_terminated_length": 357.69232177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2043307945132256,
      "epoch": 0.00087,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.114431381225586,
      "kl": 0.92607182264328,
      "learning_rate": 4.873795569201768e-05,
      "loss": 0.2947,
      "num_tokens": 2020797.0,
      "reward": 0.13884396851062775,
      "reward_std": 0.9634337425231934,
      "rewards/rollout_reward_func/mean": 0.13884396851062775,
      "rewards/rollout_reward_func/std": 0.9308338761329651,
      "sampling/importance_sampling_ratio/max": 1.9242708683013916,
      "sampling/importance_sampling_ratio/mean": 0.9603827595710754,
      "sampling/importance_sampling_ratio/min": 6.37789344182238e-05,
      "sampling/sampling_logp_difference/max": 5.706535339355469,
      "sampling/sampling_logp_difference/mean": 0.06178221479058266,
      "step": 87,
      "step_time": 13.28351962399961
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2660.0,
      "completions/max_terminated_length": 2660.0,
      "completions/mean_length": 615.625,
      "completions/mean_terminated_length": 670.4166870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1803444474935532,
      "epoch": 0.00088,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7353789806365967,
      "kl": 0.7888411469757557,
      "learning_rate": 4.872646389961369e-05,
      "loss": -0.068,
      "num_tokens": 2044890.0,
      "reward": 0.49333155155181885,
      "reward_std": 0.8267082571983337,
      "rewards/rollout_reward_func/mean": 0.49333155155181885,
      "rewards/rollout_reward_func/std": 0.81538325548172,
      "sampling/importance_sampling_ratio/max": 1.3024191856384277,
      "sampling/importance_sampling_ratio/mean": 0.8707255721092224,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.015855073928833,
      "sampling/sampling_logp_difference/mean": 0.06657309830188751,
      "step": 88,
      "step_time": 14.49677113300072
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 3286.0,
      "completions/max_terminated_length": 3286.0,
      "completions/mean_length": 828.1875,
      "completions/mean_terminated_length": 753.3846435546875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3675153851509094,
      "epoch": 0.00089,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1010125875473022,
      "kl": 1.141457837074995,
      "learning_rate": 4.871472716744746e-05,
      "loss": -0.0961,
      "num_tokens": 2073019.0,
      "reward": 0.0690470039844513,
      "reward_std": 0.822632908821106,
      "rewards/rollout_reward_func/mean": 0.0690470039844513,
      "rewards/rollout_reward_func/std": 0.9005694389343262,
      "sampling/importance_sampling_ratio/max": 1.4703506231307983,
      "sampling/importance_sampling_ratio/mean": 0.9391757249832153,
      "sampling/importance_sampling_ratio/min": 0.008030284196138382,
      "sampling/sampling_logp_difference/max": 6.041694641113281,
      "sampling/sampling_logp_difference/mean": 0.05815901607275009,
      "step": 89,
      "step_time": 16.616365278000103
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1557.0,
      "completions/max_terminated_length": 1557.0,
      "completions/mean_length": 399.3125,
      "completions/mean_terminated_length": 368.0769348144531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.438521035015583,
      "epoch": 0.0009,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1577959060668945,
      "kl": 1.1810866631567478,
      "learning_rate": 4.8702745635401616e-05,
      "loss": -0.0842,
      "num_tokens": 2094625.0,
      "reward": 0.23940911889076233,
      "reward_std": 0.8344873189926147,
      "rewards/rollout_reward_func/mean": 0.23940911889076233,
      "rewards/rollout_reward_func/std": 0.8467217087745667,
      "sampling/importance_sampling_ratio/max": 1.745457410812378,
      "sampling/importance_sampling_ratio/mean": 0.7349727153778076,
      "sampling/importance_sampling_ratio/min": 3.4280525466889866e-12,
      "sampling/sampling_logp_difference/max": 15.251540184020996,
      "sampling/sampling_logp_difference/mean": 0.20628930628299713,
      "step": 90,
      "step_time": 11.612975703000302
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1807.0,
      "completions/max_terminated_length": 1807.0,
      "completions/mean_length": 565.3125,
      "completions/mean_terminated_length": 537.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4626080393791199,
      "epoch": 0.00091,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0542365312576294,
      "kl": 1.0648687742650509,
      "learning_rate": 4.869051944627639e-05,
      "loss": 0.0084,
      "num_tokens": 2117684.0,
      "reward": 0.37735873460769653,
      "reward_std": 0.8546274304389954,
      "rewards/rollout_reward_func/mean": 0.37735873460769653,
      "rewards/rollout_reward_func/std": 0.8675792813301086,
      "sampling/importance_sampling_ratio/max": 1.07419753074646,
      "sampling/importance_sampling_ratio/mean": 0.7771488428115845,
      "sampling/importance_sampling_ratio/min": 2.309196454821994e-21,
      "sampling/sampling_logp_difference/max": 20.636154174804688,
      "sampling/sampling_logp_difference/mean": 0.18763870000839233,
      "step": 91,
      "step_time": 12.743372936000242
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1713.0,
      "completions/max_terminated_length": 1713.0,
      "completions/mean_length": 861.5,
      "completions/mean_terminated_length": 725.0,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 1.3821348175406456,
      "epoch": 0.00092,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9367092847824097,
      "kl": 1.1472019739449024,
      "learning_rate": 4.8678048745787894e-05,
      "loss": -0.0844,
      "num_tokens": 2146772.0,
      "reward": 0.505336582660675,
      "reward_std": 0.7460877895355225,
      "rewards/rollout_reward_func/mean": 0.505336582660675,
      "rewards/rollout_reward_func/std": 0.7949479818344116,
      "sampling/importance_sampling_ratio/max": 1.919449806213379,
      "sampling/importance_sampling_ratio/mean": 0.8983476758003235,
      "sampling/importance_sampling_ratio/min": 6.702574151078167e-13,
      "sampling/sampling_logp_difference/max": 4.222768783569336,
      "sampling/sampling_logp_difference/mean": 0.1255618929862976,
      "step": 92,
      "step_time": 13.289456703000042
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 1810.0,
      "completions/max_terminated_length": 1668.0,
      "completions/mean_length": 634.9375,
      "completions/mean_terminated_length": 663.888916015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.623960629105568,
      "epoch": 0.00093,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.18399977684021,
      "kl": 1.1107087098062038,
      "learning_rate": 4.866533368256645e-05,
      "loss": -0.1136,
      "num_tokens": 2171491.0,
      "reward": 0.5612123608589172,
      "reward_std": 0.8054671287536621,
      "rewards/rollout_reward_func/mean": 0.5612123608589172,
      "rewards/rollout_reward_func/std": 0.8078193664550781,
      "sampling/importance_sampling_ratio/max": 1.7370327711105347,
      "sampling/importance_sampling_ratio/mean": 0.9771984815597534,
      "sampling/importance_sampling_ratio/min": 0.34272077679634094,
      "sampling/sampling_logp_difference/max": 0.3069038391113281,
      "sampling/sampling_logp_difference/mean": 0.0357479602098465,
      "step": 93,
      "step_time": 12.865162758999759
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1686.0,
      "completions/max_terminated_length": 1686.0,
      "completions/mean_length": 624.0,
      "completions/mean_terminated_length": 550.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5121415928006172,
      "epoch": 0.00094,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.5841081142425537,
      "kl": 0.6550641357898712,
      "learning_rate": 4.865237440815477e-05,
      "loss": -0.07,
      "num_tokens": 2196304.0,
      "reward": 0.13557033240795135,
      "reward_std": 0.9623743295669556,
      "rewards/rollout_reward_func/mean": 0.13557033240795135,
      "rewards/rollout_reward_func/std": 0.9299102425575256,
      "sampling/importance_sampling_ratio/max": 2.1122381687164307,
      "sampling/importance_sampling_ratio/mean": 1.1102687120437622,
      "sampling/importance_sampling_ratio/min": 0.5373618006706238,
      "sampling/sampling_logp_difference/max": 0.44901371002197266,
      "sampling/sampling_logp_difference/mean": 0.039520200341939926,
      "step": 94,
      "step_time": 13.124361281999882
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5625,
      "completions/max_length": 2467.0,
      "completions/max_terminated_length": 2467.0,
      "completions/mean_length": 1191.4375,
      "completions/mean_terminated_length": 1162.4285888671875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 1.4300144091248512,
      "epoch": 0.00095,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2484591007232666,
      "kl": 0.5447282008826733,
      "learning_rate": 4.863917107700616e-05,
      "loss": 0.0387,
      "num_tokens": 2230976.0,
      "reward": 0.4535864591598511,
      "reward_std": 0.785423755645752,
      "rewards/rollout_reward_func/mean": 0.4535864591598511,
      "rewards/rollout_reward_func/std": 0.7843860387802124,
      "sampling/importance_sampling_ratio/max": 2.077028274536133,
      "sampling/importance_sampling_ratio/mean": 0.8381145000457764,
      "sampling/importance_sampling_ratio/min": 0.2902454435825348,
      "sampling/sampling_logp_difference/max": 0.35004425048828125,
      "sampling/sampling_logp_difference/mean": 0.04090958833694458,
      "step": 95,
      "step_time": 16.42264915299984
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1943.0,
      "completions/max_terminated_length": 916.0,
      "completions/mean_length": 477.6875,
      "completions/mean_terminated_length": 431.0769348144531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3103893548250198,
      "epoch": 0.00096,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1996201276779175,
      "kl": 0.9936919957399368,
      "learning_rate": 4.8625723846482675e-05,
      "loss": -0.0754,
      "num_tokens": 2253569.0,
      "reward": 0.09999595582485199,
      "reward_std": 0.8111519813537598,
      "rewards/rollout_reward_func/mean": 0.09999595582485199,
      "rewards/rollout_reward_func/std": 0.7995628714561462,
      "sampling/importance_sampling_ratio/max": 1.735880732536316,
      "sampling/importance_sampling_ratio/mean": 0.881166398525238,
      "sampling/importance_sampling_ratio/min": 7.344280484331348e-14,
      "sampling/sampling_logp_difference/max": 5.567135810852051,
      "sampling/sampling_logp_difference/mean": 0.11029127985239029,
      "step": 96,
      "step_time": 12.269342957999697
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1568.0,
      "completions/max_terminated_length": 987.0,
      "completions/mean_length": 597.25,
      "completions/mean_terminated_length": 400.3333435058594,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 1.5377584844827652,
      "epoch": 0.00097,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 12.938536643981934,
      "kl": 3.172784324735403,
      "learning_rate": 4.861203287685328e-05,
      "loss": -0.2646,
      "num_tokens": 2277932.0,
      "reward": 0.25241851806640625,
      "reward_std": 0.9059150218963623,
      "rewards/rollout_reward_func/mean": 0.25241851806640625,
      "rewards/rollout_reward_func/std": 0.9172124266624451,
      "sampling/importance_sampling_ratio/max": 1.6617323160171509,
      "sampling/importance_sampling_ratio/mean": 0.9768710136413574,
      "sampling/importance_sampling_ratio/min": 2.3244186735094823e-18,
      "sampling/sampling_logp_difference/max": 16.417081832885742,
      "sampling/sampling_logp_difference/mean": 0.11846176534891129,
      "step": 97,
      "step_time": 12.50525574600033
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1613.0,
      "completions/max_terminated_length": 1473.0,
      "completions/mean_length": 565.25,
      "completions/mean_terminated_length": 347.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2845763266086578,
      "epoch": 0.00098,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0279898643493652,
      "kl": 0.8884118981659412,
      "learning_rate": 4.859809833129187e-05,
      "loss": -0.0956,
      "num_tokens": 2301715.0,
      "reward": 0.5351350903511047,
      "reward_std": 0.6665053367614746,
      "rewards/rollout_reward_func/mean": 0.5351350903511047,
      "rewards/rollout_reward_func/std": 0.6536406874656677,
      "sampling/importance_sampling_ratio/max": 2.4657821655273438,
      "sampling/importance_sampling_ratio/mean": 1.169704794883728,
      "sampling/importance_sampling_ratio/min": 0.31398889422416687,
      "sampling/sampling_logp_difference/max": 0.4879746437072754,
      "sampling/sampling_logp_difference/mean": 0.033674150705337524,
      "step": 98,
      "step_time": 12.06715901600046
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 2573.0,
      "completions/max_terminated_length": 2573.0,
      "completions/mean_length": 638.125,
      "completions/mean_terminated_length": 717.7777709960938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6542884185910225,
      "epoch": 0.00099,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.211942195892334,
      "kl": 0.8857779912650585,
      "learning_rate": 4.8583920375875406e-05,
      "loss": 0.1062,
      "num_tokens": 2326800.0,
      "reward": 0.6849807500839233,
      "reward_std": 0.4539077877998352,
      "rewards/rollout_reward_func/mean": 0.6849807500839233,
      "rewards/rollout_reward_func/std": 0.7003189325332642,
      "sampling/importance_sampling_ratio/max": 1.6641603708267212,
      "sampling/importance_sampling_ratio/mean": 0.9379793405532837,
      "sampling/importance_sampling_ratio/min": 0.35520729422569275,
      "sampling/sampling_logp_difference/max": 2.3723065853118896,
      "sampling/sampling_logp_difference/mean": 0.07053622603416443,
      "step": 99,
      "step_time": 14.657188979999773
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 1761.0,
      "completions/max_terminated_length": 1617.0,
      "completions/mean_length": 638.0,
      "completions/mean_terminated_length": 531.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6315132826566696,
      "epoch": 0.001,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0639638900756836,
      "kl": 0.8950679190456867,
      "learning_rate": 4.8569499179581854e-05,
      "loss": 0.3059,
      "num_tokens": 2351396.0,
      "reward": 0.6734291315078735,
      "reward_std": 0.6232640743255615,
      "rewards/rollout_reward_func/mean": 0.6734291315078735,
      "rewards/rollout_reward_func/std": 0.6156803369522095,
      "sampling/importance_sampling_ratio/max": 1.5469081401824951,
      "sampling/importance_sampling_ratio/mean": 0.9379009008407593,
      "sampling/importance_sampling_ratio/min": 0.4286954402923584,
      "sampling/sampling_logp_difference/max": 0.3054943084716797,
      "sampling/sampling_logp_difference/mean": 0.041837964206933975,
      "step": 100,
      "step_time": 12.706590402999609
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1799.0,
      "completions/max_terminated_length": 1799.0,
      "completions/mean_length": 538.1875,
      "completions/mean_terminated_length": 432.3333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.585523322224617,
      "epoch": 0.00101,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8303850889205933,
      "kl": 0.8060843124985695,
      "learning_rate": 4.855483491428826e-05,
      "loss": -0.2447,
      "num_tokens": 2374551.0,
      "reward": -0.07351270318031311,
      "reward_std": 0.8419603109359741,
      "rewards/rollout_reward_func/mean": -0.07351270318031311,
      "rewards/rollout_reward_func/std": 0.8317177891731262,
      "sampling/importance_sampling_ratio/max": 1.7415587902069092,
      "sampling/importance_sampling_ratio/mean": 0.9468786120414734,
      "sampling/importance_sampling_ratio/min": 6.281470340686987e-16,
      "sampling/sampling_logp_difference/max": 13.250411033630371,
      "sampling/sampling_logp_difference/mean": 0.15282456576824188,
      "step": 101,
      "step_time": 12.173931203999473
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1601.0,
      "completions/max_terminated_length": 1601.0,
      "completions/mean_length": 579.0625,
      "completions/mean_terminated_length": 521.7000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9011313319206238,
      "epoch": 0.00102,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.2300543785095215,
      "kl": 0.6110033541917801,
      "learning_rate": 4.8539927754768616e-05,
      "loss": 0.5261,
      "num_tokens": 2398946.0,
      "reward": 0.4467778503894806,
      "reward_std": 0.705095648765564,
      "rewards/rollout_reward_func/mean": 0.4467778503894806,
      "rewards/rollout_reward_func/std": 0.8739113807678223,
      "sampling/importance_sampling_ratio/max": 1.8176132440567017,
      "sampling/importance_sampling_ratio/mean": 1.0508930683135986,
      "sampling/importance_sampling_ratio/min": 4.715132250064007e-16,
      "sampling/sampling_logp_difference/max": 5.167506694793701,
      "sampling/sampling_logp_difference/mean": 0.1270066350698471,
      "step": 102,
      "step_time": 12.027876874999492
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 1708.0,
      "completions/max_terminated_length": 731.0,
      "completions/mean_length": 547.875,
      "completions/mean_terminated_length": 300.4444580078125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5746607780456543,
      "epoch": 0.00103,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5726922750473022,
      "kl": 0.5137015916407108,
      "learning_rate": 4.8524777878691824e-05,
      "loss": -0.0594,
      "num_tokens": 2422209.0,
      "reward": 0.4165307581424713,
      "reward_std": 0.7622060179710388,
      "rewards/rollout_reward_func/mean": 0.4165307581424713,
      "rewards/rollout_reward_func/std": 0.7394797205924988,
      "sampling/importance_sampling_ratio/max": 1.389551043510437,
      "sampling/importance_sampling_ratio/mean": 0.9763103723526001,
      "sampling/importance_sampling_ratio/min": 0.5121654868125916,
      "sampling/sampling_logp_difference/max": 0.2609415054321289,
      "sampling/sampling_logp_difference/mean": 0.033407244831323624,
      "step": 103,
      "step_time": 12.373949243999505
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1712.0,
      "completions/max_terminated_length": 1595.0,
      "completions/mean_length": 475.375,
      "completions/mean_terminated_length": 392.6153869628906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6210803911089897,
      "epoch": 0.00104,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8864316344261169,
      "kl": 0.69330626539886,
      "learning_rate": 4.850938546661958e-05,
      "loss": -0.0519,
      "num_tokens": 2443779.0,
      "reward": 0.5597543716430664,
      "reward_std": 0.7438803315162659,
      "rewards/rollout_reward_func/mean": 0.5597543716430664,
      "rewards/rollout_reward_func/std": 0.7213174700737,
      "sampling/importance_sampling_ratio/max": 1.6869392395019531,
      "sampling/importance_sampling_ratio/mean": 0.9999841451644897,
      "sampling/importance_sampling_ratio/min": 0.28020286560058594,
      "sampling/sampling_logp_difference/max": 0.25801873207092285,
      "sampling/sampling_logp_difference/mean": 0.036903489381074905,
      "step": 104,
      "step_time": 11.207469994000348
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1621.0,
      "completions/max_terminated_length": 1621.0,
      "completions/mean_length": 548.8125,
      "completions/mean_terminated_length": 541.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6218102723360062,
      "epoch": 0.00105,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7923921346664429,
      "kl": 0.8638108782470226,
      "learning_rate": 4.849375070200421e-05,
      "loss": -0.1002,
      "num_tokens": 2467236.0,
      "reward": 0.4444293677806854,
      "reward_std": 0.8991751670837402,
      "rewards/rollout_reward_func/mean": 0.4444293677806854,
      "rewards/rollout_reward_func/std": 0.8727733492851257,
      "sampling/importance_sampling_ratio/max": 1.3536674976348877,
      "sampling/importance_sampling_ratio/mean": 0.8971585035324097,
      "sampling/importance_sampling_ratio/min": 1.8109411030309275e-05,
      "sampling/sampling_logp_difference/max": 3.5983827114105225,
      "sampling/sampling_logp_difference/mean": 0.08581438660621643,
      "step": 105,
      "step_time": 10.844490590000532
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 3044.0,
      "completions/max_terminated_length": 3044.0,
      "completions/mean_length": 852.25,
      "completions/mean_terminated_length": 1055.45458984375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.077312856912613,
      "epoch": 0.00106,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.599406898021698,
      "kl": 0.6678677275776863,
      "learning_rate": 4.8477873771186464e-05,
      "loss": -0.1008,
      "num_tokens": 2495684.0,
      "reward": 0.8193628787994385,
      "reward_std": 0.5109188556671143,
      "rewards/rollout_reward_func/mean": 0.8193628787994385,
      "rewards/rollout_reward_func/std": 0.5250175595283508,
      "sampling/importance_sampling_ratio/max": 1.3606970310211182,
      "sampling/importance_sampling_ratio/mean": 0.8700559139251709,
      "sampling/importance_sampling_ratio/min": 4.053232812643668e-27,
      "sampling/sampling_logp_difference/max": 23.633962631225586,
      "sampling/sampling_logp_difference/mean": 0.31731411814689636,
      "step": 106,
      "step_time": 16.2038826400003
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 1612.0,
      "completions/max_terminated_length": 1612.0,
      "completions/mean_length": 600.125,
      "completions/mean_terminated_length": 498.6666564941406,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8824206441640854,
      "epoch": 0.00107,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8099511861801147,
      "kl": 0.589387834072113,
      "learning_rate": 4.846175486339335e-05,
      "loss": 0.0869,
      "num_tokens": 2520201.0,
      "reward": 0.24924126267433167,
      "reward_std": 0.9072376489639282,
      "rewards/rollout_reward_func/mean": 0.24924126267433167,
      "rewards/rollout_reward_func/std": 0.9191129803657532,
      "sampling/importance_sampling_ratio/max": 1.5658494234085083,
      "sampling/importance_sampling_ratio/mean": 0.969744086265564,
      "sampling/importance_sampling_ratio/min": 0.6644586324691772,
      "sampling/sampling_logp_difference/max": 0.39588069915771484,
      "sampling/sampling_logp_difference/mean": 0.0365053154528141,
      "step": 107,
      "step_time": 12.286759744000847
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 2700.0,
      "completions/max_terminated_length": 860.0,
      "completions/mean_length": 618.25,
      "completions/mean_terminated_length": 368.77777099609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7089820057153702,
      "epoch": 0.00108,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9147666692733765,
      "kl": 0.5164729543030262,
      "learning_rate": 4.8445394170735813e-05,
      "loss": -0.1622,
      "num_tokens": 2544990.0,
      "reward": 0.5533000230789185,
      "reward_std": 0.7198785543441772,
      "rewards/rollout_reward_func/mean": 0.5533000230789185,
      "rewards/rollout_reward_func/std": 0.7261683940887451,
      "sampling/importance_sampling_ratio/max": 1.7777265310287476,
      "sampling/importance_sampling_ratio/mean": 0.9640104174613953,
      "sampling/importance_sampling_ratio/min": 1.4350082437886158e-06,
      "sampling/sampling_logp_difference/max": 9.472389221191406,
      "sampling/sampling_logp_difference/mean": 0.05173245072364807,
      "step": 108,
      "step_time": 15.597120163999534
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 886.0,
      "completions/max_terminated_length": 781.0,
      "completions/mean_length": 483.8125,
      "completions/mean_terminated_length": 358.8000183105469,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8222830295562744,
      "epoch": 0.00109,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0310304164886475,
      "kl": 0.7418055944144726,
      "learning_rate": 4.8428791888206516e-05,
      "loss": -0.0112,
      "num_tokens": 2567475.0,
      "reward": 0.2993038594722748,
      "reward_std": 0.7859958410263062,
      "rewards/rollout_reward_func/mean": 0.2993038594722748,
      "rewards/rollout_reward_func/std": 0.784038245677948,
      "sampling/importance_sampling_ratio/max": 2.3941028118133545,
      "sampling/importance_sampling_ratio/mean": 1.0557621717453003,
      "sampling/importance_sampling_ratio/min": 0.5189284086227417,
      "sampling/sampling_logp_difference/max": 0.5646533966064453,
      "sampling/sampling_logp_difference/mean": 0.035565052181482315,
      "step": 109,
      "step_time": 9.61443182300036
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 2709.0,
      "completions/max_terminated_length": 1855.0,
      "completions/mean_length": 641.9375,
      "completions/mean_terminated_length": 417.8999938964844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.725026696920395,
      "epoch": 0.0011,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0866097211837769,
      "kl": 0.514479286968708,
      "learning_rate": 4.841194821367741e-05,
      "loss": -0.1974,
      "num_tokens": 2592445.0,
      "reward": 0.44479769468307495,
      "reward_std": 0.8993635177612305,
      "rewards/rollout_reward_func/mean": 0.44479769468307495,
      "rewards/rollout_reward_func/std": 0.8715391755104065,
      "sampling/importance_sampling_ratio/max": 1.333624243736267,
      "sampling/importance_sampling_ratio/mean": 0.8027020692825317,
      "sampling/importance_sampling_ratio/min": 3.6233389471293856e-14,
      "sampling/sampling_logp_difference/max": 10.8357515335083,
      "sampling/sampling_logp_difference/mean": 0.13654404878616333,
      "step": 110,
      "step_time": 16.325197118000233
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1710.0,
      "completions/max_terminated_length": 1710.0,
      "completions/mean_length": 651.5625,
      "completions/mean_terminated_length": 624.2999877929688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.674040898680687,
      "epoch": 0.00111,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3284655809402466,
      "kl": 0.4721592143177986,
      "learning_rate": 4.8394863347897524e-05,
      "loss": 0.0663,
      "num_tokens": 2617591.0,
      "reward": 0.47023218870162964,
      "reward_std": 0.6642143130302429,
      "rewards/rollout_reward_func/mean": 0.47023218870162964,
      "rewards/rollout_reward_func/std": 0.6587124466896057,
      "sampling/importance_sampling_ratio/max": 1.5467126369476318,
      "sampling/importance_sampling_ratio/mean": 0.8192607760429382,
      "sampling/importance_sampling_ratio/min": 5.91510983054321e-18,
      "sampling/sampling_logp_difference/max": 10.373374938964844,
      "sampling/sampling_logp_difference/mean": 0.10860811173915863,
      "step": 111,
      "step_time": 13.398807575999854
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 985.0,
      "completions/max_terminated_length": 934.0,
      "completions/mean_length": 352.375,
      "completions/mean_terminated_length": 111.44444274902344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.857511818408966,
      "epoch": 0.00112,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1436257362365723,
      "kl": 0.6343204453587532,
      "learning_rate": 4.837753749449043e-05,
      "loss": 0.4032,
      "num_tokens": 2637652.0,
      "reward": 0.030777715146541595,
      "reward_std": 0.7399028539657593,
      "rewards/rollout_reward_func/mean": 0.030777715146541595,
      "rewards/rollout_reward_func/std": 0.7660248279571533,
      "sampling/importance_sampling_ratio/max": 1.4693158864974976,
      "sampling/importance_sampling_ratio/mean": 0.8582364320755005,
      "sampling/importance_sampling_ratio/min": 1.7555041026818555e-29,
      "sampling/sampling_logp_difference/max": 19.2490234375,
      "sampling/sampling_logp_difference/mean": 0.1904280036687851,
      "step": 112,
      "step_time": 10.478230928998983
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1638.0,
      "completions/max_terminated_length": 1638.0,
      "completions/mean_length": 378.9375,
      "completions/mean_terminated_length": 306.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.710848018527031,
      "epoch": 0.00113,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.8968822956085205,
      "kl": 0.7955609802156687,
      "learning_rate": 4.835997085995192e-05,
      "loss": -0.0024,
      "num_tokens": 2657841.0,
      "reward": 0.7225977778434753,
      "reward_std": 0.5137053728103638,
      "rewards/rollout_reward_func/mean": 0.7225977778434753,
      "rewards/rollout_reward_func/std": 0.49631085991859436,
      "sampling/importance_sampling_ratio/max": 2.14011549949646,
      "sampling/importance_sampling_ratio/mean": 1.0742825269699097,
      "sampling/importance_sampling_ratio/min": 0.5378368496894836,
      "sampling/sampling_logp_difference/max": 0.2268810272216797,
      "sampling/sampling_logp_difference/mean": 0.03591134771704674,
      "step": 113,
      "step_time": 11.020523139000034
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1915.0,
      "completions/max_terminated_length": 1915.0,
      "completions/mean_length": 548.1875,
      "completions/mean_terminated_length": 474.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7902559414505959,
      "epoch": 0.00114,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3231974840164185,
      "kl": 0.6815474927425385,
      "learning_rate": 4.834216365364748e-05,
      "loss": -0.0456,
      "num_tokens": 2680843.0,
      "reward": 0.4308680593967438,
      "reward_std": 0.7576680183410645,
      "rewards/rollout_reward_func/mean": 0.4308680593967438,
      "rewards/rollout_reward_func/std": 0.8081833124160767,
      "sampling/importance_sampling_ratio/max": 1.6819425821304321,
      "sampling/importance_sampling_ratio/mean": 1.0543144941329956,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.39678120613098145,
      "sampling/sampling_logp_difference/mean": 0.041170883923769,
      "step": 114,
      "step_time": 12.474389356000756
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 3075.0,
      "completions/max_terminated_length": 3075.0,
      "completions/mean_length": 548.375,
      "completions/mean_terminated_length": 455.0000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7080735638737679,
      "epoch": 0.00115,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.1096112728118896,
      "kl": 0.6797935105860233,
      "learning_rate": 4.8324116087809796e-05,
      "loss": -0.4932,
      "num_tokens": 2703828.0,
      "reward": 0.3571171760559082,
      "reward_std": 0.7076400518417358,
      "rewards/rollout_reward_func/mean": 0.3571171760559082,
      "rewards/rollout_reward_func/std": 0.8043490052223206,
      "sampling/importance_sampling_ratio/max": 2.3959882259368896,
      "sampling/importance_sampling_ratio/mean": 1.0927097797393799,
      "sampling/importance_sampling_ratio/min": 0.795071542263031,
      "sampling/sampling_logp_difference/max": 0.28455543518066406,
      "sampling/sampling_logp_difference/mean": 0.03530341759324074,
      "step": 115,
      "step_time": 15.287445441999807
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 860.0,
      "completions/max_terminated_length": 42.0,
      "completions/mean_length": 188.0625,
      "completions/mean_terminated_length": 6.44444465637207,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9127355739474297,
      "epoch": 0.00116,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.064640998840332,
      "kl": 0.5335907861590385,
      "learning_rate": 4.830582837753629e-05,
      "loss": -0.112,
      "num_tokens": 2720594.0,
      "reward": 0.7350000143051147,
      "reward_std": 0.5777266621589661,
      "rewards/rollout_reward_func/mean": 0.7350000143051147,
      "rewards/rollout_reward_func/std": 0.5991772413253784,
      "sampling/importance_sampling_ratio/max": 2.1874330043792725,
      "sampling/importance_sampling_ratio/mean": 1.0320769548416138,
      "sampling/importance_sampling_ratio/min": 8.742872837361194e-24,
      "sampling/sampling_logp_difference/max": 17.353124618530273,
      "sampling/sampling_logp_difference/mean": 0.21441768109798431,
      "step": 116,
      "step_time": 8.852478704999612
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 2655.0,
      "completions/max_terminated_length": 1663.0,
      "completions/mean_length": 576.875,
      "completions/mean_terminated_length": 403.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.059985786676407,
      "epoch": 0.00117,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2479445934295654,
      "kl": 0.6050208248198032,
      "learning_rate": 4.828730074078649e-05,
      "loss": -0.4169,
      "num_tokens": 2744062.0,
      "reward": 0.40769922733306885,
      "reward_std": 0.7674384117126465,
      "rewards/rollout_reward_func/mean": 0.40769922733306885,
      "rewards/rollout_reward_func/std": 0.7446826100349426,
      "sampling/importance_sampling_ratio/max": 2.440220832824707,
      "sampling/importance_sampling_ratio/mean": 0.9709947109222412,
      "sampling/importance_sampling_ratio/min": 0.32839033007621765,
      "sampling/sampling_logp_difference/max": 0.2365732192993164,
      "sampling/sampling_logp_difference/mean": 0.03796929121017456,
      "step": 117,
      "step_time": 15.249566166999557
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 2394.0,
      "completions/max_terminated_length": 735.0,
      "completions/mean_length": 461.0,
      "completions/mean_terminated_length": 196.375,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 2.0060382187366486,
      "epoch": 0.00118,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7781389951705933,
      "kl": 0.5616654045879841,
      "learning_rate": 4.8268533398379454e-05,
      "loss": -0.0134,
      "num_tokens": 2765519.0,
      "reward": 0.5484297871589661,
      "reward_std": 0.7509914636611938,
      "rewards/rollout_reward_func/mean": 0.5484297871589661,
      "rewards/rollout_reward_func/std": 0.7289549112319946,
      "sampling/importance_sampling_ratio/max": 1.5280230045318604,
      "sampling/importance_sampling_ratio/mean": 1.0139193534851074,
      "sampling/importance_sampling_ratio/min": 0.44538429379463196,
      "sampling/sampling_logp_difference/max": 0.3107430934906006,
      "sampling/sampling_logp_difference/mean": 0.03579236567020416,
      "step": 118,
      "step_time": 15.26357723000092
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5625,
      "completions/max_length": 3678.0,
      "completions/max_terminated_length": 30.0,
      "completions/mean_length": 613.625,
      "completions/mean_terminated_length": 19.285715103149414,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 1.9959446340799332,
      "epoch": 0.00119,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4228718280792236,
      "kl": 0.5257328264415264,
      "learning_rate": 4.8249526573991135e-05,
      "loss": -0.1071,
      "num_tokens": 2789098.0,
      "reward": 0.546511173248291,
      "reward_std": 0.7622096538543701,
      "rewards/rollout_reward_func/mean": 0.546511173248291,
      "rewards/rollout_reward_func/std": 0.7404257655143738,
      "sampling/importance_sampling_ratio/max": 1.271688461303711,
      "sampling/importance_sampling_ratio/mean": 0.8773758411407471,
      "sampling/importance_sampling_ratio/min": 0.34892964363098145,
      "sampling/sampling_logp_difference/max": 0.4599025249481201,
      "sampling/sampling_logp_difference/mean": 0.03672913834452629,
      "step": 119,
      "step_time": 18.721760522000295
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.625,
      "completions/max_length": 1928.0,
      "completions/max_terminated_length": 868.0,
      "completions/mean_length": 557.0625,
      "completions/mean_terminated_length": 290.8333435058594,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 2.1219392865896225,
      "epoch": 0.0012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6175590753555298,
      "kl": 0.936551108956337,
      "learning_rate": 4.8230280494151734e-05,
      "loss": 0.1257,
      "num_tokens": 2812827.0,
      "reward": 0.25217971205711365,
      "reward_std": 0.7999173402786255,
      "rewards/rollout_reward_func/mean": 0.25217971205711365,
      "rewards/rollout_reward_func/std": 0.8358803391456604,
      "sampling/importance_sampling_ratio/max": 1.5226387977600098,
      "sampling/importance_sampling_ratio/mean": 0.7933346033096313,
      "sampling/importance_sampling_ratio/min": 2.1993026999658794e-28,
      "sampling/sampling_logp_difference/max": 18.93212890625,
      "sampling/sampling_logp_difference/mean": 0.23466242849826813,
      "step": 120,
      "step_time": 14.220763490000536
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5625,
      "completions/max_length": 1637.0,
      "completions/max_terminated_length": 1637.0,
      "completions/mean_length": 520.25,
      "completions/mean_terminated_length": 495.14288330078125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9810590520501137,
      "epoch": 0.00121,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.776102900505066,
      "kl": 0.7075602151453495,
      "learning_rate": 4.821079538824297e-05,
      "loss": -0.0766,
      "num_tokens": 2836016.0,
      "reward": 0.1517150104045868,
      "reward_std": 0.7530897855758667,
      "rewards/rollout_reward_func/mean": 0.1517150104045868,
      "rewards/rollout_reward_func/std": 0.7532244920730591,
      "sampling/importance_sampling_ratio/max": 1.4925414323806763,
      "sampling/importance_sampling_ratio/mean": 1.0179210901260376,
      "sampling/importance_sampling_ratio/min": 0.5755860209465027,
      "sampling/sampling_logp_difference/max": 0.352785587310791,
      "sampling/sampling_logp_difference/mean": 0.035561878234148026,
      "step": 121,
      "step_time": 12.923838930998954
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.625,
      "completions/max_length": 1787.0,
      "completions/max_terminated_length": 771.0,
      "completions/mean_length": 562.25,
      "completions/mean_terminated_length": 266.8333435058594,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 2.168867364525795,
      "epoch": 0.00122,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.6303906440734863,
      "kl": 0.5567777045071125,
      "learning_rate": 4.8191071488495374e-05,
      "loss": 0.4233,
      "num_tokens": 2859729.0,
      "reward": 0.6890508532524109,
      "reward_std": 0.6440276503562927,
      "rewards/rollout_reward_func/mean": 0.6890508532524109,
      "rewards/rollout_reward_func/std": 0.6909214854240417,
      "sampling/importance_sampling_ratio/max": 2.45284104347229,
      "sampling/importance_sampling_ratio/mean": 0.9857180118560791,
      "sampling/importance_sampling_ratio/min": 4.4488416185158935e-18,
      "sampling/sampling_logp_difference/max": 19.179176330566406,
      "sampling/sampling_logp_difference/mean": 0.08935122936964035,
      "step": 122,
      "step_time": 13.93510346999983
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.6875,
      "completions/max_length": 2324.0,
      "completions/max_terminated_length": 740.0,
      "completions/mean_length": 516.9375,
      "completions/mean_terminated_length": 283.20001220703125,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 2.1699243932962418,
      "epoch": 0.00123,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6540182828903198,
      "kl": 0.5934898126870394,
      "learning_rate": 4.817110902998548e-05,
      "loss": 0.2173,
      "num_tokens": 2882284.0,
      "reward": 0.39933669567108154,
      "reward_std": 0.5825309753417969,
      "rewards/rollout_reward_func/mean": 0.39933669567108154,
      "rewards/rollout_reward_func/std": 0.6579245328903198,
      "sampling/importance_sampling_ratio/max": 1.3648136854171753,
      "sampling/importance_sampling_ratio/mean": 0.9745737910270691,
      "sampling/importance_sampling_ratio/min": 0.5656639933586121,
      "sampling/sampling_logp_difference/max": 0.34319019317626953,
      "sampling/sampling_logp_difference/mean": 0.034065887331962585,
      "step": 123,
      "step_time": 15.00999253800046
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5625,
      "completions/max_length": 1817.0,
      "completions/max_terminated_length": 889.0,
      "completions/mean_length": 721.9375,
      "completions/mean_terminated_length": 479.2857360839844,
      "completions/min_length": 16.0,
      "completions/min_terminated_length": 16.0,
      "entropy": 1.8830859512090683,
      "epoch": 0.00124,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.804628849029541,
      "kl": 0.508976299315691,
      "learning_rate": 4.81509082506331e-05,
      "loss": -0.2157,
      "num_tokens": 2909002.0,
      "reward": 0.6105675101280212,
      "reward_std": 0.5529724955558777,
      "rewards/rollout_reward_func/mean": 0.6105675101280212,
      "rewards/rollout_reward_func/std": 0.6298139691352844,
      "sampling/importance_sampling_ratio/max": 1.5507885217666626,
      "sampling/importance_sampling_ratio/mean": 1.0723888874053955,
      "sampling/importance_sampling_ratio/min": 0.5575966238975525,
      "sampling/sampling_logp_difference/max": 0.44043421745300293,
      "sampling/sampling_logp_difference/mean": 0.03469943627715111,
      "step": 124,
      "step_time": 15.350675042999228
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.75,
      "completions/max_length": 975.0,
      "completions/max_terminated_length": 911.0,
      "completions/mean_length": 380.0625,
      "completions/mean_terminated_length": 234.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.223814442753792,
      "epoch": 0.00125,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.467650055885315,
      "kl": 0.6756788827478886,
      "learning_rate": 4.813046939119838e-05,
      "loss": 0.0694,
      "num_tokens": 2929962.0,
      "reward": 0.35136348009109497,
      "reward_std": 0.748630940914154,
      "rewards/rollout_reward_func/mean": 0.35136348009109497,
      "rewards/rollout_reward_func/std": 0.723382294178009,
      "sampling/importance_sampling_ratio/max": 1.8278698921203613,
      "sampling/importance_sampling_ratio/mean": 0.7971331477165222,
      "sampling/importance_sampling_ratio/min": 1.6283841995801957e-20,
      "sampling/sampling_logp_difference/max": 18.820524215698242,
      "sampling/sampling_logp_difference/mean": 0.18106620013713837,
      "step": 125,
      "step_time": 11.51950903299985
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.625,
      "completions/max_length": 1606.0,
      "completions/max_terminated_length": 807.0,
      "completions/mean_length": 519.875,
      "completions/mean_terminated_length": 270.8333435058594,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 1.9699308425188065,
      "epoch": 0.00126,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8577008247375488,
      "kl": 0.5672223754227161,
      "learning_rate": 4.810979269527904e-05,
      "loss": 0.1806,
      "num_tokens": 2952790.0,
      "reward": 0.3581721782684326,
      "reward_std": 0.7846478819847107,
      "rewards/rollout_reward_func/mean": 0.3581721782684326,
      "rewards/rollout_reward_func/std": 0.8053376078605652,
      "sampling/importance_sampling_ratio/max": 1.5444529056549072,
      "sampling/importance_sampling_ratio/mean": 0.9249688982963562,
      "sampling/importance_sampling_ratio/min": 1.7731086268662078e-16,
      "sampling/sampling_logp_difference/max": 17.372316360473633,
      "sampling/sampling_logp_difference/mean": 0.10867972671985626,
      "step": 126,
      "step_time": 13.415750936999757
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.75,
      "completions/max_length": 2670.0,
      "completions/max_terminated_length": 44.0,
      "completions/mean_length": 731.1875,
      "completions/mean_terminated_length": 27.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.2371144890785217,
      "epoch": 0.00127,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3228814601898193,
      "kl": 0.5170325376093388,
      "learning_rate": 4.80888784093074e-05,
      "loss": -0.2764,
      "num_tokens": 2979575.0,
      "reward": 0.6027045845985413,
      "reward_std": 0.5759428143501282,
      "rewards/rollout_reward_func/mean": 0.6027045845985413,
      "rewards/rollout_reward_func/std": 0.6418101191520691,
      "sampling/importance_sampling_ratio/max": 2.1023471355438232,
      "sampling/importance_sampling_ratio/mean": 0.8917135000228882,
      "sampling/importance_sampling_ratio/min": 1.4450139695920683e-10,
      "sampling/sampling_logp_difference/max": 12.931964874267578,
      "sampling/sampling_logp_difference/mean": 0.07312598079442978,
      "step": 127,
      "step_time": 17.3916893920009
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 1733.0,
      "completions/max_terminated_length": 1733.0,
      "completions/mean_length": 628.25,
      "completions/mean_terminated_length": 374.8888854980469,
      "completions/min_length": 7.0,
      "completions/min_terminated_length": 7.0,
      "entropy": 2.033028095960617,
      "epoch": 0.00128,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9050607681274414,
      "kl": 0.8974899686872959,
      "learning_rate": 4.8067726782547474e-05,
      "loss": 0.0546,
      "num_tokens": 3004277.0,
      "reward": 0.49791714549064636,
      "reward_std": 0.8260049223899841,
      "rewards/rollout_reward_func/mean": 0.49791714549064636,
      "rewards/rollout_reward_func/std": 0.8136758804321289,
      "sampling/importance_sampling_ratio/max": 1.7927517890930176,
      "sampling/importance_sampling_ratio/mean": 0.8707818984985352,
      "sampling/importance_sampling_ratio/min": 4.863995854975656e-06,
      "sampling/sampling_logp_difference/max": 6.745263576507568,
      "sampling/sampling_logp_difference/mean": 0.05757336691021919,
      "step": 128,
      "step_time": 13.210125932999745
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.75,
      "completions/max_length": 1675.0,
      "completions/max_terminated_length": 1675.0,
      "completions/mean_length": 455.0,
      "completions/mean_terminated_length": 633.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7723678350448608,
      "epoch": 0.00129,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9346904754638672,
      "kl": 0.5633066967129707,
      "learning_rate": 4.804633806709197e-05,
      "loss": -0.1445,
      "num_tokens": 3025612.0,
      "reward": 0.6121518611907959,
      "reward_std": 0.73875492811203,
      "rewards/rollout_reward_func/mean": 0.6121518611907959,
      "rewards/rollout_reward_func/std": 0.7294540405273438,
      "sampling/importance_sampling_ratio/max": 1.6349518299102783,
      "sampling/importance_sampling_ratio/mean": 1.0034732818603516,
      "sampling/importance_sampling_ratio/min": 0.34239867329597473,
      "sampling/sampling_logp_difference/max": 0.3868241310119629,
      "sampling/sampling_logp_difference/mean": 0.0328049436211586,
      "step": 129,
      "step_time": 13.225849770999503
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.9375,
      "completions/max_length": 1844.0,
      "completions/max_terminated_length": 11.0,
      "completions/mean_length": 675.75,
      "completions/mean_terminated_length": 11.0,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 1.5643987283110619,
      "epoch": 0.0013,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8665101528167725,
      "kl": 0.4588968865573406,
      "learning_rate": 4.8024712517859316e-05,
      "loss": -0.1587,
      "num_tokens": 3051821.0,
      "reward": 0.5345557332038879,
      "reward_std": 0.6657187342643738,
      "rewards/rollout_reward_func/mean": 0.5345557332038879,
      "rewards/rollout_reward_func/std": 0.6516700387001038,
      "sampling/importance_sampling_ratio/max": 1.5221291780471802,
      "sampling/importance_sampling_ratio/mean": 0.916024923324585,
      "sampling/importance_sampling_ratio/min": 0.48253560066223145,
      "sampling/sampling_logp_difference/max": 0.23258590698242188,
      "sampling/sampling_logp_difference/mean": 0.029359307140111923,
      "step": 130,
      "step_time": 14.636339780000071
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.625,
      "completions/max_length": 1849.0,
      "completions/max_terminated_length": 919.0,
      "completions/mean_length": 417.5625,
      "completions/mean_terminated_length": 165.6666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9864765852689743,
      "epoch": 0.00131,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 24.10470962524414,
      "kl": 4.780883304774761,
      "learning_rate": 4.8002850392590614e-05,
      "loss": 0.1461,
      "num_tokens": 3073508.0,
      "reward": 0.4918329119682312,
      "reward_std": 0.8349891304969788,
      "rewards/rollout_reward_func/mean": 0.4918329119682312,
      "rewards/rollout_reward_func/std": 0.8216099143028259,
      "sampling/importance_sampling_ratio/max": 1.2414225339889526,
      "sampling/importance_sampling_ratio/mean": 0.6819342970848083,
      "sampling/importance_sampling_ratio/min": 4.3330356191972896e-08,
      "sampling/sampling_logp_difference/max": 11.350221633911133,
      "sampling/sampling_logp_difference/mean": 0.13151638209819794,
      "step": 131,
      "step_time": 14.328286448000199
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.75,
      "completions/max_length": 878.0,
      "completions/max_terminated_length": 30.0,
      "completions/mean_length": 284.9375,
      "completions/mean_terminated_length": 17.0,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 1.82517971098423,
      "epoch": 0.00132,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.288135051727295,
      "kl": 0.5887972339987755,
      "learning_rate": 4.7980751951846565e-05,
      "loss": -0.0925,
      "num_tokens": 3092248.0,
      "reward": 0.604968786239624,
      "reward_std": 0.6548165082931519,
      "rewards/rollout_reward_func/mean": 0.604968786239624,
      "rewards/rollout_reward_func/std": 0.638678789138794,
      "sampling/importance_sampling_ratio/max": 1.2372016906738281,
      "sampling/importance_sampling_ratio/mean": 0.731352686882019,
      "sampling/importance_sampling_ratio/min": 4.5146625695692555e-14,
      "sampling/sampling_logp_difference/max": 10.72125244140625,
      "sampling/sampling_logp_difference/mean": 0.1976451575756073,
      "step": 132,
      "step_time": 10.752250343000014
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.8125,
      "completions/max_length": 950.0,
      "completions/max_terminated_length": 754.0,
      "completions/mean_length": 454.4375,
      "completions/mean_terminated_length": 256.0,
      "completions/min_length": 6.0,
      "completions/min_terminated_length": 6.0,
      "entropy": 1.5140879154205322,
      "epoch": 0.00133,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7837804555892944,
      "kl": 0.5004850625991821,
      "learning_rate": 4.7958417459004353e-05,
      "loss": 0.0815,
      "num_tokens": 3114344.0,
      "reward": 0.413582444190979,
      "reward_std": 0.7232597470283508,
      "rewards/rollout_reward_func/mean": 0.413582444190979,
      "rewards/rollout_reward_func/std": 0.7358187437057495,
      "sampling/importance_sampling_ratio/max": 1.082815170288086,
      "sampling/importance_sampling_ratio/mean": 0.8696004152297974,
      "sampling/importance_sampling_ratio/min": 0.41922488808631897,
      "sampling/sampling_logp_difference/max": 0.246002197265625,
      "sampling/sampling_logp_difference/mean": 0.03103857859969139,
      "step": 133,
      "step_time": 11.313963209999656
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 2643.0,
      "completions/max_terminated_length": 1005.0,
      "completions/mean_length": 443.875,
      "completions/mean_terminated_length": 151.625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 1.5596542805433273,
      "epoch": 0.00134,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.808027744293213,
      "kl": 0.8299332931637764,
      "learning_rate": 4.793584718025453e-05,
      "loss": -0.4078,
      "num_tokens": 3135716.0,
      "reward": 0.665263295173645,
      "reward_std": 0.4773627817630768,
      "rewards/rollout_reward_func/mean": 0.665263295173645,
      "rewards/rollout_reward_func/std": 0.5137479305267334,
      "sampling/importance_sampling_ratio/max": 1.9282159805297852,
      "sampling/importance_sampling_ratio/mean": 0.8709302544593811,
      "sampling/importance_sampling_ratio/min": 1.9177454424284222e-14,
      "sampling/sampling_logp_difference/max": 10.56047248840332,
      "sampling/sampling_logp_difference/mean": 0.14425066113471985,
      "step": 134,
      "step_time": 15.878661510999791
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.6875,
      "completions/max_length": 1850.0,
      "completions/max_terminated_length": 1850.0,
      "completions/mean_length": 527.5625,
      "completions/mean_terminated_length": 560.7999877929688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2900739461183548,
      "epoch": 0.00135,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6424548625946045,
      "kl": 1.933522280305624,
      "learning_rate": 4.791304138459782e-05,
      "loss": -0.0934,
      "num_tokens": 3158914.0,
      "reward": 0.46451330184936523,
      "reward_std": 0.6659431457519531,
      "rewards/rollout_reward_func/mean": 0.46451330184936523,
      "rewards/rollout_reward_func/std": 0.6632189154624939,
      "sampling/importance_sampling_ratio/max": 1.573952317237854,
      "sampling/importance_sampling_ratio/mean": 0.920183002948761,
      "sampling/importance_sampling_ratio/min": 0.41945841908454895,
      "sampling/sampling_logp_difference/max": 0.263394832611084,
      "sampling/sampling_logp_difference/mean": 0.0311972014605999,
      "step": 135,
      "step_time": 13.775552462999713
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.8125,
      "completions/max_length": 915.0,
      "completions/max_terminated_length": 817.0,
      "completions/mean_length": 450.8125,
      "completions/mean_terminated_length": 523.6666870117188,
      "completions/min_length": 12.0,
      "completions/min_terminated_length": 12.0,
      "entropy": 1.5702635869383812,
      "epoch": 0.00136,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.8623454570770264,
      "kl": 0.5921471118927002,
      "learning_rate": 4.789000034384193e-05,
      "loss": 0.2547,
      "num_tokens": 3180970.0,
      "reward": 0.41561654210090637,
      "reward_std": 0.7429360151290894,
      "rewards/rollout_reward_func/mean": 0.41561654210090637,
      "rewards/rollout_reward_func/std": 0.7366409301757812,
      "sampling/importance_sampling_ratio/max": 1.6125787496566772,
      "sampling/importance_sampling_ratio/mean": 1.0141085386276245,
      "sampling/importance_sampling_ratio/min": 0.4993981420993805,
      "sampling/sampling_logp_difference/max": 0.4883403778076172,
      "sampling/sampling_logp_difference/mean": 0.032053884118795395,
      "step": 136,
      "step_time": 10.861362752999867
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.625,
      "completions/max_length": 958.0,
      "completions/max_terminated_length": 52.0,
      "completions/mean_length": 342.3125,
      "completions/mean_terminated_length": 24.33333396911621,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4446882754564285,
      "epoch": 0.00137,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5591638088226318,
      "kl": 0.521590031683445,
      "learning_rate": 4.7866724332598274e-05,
      "loss": 0.2822,
      "num_tokens": 3200574.0,
      "reward": 0.5487580299377441,
      "reward_std": 0.7568591833114624,
      "rewards/rollout_reward_func/mean": 0.5487580299377441,
      "rewards/rollout_reward_func/std": 0.7352384924888611,
      "sampling/importance_sampling_ratio/max": 1.6307905912399292,
      "sampling/importance_sampling_ratio/mean": 0.8677924275398254,
      "sampling/importance_sampling_ratio/min": 3.69244425718607e-17,
      "sampling/sampling_logp_difference/max": 20.023073196411133,
      "sampling/sampling_logp_difference/mean": 0.13651207089424133,
      "step": 137,
      "step_time": 10.855503429001601
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.75,
      "completions/max_length": 1647.0,
      "completions/max_terminated_length": 1647.0,
      "completions/mean_length": 547.0625,
      "completions/mean_terminated_length": 810.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3837649077177048,
      "epoch": 0.00138,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4721856117248535,
      "kl": 0.5649922601878643,
      "learning_rate": 4.784321362827878e-05,
      "loss": -0.1941,
      "num_tokens": 3224481.0,
      "reward": 0.5343878865242004,
      "reward_std": 0.6691931486129761,
      "rewards/rollout_reward_func/mean": 0.5343878865242004,
      "rewards/rollout_reward_func/std": 0.6569817066192627,
      "sampling/importance_sampling_ratio/max": 1.7321274280548096,
      "sampling/importance_sampling_ratio/mean": 0.8815635442733765,
      "sampling/importance_sampling_ratio/min": 0.04875391349196434,
      "sampling/sampling_logp_difference/max": 1.111185073852539,
      "sampling/sampling_logp_difference/mean": 0.036341216415166855,
      "step": 138,
      "step_time": 13.611686502999873
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 958.0,
      "completions/max_terminated_length": 951.0,
      "completions/mean_length": 503.9375,
      "completions/mean_terminated_length": 484.625,
      "completions/min_length": 12.0,
      "completions/min_terminated_length": 12.0,
      "entropy": 1.463924154639244,
      "epoch": 0.00139,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.7280540466308594,
      "kl": 0.7231616713106632,
      "learning_rate": 4.781946851109249e-05,
      "loss": 0.3336,
      "num_tokens": 3247509.0,
      "reward": 0.48596131801605225,
      "reward_std": 0.5993642210960388,
      "rewards/rollout_reward_func/mean": 0.48596131801605225,
      "rewards/rollout_reward_func/std": 0.7298629879951477,
      "sampling/importance_sampling_ratio/max": 1.5858944654464722,
      "sampling/importance_sampling_ratio/mean": 0.9457461833953857,
      "sampling/importance_sampling_ratio/min": 1.4352724519994808e-06,
      "sampling/sampling_logp_difference/max": 7.930485248565674,
      "sampling/sampling_logp_difference/mean": 0.05568278208374977,
      "step": 139,
      "step_time": 10.885603967999486
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 953.0,
      "completions/max_terminated_length": 953.0,
      "completions/mean_length": 493.875,
      "completions/mean_terminated_length": 430.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3332729861140251,
      "epoch": 0.0014,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3705096244812012,
      "kl": 0.8199928477406502,
      "learning_rate": 4.779548926404229e-05,
      "loss": 0.0718,
      "num_tokens": 3269662.0,
      "reward": 0.21676582098007202,
      "reward_std": 0.6961220502853394,
      "rewards/rollout_reward_func/mean": 0.21676582098007202,
      "rewards/rollout_reward_func/std": 0.68828946352005,
      "sampling/importance_sampling_ratio/max": 1.3183085918426514,
      "sampling/importance_sampling_ratio/mean": 0.9081477522850037,
      "sampling/importance_sampling_ratio/min": 0.6236458420753479,
      "sampling/sampling_logp_difference/max": 0.2718318700790405,
      "sampling/sampling_logp_difference/mean": 0.03436765819787979,
      "step": 140,
      "step_time": 10.171498322999469
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.625,
      "completions/max_length": 1794.0,
      "completions/max_terminated_length": 813.0,
      "completions/mean_length": 520.4375,
      "completions/mean_terminated_length": 238.33334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5081892311573029,
      "epoch": 0.00141,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2975428104400635,
      "kl": 0.47380803897976875,
      "learning_rate": 4.777127617292148e-05,
      "loss": 0.0142,
      "num_tokens": 3292435.0,
      "reward": 0.6673074960708618,
      "reward_std": 0.5209022760391235,
      "rewards/rollout_reward_func/mean": 0.6673074960708618,
      "rewards/rollout_reward_func/std": 0.5101658701896667,
      "sampling/importance_sampling_ratio/max": 1.2167648077011108,
      "sampling/importance_sampling_ratio/mean": 0.7932035326957703,
      "sampling/importance_sampling_ratio/min": 1.401298464324817e-45,
      "sampling/sampling_logp_difference/max": 24.843992233276367,
      "sampling/sampling_logp_difference/mean": 0.20544315874576569,
      "step": 141,
      "step_time": 13.725466193000102
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5625,
      "completions/max_length": 2608.0,
      "completions/max_terminated_length": 2608.0,
      "completions/mean_length": 663.4375,
      "completions/mean_terminated_length": 831.0000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.609919399023056,
      "epoch": 0.00142,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.174354076385498,
      "kl": 0.529135188087821,
      "learning_rate": 4.774682952631044e-05,
      "loss": 0.3059,
      "num_tokens": 3317955.0,
      "reward": 0.6763879060745239,
      "reward_std": 0.6196744441986084,
      "rewards/rollout_reward_func/mean": 0.6763879060745239,
      "rewards/rollout_reward_func/std": 0.610630452632904,
      "sampling/importance_sampling_ratio/max": 2.382641077041626,
      "sampling/importance_sampling_ratio/mean": 0.8881394863128662,
      "sampling/importance_sampling_ratio/min": 2.2438300878251626e-14,
      "sampling/sampling_logp_difference/max": 19.434280395507812,
      "sampling/sampling_logp_difference/mean": 0.11524233222007751,
      "step": 142,
      "step_time": 16.59388557200009
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 2724.0,
      "completions/max_terminated_length": 2724.0,
      "completions/mean_length": 674.6875,
      "completions/mean_terminated_length": 518.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.333674930036068,
      "epoch": 0.00143,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.9746310710906982,
      "kl": 0.75487370043993,
      "learning_rate": 4.77221496155731e-05,
      "loss": 0.0168,
      "num_tokens": 3343619.0,
      "reward": 0.7259149551391602,
      "reward_std": 0.4769268035888672,
      "rewards/rollout_reward_func/mean": 0.7259149551391602,
      "rewards/rollout_reward_func/std": 0.49035772681236267,
      "sampling/importance_sampling_ratio/max": 2.4071521759033203,
      "sampling/importance_sampling_ratio/mean": 1.1548748016357422,
      "sampling/importance_sampling_ratio/min": 2.1897633064399856e-17,
      "sampling/sampling_logp_difference/max": 6.895646572113037,
      "sampling/sampling_logp_difference/mean": 0.09718334674835205,
      "step": 143,
      "step_time": 15.719335339999361
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1742.0,
      "completions/max_terminated_length": 1742.0,
      "completions/mean_length": 470.5625,
      "completions/mean_terminated_length": 414.1000061035156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3193936198949814,
      "epoch": 0.00144,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6261600255966187,
      "kl": 0.7553512528538704,
      "learning_rate": 4.7697236734853564e-05,
      "loss": -0.1182,
      "num_tokens": 3365310.0,
      "reward": 0.5370469093322754,
      "reward_std": 0.5494652986526489,
      "rewards/rollout_reward_func/mean": 0.5370469093322754,
      "rewards/rollout_reward_func/std": 0.6477569341659546,
      "sampling/importance_sampling_ratio/max": 2.3373217582702637,
      "sampling/importance_sampling_ratio/mean": 1.0256404876708984,
      "sampling/importance_sampling_ratio/min": 6.973025654433919e-26,
      "sampling/sampling_logp_difference/max": 11.539703369140625,
      "sampling/sampling_logp_difference/mean": 0.17131836712360382,
      "step": 144,
      "step_time": 12.353602240999862
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 1544.0,
      "completions/max_terminated_length": 1544.0,
      "completions/mean_length": 453.375,
      "completions/mean_terminated_length": 372.77777099609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3956190645694733,
      "epoch": 0.00145,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.979783058166504,
      "kl": 0.7486136853694916,
      "learning_rate": 4.767209118107252e-05,
      "loss": 0.3133,
      "num_tokens": 3387407.0,
      "reward": 0.5336740016937256,
      "reward_std": 0.6350620985031128,
      "rewards/rollout_reward_func/mean": 0.5336740016937256,
      "rewards/rollout_reward_func/std": 0.6553295254707336,
      "sampling/importance_sampling_ratio/max": 1.796502709388733,
      "sampling/importance_sampling_ratio/mean": 0.8481465578079224,
      "sampling/importance_sampling_ratio/min": 9.711950594101756e-23,
      "sampling/sampling_logp_difference/max": 22.47354507446289,
      "sampling/sampling_logp_difference/mean": 0.1730124056339264,
      "step": 145,
      "step_time": 12.304313423999702
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1713.0,
      "completions/max_terminated_length": 1662.0,
      "completions/mean_length": 645.1875,
      "completions/mean_terminated_length": 698.45458984375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.205468311905861,
      "epoch": 0.00146,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3117578029632568,
      "kl": 0.6701645627617836,
      "learning_rate": 4.7646713253923755e-05,
      "loss": 0.0852,
      "num_tokens": 3412578.0,
      "reward": 0.4785740077495575,
      "reward_std": 0.6527953743934631,
      "rewards/rollout_reward_func/mean": 0.4785740077495575,
      "rewards/rollout_reward_func/std": 0.6507315635681152,
      "sampling/importance_sampling_ratio/max": 2.0066137313842773,
      "sampling/importance_sampling_ratio/mean": 0.8661498427391052,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 21.557090759277344,
      "sampling/sampling_logp_difference/mean": 0.09094826132059097,
      "step": 146,
      "step_time": 12.470201309000458
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1981.0,
      "completions/max_terminated_length": 1785.0,
      "completions/mean_length": 638.5,
      "completions/mean_terminated_length": 562.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3695790618658066,
      "epoch": 0.00147,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7273380756378174,
      "kl": 0.7973497547209263,
      "learning_rate": 4.762110325587056e-05,
      "loss": -0.0739,
      "num_tokens": 3437862.0,
      "reward": 0.29265519976615906,
      "reward_std": 0.7425129413604736,
      "rewards/rollout_reward_func/mean": 0.29265519976615906,
      "rewards/rollout_reward_func/std": 0.7956139445304871,
      "sampling/importance_sampling_ratio/max": 2.305065155029297,
      "sampling/importance_sampling_ratio/mean": 1.0837656259536743,
      "sampling/importance_sampling_ratio/min": 0.4340295195579529,
      "sampling/sampling_logp_difference/max": 0.33528947830200195,
      "sampling/sampling_logp_difference/mean": 0.030292969197034836,
      "step": 147,
      "step_time": 13.637924786999974
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1875.0,
      "completions/max_terminated_length": 1875.0,
      "completions/mean_length": 595.625,
      "completions/mean_terminated_length": 725.1538696289062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4972247406840324,
      "epoch": 0.00148,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0761669874191284,
      "kl": 0.7829418461769819,
      "learning_rate": 4.759526149214215e-05,
      "loss": 0.3063,
      "num_tokens": 3461901.0,
      "reward": 0.19861343502998352,
      "reward_std": 0.8462358713150024,
      "rewards/rollout_reward_func/mean": 0.19861343502998352,
      "rewards/rollout_reward_func/std": 0.8859919905662537,
      "sampling/importance_sampling_ratio/max": 1.3339331150054932,
      "sampling/importance_sampling_ratio/mean": 0.9323325157165527,
      "sampling/importance_sampling_ratio/min": 6.389309905397327e-29,
      "sampling/sampling_logp_difference/max": 14.574575424194336,
      "sampling/sampling_logp_difference/mean": 0.2348863184452057,
      "step": 148,
      "step_time": 11.835898734000239
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1613.0,
      "completions/max_terminated_length": 939.0,
      "completions/mean_length": 587.5,
      "completions/mean_terminated_length": 510.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2356909960508347,
      "epoch": 0.00149,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9591510891914368,
      "kl": 0.5283329878002405,
      "learning_rate": 4.756918827072995e-05,
      "loss": 0.0328,
      "num_tokens": 3486319.0,
      "reward": 0.23615360260009766,
      "reward_std": 0.8709913492202759,
      "rewards/rollout_reward_func/mean": 0.23615360260009766,
      "rewards/rollout_reward_func/std": 0.8551742434501648,
      "sampling/importance_sampling_ratio/max": 1.2259294986724854,
      "sampling/importance_sampling_ratio/mean": 0.9030539989471436,
      "sampling/importance_sampling_ratio/min": 0.31268006563186646,
      "sampling/sampling_logp_difference/max": 0.3246006965637207,
      "sampling/sampling_logp_difference/mean": 0.027560852468013763,
      "step": 149,
      "step_time": 11.690731229001358
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1755.0,
      "completions/max_terminated_length": 1755.0,
      "completions/mean_length": 615.25,
      "completions/mean_terminated_length": 625.8333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.253303200006485,
      "epoch": 0.0015,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9370242357254028,
      "kl": 1.4131730943918228,
      "learning_rate": 4.754288390238406e-05,
      "loss": -0.2806,
      "num_tokens": 3510561.0,
      "reward": 0.8684695959091187,
      "reward_std": 0.3720240592956543,
      "rewards/rollout_reward_func/mean": 0.8684695959091187,
      "rewards/rollout_reward_func/std": 0.3597652018070221,
      "sampling/importance_sampling_ratio/max": 1.9519286155700684,
      "sampling/importance_sampling_ratio/mean": 1.1535276174545288,
      "sampling/importance_sampling_ratio/min": 0.8137560486793518,
      "sampling/sampling_logp_difference/max": 0.268878698348999,
      "sampling/sampling_logp_difference/mean": 0.030846932902932167,
      "step": 150,
      "step_time": 11.731314632000249
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 896.0,
      "completions/max_terminated_length": 896.0,
      "completions/mean_length": 461.0,
      "completions/mean_terminated_length": 413.4285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0310759469866753,
      "epoch": 0.00151,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0078612565994263,
      "kl": 0.9579578973352909,
      "learning_rate": 4.751634870060941e-05,
      "loss": 0.1169,
      "num_tokens": 3532260.0,
      "reward": 0.7965092658996582,
      "reward_std": 0.4451878070831299,
      "rewards/rollout_reward_func/mean": 0.7965092658996582,
      "rewards/rollout_reward_func/std": 0.4376519024372101,
      "sampling/importance_sampling_ratio/max": 1.647131085395813,
      "sampling/importance_sampling_ratio/mean": 1.0084922313690186,
      "sampling/importance_sampling_ratio/min": 0.5035496354103088,
      "sampling/sampling_logp_difference/max": 0.3262462615966797,
      "sampling/sampling_logp_difference/mean": 0.031016474589705467,
      "step": 151,
      "step_time": 8.865029634000166
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1711.0,
      "completions/max_terminated_length": 1711.0,
      "completions/mean_length": 482.6875,
      "completions/mean_terminated_length": 459.66668701171875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 1.2064750045537949,
      "epoch": 0.00152,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7571476697921753,
      "kl": 1.3951954953372478,
      "learning_rate": 4.748958298166213e-05,
      "loss": -0.0522,
      "num_tokens": 3553588.0,
      "reward": 0.809762716293335,
      "reward_std": 0.5380722284317017,
      "rewards/rollout_reward_func/mean": 0.809762716293335,
      "rewards/rollout_reward_func/std": 0.5403003692626953,
      "sampling/importance_sampling_ratio/max": 1.2887388467788696,
      "sampling/importance_sampling_ratio/mean": 0.9872652292251587,
      "sampling/importance_sampling_ratio/min": 0.6929020285606384,
      "sampling/sampling_logp_difference/max": 0.2586801052093506,
      "sampling/sampling_logp_difference/mean": 0.02659543789923191,
      "step": 152,
      "step_time": 11.269429912999385
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1856.0,
      "completions/max_terminated_length": 1856.0,
      "completions/mean_length": 755.5625,
      "completions/mean_terminated_length": 747.4667358398438,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 1.1073381081223488,
      "epoch": 0.00153,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2418047189712524,
      "kl": 0.6511271446943283,
      "learning_rate": 4.7462587064545686e-05,
      "loss": -0.0074,
      "num_tokens": 3581011.0,
      "reward": 0.47330138087272644,
      "reward_std": 0.5959897041320801,
      "rewards/rollout_reward_func/mean": 0.47330138087272644,
      "rewards/rollout_reward_func/std": 0.651893675327301,
      "sampling/importance_sampling_ratio/max": 1.3509845733642578,
      "sampling/importance_sampling_ratio/mean": 0.9331455230712891,
      "sampling/importance_sampling_ratio/min": 0.4873972237110138,
      "sampling/sampling_logp_difference/max": 0.3032035827636719,
      "sampling/sampling_logp_difference/mean": 0.027998805046081543,
      "step": 153,
      "step_time": 12.562366718999783
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1682.0,
      "completions/max_terminated_length": 1682.0,
      "completions/mean_length": 492.1875,
      "completions/mean_terminated_length": 500.0714416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1063289418816566,
      "epoch": 0.00154,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0568978786468506,
      "kl": 1.3024825043976307,
      "learning_rate": 4.7435361271007185e-05,
      "loss": -0.1071,
      "num_tokens": 3604225.0,
      "reward": 0.4889979362487793,
      "reward_std": 0.8351126909255981,
      "rewards/rollout_reward_func/mean": 0.4889979362487793,
      "rewards/rollout_reward_func/std": 0.8218865394592285,
      "sampling/importance_sampling_ratio/max": 1.4620451927185059,
      "sampling/importance_sampling_ratio/mean": 0.9312275648117065,
      "sampling/importance_sampling_ratio/min": 1.186978975854306e-19,
      "sampling/sampling_logp_difference/max": 21.899948120117188,
      "sampling/sampling_logp_difference/mean": 0.19362102448940277,
      "step": 154,
      "step_time": 10.72423924099985
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1670.0,
      "completions/max_terminated_length": 1670.0,
      "completions/mean_length": 584.625,
      "completions/mean_terminated_length": 584.625,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.8671314492821693,
      "epoch": 0.00155,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1438034772872925,
      "kl": 1.4001969248056412,
      "learning_rate": 4.740790592553345e-05,
      "loss": -0.0359,
      "num_tokens": 3628399.0,
      "reward": 0.607708215713501,
      "reward_std": 0.6320009231567383,
      "rewards/rollout_reward_func/mean": 0.607708215713501,
      "rewards/rollout_reward_func/std": 0.634293258190155,
      "sampling/importance_sampling_ratio/max": 1.253600001335144,
      "sampling/importance_sampling_ratio/mean": 0.936351478099823,
      "sampling/importance_sampling_ratio/min": 5.94833515776827e-08,
      "sampling/sampling_logp_difference/max": 18.03852081298828,
      "sampling/sampling_logp_difference/mean": 0.10013365745544434,
      "step": 155,
      "step_time": 10.986543115999211
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1783.0,
      "completions/max_terminated_length": 861.0,
      "completions/mean_length": 599.4375,
      "completions/mean_terminated_length": 474.923095703125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.882049635052681,
      "epoch": 0.00156,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9213864803314209,
      "kl": 1.608371153473854,
      "learning_rate": 4.7380221355347196e-05,
      "loss": -0.1526,
      "num_tokens": 3653121.0,
      "reward": 0.5495386719703674,
      "reward_std": 0.7392517328262329,
      "rewards/rollout_reward_func/mean": 0.5495386719703674,
      "rewards/rollout_reward_func/std": 0.7373008131980896,
      "sampling/importance_sampling_ratio/max": 1.578055500984192,
      "sampling/importance_sampling_ratio/mean": 0.8822921514511108,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.3303360939025879,
      "sampling/sampling_logp_difference/mean": 0.027845967561006546,
      "step": 156,
      "step_time": 11.710642036000536
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1644.0,
      "completions/max_terminated_length": 1644.0,
      "completions/mean_length": 531.3125,
      "completions/mean_terminated_length": 534.6154174804688,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.9022594578564167,
      "epoch": 0.00157,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7497479915618896,
      "kl": 0.788049690425396,
      "learning_rate": 4.735230789040311e-05,
      "loss": 0.0978,
      "num_tokens": 3676667.0,
      "reward": 0.29442542791366577,
      "reward_std": 0.8134881854057312,
      "rewards/rollout_reward_func/mean": 0.29442542791366577,
      "rewards/rollout_reward_func/std": 0.7955484390258789,
      "sampling/importance_sampling_ratio/max": 1.3284611701965332,
      "sampling/importance_sampling_ratio/mean": 0.9607803225517273,
      "sampling/importance_sampling_ratio/min": 0.6178309917449951,
      "sampling/sampling_logp_difference/max": 0.2925400733947754,
      "sampling/sampling_logp_difference/mean": 0.02374746836721897,
      "step": 157,
      "step_time": 11.271723781999754
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1755.0,
      "completions/max_terminated_length": 1755.0,
      "completions/mean_length": 497.6875,
      "completions/mean_terminated_length": 483.60003662109375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.7864136509597301,
      "epoch": 0.00158,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6584041714668274,
      "kl": 1.4257199726998806,
      "learning_rate": 4.732416586338396e-05,
      "loss": -0.1027,
      "num_tokens": 3698922.0,
      "reward": 0.6598631143569946,
      "reward_std": 0.5352576971054077,
      "rewards/rollout_reward_func/mean": 0.6598631143569946,
      "rewards/rollout_reward_func/std": 0.5212937593460083,
      "sampling/importance_sampling_ratio/max": 1.071813702583313,
      "sampling/importance_sampling_ratio/mean": 0.9881418943405151,
      "sampling/importance_sampling_ratio/min": 0.8299121260643005,
      "sampling/sampling_logp_difference/max": 0.2287883758544922,
      "sampling/sampling_logp_difference/mean": 0.02226363494992256,
      "step": 158,
      "step_time": 10.570238234000954
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2452.0,
      "completions/max_terminated_length": 2452.0,
      "completions/mean_length": 527.125,
      "completions/mean_terminated_length": 500.357177734375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.7215167563408613,
      "epoch": 0.00159,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8740927577018738,
      "kl": 1.77104552090168,
      "learning_rate": 4.7295795609696547e-05,
      "loss": -0.0684,
      "num_tokens": 3722307.0,
      "reward": 0.4847217798233032,
      "reward_std": 0.6870274543762207,
      "rewards/rollout_reward_func/mean": 0.4847217798233032,
      "rewards/rollout_reward_func/std": 0.7260687351226807,
      "sampling/importance_sampling_ratio/max": 1.0416345596313477,
      "sampling/importance_sampling_ratio/mean": 0.9490992426872253,
      "sampling/importance_sampling_ratio/min": 0.7156460881233215,
      "sampling/sampling_logp_difference/max": 0.1399986743927002,
      "sampling/sampling_logp_difference/mean": 0.02017848752439022,
      "step": 159,
      "step_time": 12.159549965999304
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 983.0,
      "completions/max_terminated_length": 902.0,
      "completions/mean_length": 531.1875,
      "completions/mean_terminated_length": 501.0666809082031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9007115662097931,
      "epoch": 0.0016,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9932447075843811,
      "kl": 1.2698674350976944,
      "learning_rate": 4.726719746746782e-05,
      "loss": 0.0456,
      "num_tokens": 3746167.0,
      "reward": 0.45763522386550903,
      "reward_std": 0.5801395177841187,
      "rewards/rollout_reward_func/mean": 0.45763522386550903,
      "rewards/rollout_reward_func/std": 0.5604907274246216,
      "sampling/importance_sampling_ratio/max": 1.450630784034729,
      "sampling/importance_sampling_ratio/mean": 1.0389776229858398,
      "sampling/importance_sampling_ratio/min": 0.6431691646575928,
      "sampling/sampling_logp_difference/max": 0.334993839263916,
      "sampling/sampling_logp_difference/mean": 0.025158945471048355,
      "step": 160,
      "step_time": 8.501634291999835
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 869.0,
      "completions/max_terminated_length": 869.0,
      "completions/mean_length": 347.1875,
      "completions/mean_terminated_length": 347.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8279194813221693,
      "epoch": 0.00161,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7670500874519348,
      "kl": 1.4812407940626144,
      "learning_rate": 4.7238371777540744e-05,
      "loss": -0.0427,
      "num_tokens": 3766561.0,
      "reward": 0.4935108721256256,
      "reward_std": 0.7634821534156799,
      "rewards/rollout_reward_func/mean": 0.4935108721256256,
      "rewards/rollout_reward_func/std": 0.8149933815002441,
      "sampling/importance_sampling_ratio/max": 1.2361207008361816,
      "sampling/importance_sampling_ratio/mean": 1.0274944305419922,
      "sampling/importance_sampling_ratio/min": 0.868384838104248,
      "sampling/sampling_logp_difference/max": 0.20805776119232178,
      "sampling/sampling_logp_difference/mean": 0.01983175426721573,
      "step": 161,
      "step_time": 7.386108802999843
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 944.0,
      "completions/max_terminated_length": 944.0,
      "completions/mean_length": 644.625,
      "completions/mean_terminated_length": 644.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7273679599165916,
      "epoch": 0.00162,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.65094393491745,
      "kl": 1.2203245162963867,
      "learning_rate": 4.72093188834703e-05,
      "loss": 0.0711,
      "num_tokens": 3791619.0,
      "reward": 0.39639851450920105,
      "reward_std": 0.5242627263069153,
      "rewards/rollout_reward_func/mean": 0.39639851450920105,
      "rewards/rollout_reward_func/std": 0.5498889684677124,
      "sampling/importance_sampling_ratio/max": 1.1923495531082153,
      "sampling/importance_sampling_ratio/mean": 0.9846144914627075,
      "sampling/importance_sampling_ratio/min": 0.5824089646339417,
      "sampling/sampling_logp_difference/max": 0.2641134262084961,
      "sampling/sampling_logp_difference/mean": 0.018334196880459785,
      "step": 162,
      "step_time": 8.4244657990007
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 968.0,
      "completions/max_terminated_length": 968.0,
      "completions/mean_length": 397.9375,
      "completions/mean_terminated_length": 397.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.73673458956182,
      "epoch": 0.00163,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.617176353931427,
      "kl": 1.5956930071115494,
      "learning_rate": 4.7180039131519354e-05,
      "loss": 0.0228,
      "num_tokens": 3812510.0,
      "reward": 0.41280263662338257,
      "reward_std": 0.7235009670257568,
      "rewards/rollout_reward_func/mean": 0.41280263662338257,
      "rewards/rollout_reward_func/std": 0.7410818338394165,
      "sampling/importance_sampling_ratio/max": 1.0918843746185303,
      "sampling/importance_sampling_ratio/mean": 0.987321138381958,
      "sampling/importance_sampling_ratio/min": 0.8782848715782166,
      "sampling/sampling_logp_difference/max": 0.12225914001464844,
      "sampling/sampling_logp_difference/mean": 0.017314743250608444,
      "step": 163,
      "step_time": 7.701815213999907
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1782.0,
      "completions/max_terminated_length": 1782.0,
      "completions/mean_length": 614.875,
      "completions/mean_terminated_length": 613.4000244140625,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.8377712089568377,
      "epoch": 0.00164,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8056310415267944,
      "kl": 1.5706072300672531,
      "learning_rate": 4.715053287065456e-05,
      "loss": -0.0669,
      "num_tokens": 3837277.0,
      "reward": 0.5263782739639282,
      "reward_std": 0.5327835083007812,
      "rewards/rollout_reward_func/mean": 0.5263782739639282,
      "rewards/rollout_reward_func/std": 0.555076539516449,
      "sampling/importance_sampling_ratio/max": 1.1827281713485718,
      "sampling/importance_sampling_ratio/mean": 1.0122874975204468,
      "sampling/importance_sampling_ratio/min": 0.8339285850524902,
      "sampling/sampling_logp_difference/max": 0.36212682723999023,
      "sampling/sampling_logp_difference/mean": 0.024497613310813904,
      "step": 164,
      "step_time": 10.906703667999864
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 853.0,
      "completions/max_terminated_length": 853.0,
      "completions/mean_length": 380.4375,
      "completions/mean_terminated_length": 380.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8351107053458691,
      "epoch": 0.00165,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.48486247658729553,
      "kl": 1.69515909999609,
      "learning_rate": 4.7120800452542176e-05,
      "loss": 0.0244,
      "num_tokens": 3858369.0,
      "reward": 0.724025547504425,
      "reward_std": 0.2951430380344391,
      "rewards/rollout_reward_func/mean": 0.724025547504425,
      "rewards/rollout_reward_func/std": 0.4938052296638489,
      "sampling/importance_sampling_ratio/max": 1.3350915908813477,
      "sampling/importance_sampling_ratio/mean": 1.0077705383300781,
      "sampling/importance_sampling_ratio/min": 0.6963145732879639,
      "sampling/sampling_logp_difference/max": 0.17175078392028809,
      "sampling/sampling_logp_difference/mean": 0.020160436630249023,
      "step": 165,
      "step_time": 7.790227514999515
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1638.0,
      "completions/max_terminated_length": 1638.0,
      "completions/mean_length": 393.6875,
      "completions/mean_terminated_length": 393.6875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6303486451506615,
      "epoch": 0.00166,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9168211817741394,
      "kl": 1.7770038917660713,
      "learning_rate": 4.709084223154389e-05,
      "loss": -0.1199,
      "num_tokens": 3878930.0,
      "reward": 0.53781658411026,
      "reward_std": 0.6253374814987183,
      "rewards/rollout_reward_func/mean": 0.53781658411026,
      "rewards/rollout_reward_func/std": 0.6533367037773132,
      "sampling/importance_sampling_ratio/max": 2.3720765113830566,
      "sampling/importance_sampling_ratio/mean": 1.012768268585205,
      "sampling/importance_sampling_ratio/min": 6.354056035008556e-17,
      "sampling/sampling_logp_difference/max": 15.065998077392578,
      "sampling/sampling_logp_difference/mean": 0.24380485713481903,
      "step": 166,
      "step_time": 9.904431191000185
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 791.0,
      "completions/max_terminated_length": 791.0,
      "completions/mean_length": 444.3125,
      "completions/mean_terminated_length": 422.0666809082031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6627235673367977,
      "epoch": 0.00167,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.152322769165039,
      "kl": 1.3393200263381004,
      "learning_rate": 4.70606585647126e-05,
      "loss": 0.0559,
      "num_tokens": 3901102.0,
      "reward": 0.7282973527908325,
      "reward_std": 0.5031055212020874,
      "rewards/rollout_reward_func/mean": 0.7282973527908325,
      "rewards/rollout_reward_func/std": 0.48612818121910095,
      "sampling/importance_sampling_ratio/max": 1.243238091468811,
      "sampling/importance_sampling_ratio/mean": 0.9452941417694092,
      "sampling/importance_sampling_ratio/min": 0.467001348733902,
      "sampling/sampling_logp_difference/max": 0.3365671634674072,
      "sampling/sampling_logp_difference/mean": 0.019870227202773094,
      "step": 167,
      "step_time": 8.225956344000224
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 932.0,
      "completions/max_terminated_length": 932.0,
      "completions/mean_length": 473.625,
      "completions/mean_terminated_length": 473.625,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6972642838954926,
      "epoch": 0.00168,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9349740743637085,
      "kl": 1.6342423409223557,
      "learning_rate": 4.7030249811788126e-05,
      "loss": 0.1034,
      "num_tokens": 3923758.0,
      "reward": 0.38896745443344116,
      "reward_std": 0.5710576772689819,
      "rewards/rollout_reward_func/mean": 0.38896745443344116,
      "rewards/rollout_reward_func/std": 0.5569611191749573,
      "sampling/importance_sampling_ratio/max": 1.258313536643982,
      "sampling/importance_sampling_ratio/mean": 0.910763680934906,
      "sampling/importance_sampling_ratio/min": 6.168294208741543e-10,
      "sampling/sampling_logp_difference/max": 21.6185302734375,
      "sampling/sampling_logp_difference/mean": 0.13531209528446198,
      "step": 168,
      "step_time": 8.286803325000164
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1744.0,
      "completions/max_terminated_length": 1744.0,
      "completions/mean_length": 683.875,
      "completions/mean_terminated_length": 673.933349609375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 1.0073108654469252,
      "epoch": 0.00169,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.821285367012024,
      "kl": 1.23967494815588,
      "learning_rate": 4.699961633519297e-05,
      "loss": -0.2397,
      "num_tokens": 3950069.0,
      "reward": 0.26997750997543335,
      "reward_std": 0.60832679271698,
      "rewards/rollout_reward_func/mean": 0.26997750997543335,
      "rewards/rollout_reward_func/std": 0.6202860474586487,
      "sampling/importance_sampling_ratio/max": 1.3197342157363892,
      "sampling/importance_sampling_ratio/mean": 0.9762835502624512,
      "sampling/importance_sampling_ratio/min": 0.5959365963935852,
      "sampling/sampling_logp_difference/max": 0.27758312225341797,
      "sampling/sampling_logp_difference/mean": 0.029538070783019066,
      "step": 169,
      "step_time": 11.438789794999593
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1631.0,
      "completions/max_terminated_length": 1631.0,
      "completions/mean_length": 339.625,
      "completions/mean_terminated_length": 360.13336181640625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6393378004431725,
      "epoch": 0.0017,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4424673318862915,
      "kl": 2.0448066741228104,
      "learning_rate": 4.6968758500027945e-05,
      "loss": 0.0811,
      "num_tokens": 3969584.0,
      "reward": 0.5997680425643921,
      "reward_std": 0.5784438848495483,
      "rewards/rollout_reward_func/mean": 0.5997680425643921,
      "rewards/rollout_reward_func/std": 0.6449007987976074,
      "sampling/importance_sampling_ratio/max": 1.3768473863601685,
      "sampling/importance_sampling_ratio/mean": 1.0412057638168335,
      "sampling/importance_sampling_ratio/min": 0.9263357520103455,
      "sampling/sampling_logp_difference/max": 0.22372126579284668,
      "sampling/sampling_logp_difference/mean": 0.02219877950847149,
      "step": 170,
      "step_time": 10.453374952999638
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 913.0,
      "completions/max_terminated_length": 913.0,
      "completions/mean_length": 372.3125,
      "completions/mean_terminated_length": 372.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6244254820048809,
      "epoch": 0.00171,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6138743758201599,
      "kl": 1.9389575198292732,
      "learning_rate": 4.693767667406787e-05,
      "loss": -0.0017,
      "num_tokens": 3989847.0,
      "reward": 0.41501063108444214,
      "reward_std": 0.6675240993499756,
      "rewards/rollout_reward_func/mean": 0.41501063108444214,
      "rewards/rollout_reward_func/std": 0.7373371720314026,
      "sampling/importance_sampling_ratio/max": 1.1887915134429932,
      "sampling/importance_sampling_ratio/mean": 0.9618361592292786,
      "sampling/importance_sampling_ratio/min": 0.6269465684890747,
      "sampling/sampling_logp_difference/max": 0.2994422912597656,
      "sampling/sampling_logp_difference/mean": 0.027273712679743767,
      "step": 171,
      "step_time": 8.13458006099927
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1842.0,
      "completions/max_terminated_length": 1842.0,
      "completions/mean_length": 643.4375,
      "completions/mean_terminated_length": 643.4375,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6273978129029274,
      "epoch": 0.00172,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.452479600906372,
      "kl": 1.453748732805252,
      "learning_rate": 4.690637122775717e-05,
      "loss": -0.0422,
      "num_tokens": 4014687.0,
      "reward": 0.4843997657299042,
      "reward_std": 0.7644885778427124,
      "rewards/rollout_reward_func/mean": 0.4843997657299042,
      "rewards/rollout_reward_func/std": 0.7385701537132263,
      "sampling/importance_sampling_ratio/max": 1.5706771612167358,
      "sampling/importance_sampling_ratio/mean": 1.0888066291809082,
      "sampling/importance_sampling_ratio/min": 0.7032144665718079,
      "sampling/sampling_logp_difference/max": 0.3091338872909546,
      "sampling/sampling_logp_difference/mean": 0.023822421208024025,
      "step": 172,
      "step_time": 11.413303566999275
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1716.0,
      "completions/max_terminated_length": 1716.0,
      "completions/mean_length": 539.875,
      "completions/mean_terminated_length": 539.875,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5514593385159969,
      "epoch": 0.00173,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.255904197692871,
      "kl": 1.175136648118496,
      "learning_rate": 4.6874842534205433e-05,
      "loss": -0.0812,
      "num_tokens": 4038019.0,
      "reward": 0.7389968633651733,
      "reward_std": 0.5690313577651978,
      "rewards/rollout_reward_func/mean": 0.7389968633651733,
      "rewards/rollout_reward_func/std": 0.5931702256202698,
      "sampling/importance_sampling_ratio/max": 1.4699289798736572,
      "sampling/importance_sampling_ratio/mean": 1.0486552715301514,
      "sampling/importance_sampling_ratio/min": 0.8697987794876099,
      "sampling/sampling_logp_difference/max": 0.26449620723724365,
      "sampling/sampling_logp_difference/mean": 0.01569245010614395,
      "step": 173,
      "step_time": 10.986948811999355
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1644.0,
      "completions/max_terminated_length": 1644.0,
      "completions/mean_length": 378.0,
      "completions/mean_terminated_length": 378.0,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.5079810563474894,
      "epoch": 0.00174,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0563230514526367,
      "kl": 2.071729391813278,
      "learning_rate": 4.6843090969183025e-05,
      "loss": 0.0744,
      "num_tokens": 4058449.0,
      "reward": 0.34004658460617065,
      "reward_std": 0.7500537633895874,
      "rewards/rollout_reward_func/mean": 0.34004658460617065,
      "rewards/rollout_reward_func/std": 0.7383074760437012,
      "sampling/importance_sampling_ratio/max": 1.307531476020813,
      "sampling/importance_sampling_ratio/mean": 0.9835816025733948,
      "sampling/importance_sampling_ratio/min": 0.7718162536621094,
      "sampling/sampling_logp_difference/max": 0.2247772216796875,
      "sampling/sampling_logp_difference/mean": 0.020136190578341484,
      "step": 174,
      "step_time": 9.914837496999553
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1627.0,
      "completions/max_terminated_length": 1627.0,
      "completions/mean_length": 549.5,
      "completions/mean_terminated_length": 549.5,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.4937207866460085,
      "epoch": 0.00175,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6368958353996277,
      "kl": 1.484794795513153,
      "learning_rate": 4.681111691111654e-05,
      "loss": -0.0773,
      "num_tokens": 4081940.0,
      "reward": 0.7229952812194824,
      "reward_std": 0.5129567384719849,
      "rewards/rollout_reward_func/mean": 0.7229952812194824,
      "rewards/rollout_reward_func/std": 0.49560147523880005,
      "sampling/importance_sampling_ratio/max": 1.1822749376296997,
      "sampling/importance_sampling_ratio/mean": 0.9978392124176025,
      "sampling/importance_sampling_ratio/min": 0.7689469456672668,
      "sampling/sampling_logp_difference/max": 0.1583806276321411,
      "sampling/sampling_logp_difference/mean": 0.012237505055963993,
      "step": 175,
      "step_time": 10.381077340000502
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1804.0,
      "completions/max_terminated_length": 1804.0,
      "completions/mean_length": 528.6875,
      "completions/mean_terminated_length": 524.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6191283408552408,
      "epoch": 0.00176,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.19402015209198,
      "kl": 1.364200059324503,
      "learning_rate": 4.677892074108435e-05,
      "loss": -0.0948,
      "num_tokens": 4105129.0,
      "reward": 0.13114650547504425,
      "reward_std": 0.9320549964904785,
      "rewards/rollout_reward_func/mean": 0.13114650547504425,
      "rewards/rollout_reward_func/std": 0.9389170408248901,
      "sampling/importance_sampling_ratio/max": 1.5883028507232666,
      "sampling/importance_sampling_ratio/mean": 1.005928635597229,
      "sampling/importance_sampling_ratio/min": 0.6277012825012207,
      "sampling/sampling_logp_difference/max": 0.3319661617279053,
      "sampling/sampling_logp_difference/mean": 0.018472105264663696,
      "step": 176,
      "step_time": 11.40200411600108
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1581.0,
      "completions/max_terminated_length": 1581.0,
      "completions/mean_length": 682.0625,
      "completions/mean_terminated_length": 667.800048828125,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.6507823541760445,
      "epoch": 0.00177,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1116613149642944,
      "kl": 1.0576824992895126,
      "learning_rate": 4.6746502842812015e-05,
      "loss": 0.0228,
      "num_tokens": 4131434.0,
      "reward": 0.3810771703720093,
      "reward_std": 0.8804659843444824,
      "rewards/rollout_reward_func/mean": 0.3810771703720093,
      "rewards/rollout_reward_func/std": 0.8662812113761902,
      "sampling/importance_sampling_ratio/max": 1.5521836280822754,
      "sampling/importance_sampling_ratio/mean": 0.9664379358291626,
      "sampling/importance_sampling_ratio/min": 0.6033803820610046,
      "sampling/sampling_logp_difference/max": 0.22630596160888672,
      "sampling/sampling_logp_difference/mean": 0.021305255591869354,
      "step": 177,
      "step_time": 11.268170161999933
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1804.0,
      "completions/max_terminated_length": 1804.0,
      "completions/mean_length": 625.0625,
      "completions/mean_terminated_length": 625.0625,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.830624021589756,
      "epoch": 0.00178,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6292168498039246,
      "kl": 1.2418881431221962,
      "learning_rate": 4.671386360266774e-05,
      "loss": -0.0236,
      "num_tokens": 4156449.0,
      "reward": 0.015773557126522064,
      "reward_std": 1.0201389789581299,
      "rewards/rollout_reward_func/mean": 0.015773557126522064,
      "rewards/rollout_reward_func/std": 1.0169776678085327,
      "sampling/importance_sampling_ratio/max": 1.1397138833999634,
      "sampling/importance_sampling_ratio/mean": 0.9578301310539246,
      "sampling/importance_sampling_ratio/min": 0.7628179788589478,
      "sampling/sampling_logp_difference/max": 0.16054701805114746,
      "sampling/sampling_logp_difference/mean": 0.01391681469976902,
      "step": 178,
      "step_time": 10.783358027000304
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1882.0,
      "completions/max_terminated_length": 1882.0,
      "completions/mean_length": 715.9375,
      "completions/mean_terminated_length": 715.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6769013814628124,
      "epoch": 0.00179,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.873458743095398,
      "kl": 1.25007114559412,
      "learning_rate": 4.668100340965779e-05,
      "loss": 0.1335,
      "num_tokens": 4182616.0,
      "reward": 0.19279319047927856,
      "reward_std": 0.9000821113586426,
      "rewards/rollout_reward_func/mean": 0.19279319047927856,
      "rewards/rollout_reward_func/std": 0.8910132646560669,
      "sampling/importance_sampling_ratio/max": 1.4054218530654907,
      "sampling/importance_sampling_ratio/mean": 1.0629955530166626,
      "sampling/importance_sampling_ratio/min": 0.6903194785118103,
      "sampling/sampling_logp_difference/max": 0.2322993278503418,
      "sampling/sampling_logp_difference/mean": 0.02052212506532669,
      "step": 179,
      "step_time": 11.556659124999442
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1904.0,
      "completions/max_terminated_length": 1721.0,
      "completions/mean_length": 745.4375,
      "completions/mean_terminated_length": 668.2000122070312,
      "completions/min_length": 5.0,
      "completions/min_terminated_length": 5.0,
      "entropy": 0.7604969888925552,
      "epoch": 0.0018,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6802727580070496,
      "kl": 1.249889388680458,
      "learning_rate": 4.664792265542178e-05,
      "loss": 0.059,
      "num_tokens": 4209517.0,
      "reward": 0.36551615595817566,
      "reward_std": 0.8233534693717957,
      "rewards/rollout_reward_func/mean": 0.36551615595817566,
      "rewards/rollout_reward_func/std": 0.8057928085327148,
      "sampling/importance_sampling_ratio/max": 1.0767619609832764,
      "sampling/importance_sampling_ratio/mean": 0.922196626663208,
      "sampling/importance_sampling_ratio/min": 0.41155362129211426,
      "sampling/sampling_logp_difference/max": 0.3975827693939209,
      "sampling/sampling_logp_difference/mean": 0.02418501302599907,
      "step": 180,
      "step_time": 11.514578621000055
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1877.0,
      "completions/max_terminated_length": 1877.0,
      "completions/mean_length": 798.125,
      "completions/mean_terminated_length": 798.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7861449345946312,
      "epoch": 0.00181,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9569521546363831,
      "kl": 0.944803848862648,
      "learning_rate": 4.6614621734228084e-05,
      "loss": -0.1094,
      "num_tokens": 4237612.0,
      "reward": 0.3779129981994629,
      "reward_std": 0.8917311429977417,
      "rewards/rollout_reward_func/mean": 0.3779129981994629,
      "rewards/rollout_reward_func/std": 0.8712953925132751,
      "sampling/importance_sampling_ratio/max": 1.1152533292770386,
      "sampling/importance_sampling_ratio/mean": 0.9198100566864014,
      "sampling/importance_sampling_ratio/min": 0.7574284672737122,
      "sampling/sampling_logp_difference/max": 0.2290339469909668,
      "sampling/sampling_logp_difference/mean": 0.018821291625499725,
      "step": 181,
      "step_time": 11.813742917000127
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1708.0,
      "completions/max_terminated_length": 1708.0,
      "completions/mean_length": 630.5,
      "completions/mean_terminated_length": 630.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2100123949348927,
      "epoch": 0.00182,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8177403211593628,
      "kl": 0.9096072316169739,
      "learning_rate": 4.6581101042969104e-05,
      "loss": -0.0639,
      "num_tokens": 4262528.0,
      "reward": -0.04173080623149872,
      "reward_std": 1.0042924880981445,
      "rewards/rollout_reward_func/mean": -0.04173080623149872,
      "rewards/rollout_reward_func/std": 0.9737411141395569,
      "sampling/importance_sampling_ratio/max": 1.235540747642517,
      "sampling/importance_sampling_ratio/mean": 0.9897031784057617,
      "sampling/importance_sampling_ratio/min": 0.7679063677787781,
      "sampling/sampling_logp_difference/max": 0.21337556838989258,
      "sampling/sampling_logp_difference/mean": 0.02514311857521534,
      "step": 182,
      "step_time": 10.340481741999156
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1626.0,
      "completions/max_terminated_length": 1626.0,
      "completions/mean_length": 639.9375,
      "completions/mean_terminated_length": 639.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1087368354201317,
      "epoch": 0.00183,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5079019069671631,
      "kl": 0.543395109474659,
      "learning_rate": 4.654736098115652e-05,
      "loss": -0.0086,
      "num_tokens": 4288046.0,
      "reward": 0.2475505769252777,
      "reward_std": 0.9187079668045044,
      "rewards/rollout_reward_func/mean": 0.2475505769252777,
      "rewards/rollout_reward_func/std": 0.9259052276611328,
      "sampling/importance_sampling_ratio/max": 1.400246024131775,
      "sampling/importance_sampling_ratio/mean": 1.0169422626495361,
      "sampling/importance_sampling_ratio/min": 0.7865316867828369,
      "sampling/sampling_logp_difference/max": 0.1698167324066162,
      "sampling/sampling_logp_difference/mean": 0.018690265715122223,
      "step": 183,
      "step_time": 10.942726201999903
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1634.0,
      "completions/max_terminated_length": 1634.0,
      "completions/mean_length": 581.0,
      "completions/mean_terminated_length": 581.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.365300815552473,
      "epoch": 0.00184,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8381696343421936,
      "kl": 0.8696883283555508,
      "learning_rate": 4.651340195091656e-05,
      "loss": 0.1769,
      "num_tokens": 4312216.0,
      "reward": 0.3070935010910034,
      "reward_std": 0.8937063217163086,
      "rewards/rollout_reward_func/mean": 0.3070935010910034,
      "rewards/rollout_reward_func/std": 0.8666096329689026,
      "sampling/importance_sampling_ratio/max": 1.4297659397125244,
      "sampling/importance_sampling_ratio/mean": 0.9560354948043823,
      "sampling/importance_sampling_ratio/min": 4.429838185160406e-08,
      "sampling/sampling_logp_difference/max": 16.78682518005371,
      "sampling/sampling_logp_difference/mean": 0.14351403713226318,
      "step": 184,
      "step_time": 10.135254337000333
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2946.0,
      "completions/max_terminated_length": 2946.0,
      "completions/mean_length": 650.3125,
      "completions/mean_terminated_length": 630.5333862304688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0899128168821335,
      "epoch": 0.00185,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5613584518432617,
      "kl": 1.0186860226094723,
      "learning_rate": 4.647922435698521e-05,
      "loss": -0.0149,
      "num_tokens": 4337472.0,
      "reward": 0.33232200145721436,
      "reward_std": 0.935293436050415,
      "rewards/rollout_reward_func/mean": 0.33232200145721436,
      "rewards/rollout_reward_func/std": 0.923462450504303,
      "sampling/importance_sampling_ratio/max": 1.2431325912475586,
      "sampling/importance_sampling_ratio/mean": 0.9585608243942261,
      "sampling/importance_sampling_ratio/min": 0.6301966905593872,
      "sampling/sampling_logp_difference/max": 0.16792607307434082,
      "sampling/sampling_logp_difference/mean": 0.024770144373178482,
      "step": 185,
      "step_time": 13.757230327000343
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 3100.0,
      "completions/max_terminated_length": 3100.0,
      "completions/mean_length": 640.6875,
      "completions/mean_terminated_length": 628.6428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2842495068907738,
      "epoch": 0.00186,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6638988256454468,
      "kl": 0.9619948361068964,
      "learning_rate": 4.644482860670337e-05,
      "loss": 0.0156,
      "num_tokens": 4362124.0,
      "reward": 0.19545993208885193,
      "reward_std": 0.8948702812194824,
      "rewards/rollout_reward_func/mean": 0.19545993208885193,
      "rewards/rollout_reward_func/std": 0.8910754323005676,
      "sampling/importance_sampling_ratio/max": 1.0343917608261108,
      "sampling/importance_sampling_ratio/mean": 0.91145920753479,
      "sampling/importance_sampling_ratio/min": 0.6803298592567444,
      "sampling/sampling_logp_difference/max": 0.29377031326293945,
      "sampling/sampling_logp_difference/mean": 0.03418881073594093,
      "step": 186,
      "step_time": 14.202301616999648
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1791.0,
      "completions/max_terminated_length": 1791.0,
      "completions/mean_length": 544.1875,
      "completions/mean_terminated_length": 544.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4778831228613853,
      "epoch": 0.00187,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4665415287017822,
      "kl": 1.113618142902851,
      "learning_rate": 4.6410215110011995e-05,
      "loss": -0.0401,
      "num_tokens": 4385043.0,
      "reward": 0.21665765345096588,
      "reward_std": 0.777915358543396,
      "rewards/rollout_reward_func/mean": 0.21665765345096588,
      "rewards/rollout_reward_func/std": 0.940734326839447,
      "sampling/importance_sampling_ratio/max": 1.2576684951782227,
      "sampling/importance_sampling_ratio/mean": 1.0216612815856934,
      "sampling/importance_sampling_ratio/min": 0.7495752573013306,
      "sampling/sampling_logp_difference/max": 0.20101547241210938,
      "sampling/sampling_logp_difference/mean": 0.037659116089344025,
      "step": 187,
      "step_time": 10.170875044999775
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1794.0,
      "completions/max_terminated_length": 1794.0,
      "completions/mean_length": 667.5,
      "completions/mean_terminated_length": 667.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3149475380778313,
      "epoch": 0.00188,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.3335074186325073,
      "kl": 1.0449767895042896,
      "learning_rate": 4.6375384279447234e-05,
      "loss": 0.0306,
      "num_tokens": 4410340.0,
      "reward": 0.4482121467590332,
      "reward_std": 0.4771415591239929,
      "rewards/rollout_reward_func/mean": 0.4482121467590332,
      "rewards/rollout_reward_func/std": 0.8658757209777832,
      "sampling/importance_sampling_ratio/max": 1.18205988407135,
      "sampling/importance_sampling_ratio/mean": 0.9115084409713745,
      "sampling/importance_sampling_ratio/min": 3.309378118765416e-11,
      "sampling/sampling_logp_difference/max": 23.599205017089844,
      "sampling/sampling_logp_difference/mean": 0.36242005228996277,
      "step": 188,
      "step_time": 10.169093650999002
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1880.0,
      "completions/max_terminated_length": 1880.0,
      "completions/mean_length": 729.4375,
      "completions/mean_terminated_length": 729.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2018248364329338,
      "epoch": 0.00189,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6547307968139648,
      "kl": 0.9795884303748608,
      "learning_rate": 4.634033653013548e-05,
      "loss": 0.0308,
      "num_tokens": 4436678.0,
      "reward": -0.00010123848915100098,
      "reward_std": 0.744199812412262,
      "rewards/rollout_reward_func/mean": -0.00010123848915100098,
      "rewards/rollout_reward_func/std": 0.8635081052780151,
      "sampling/importance_sampling_ratio/max": 1.1981161832809448,
      "sampling/importance_sampling_ratio/mean": 1.0024056434631348,
      "sampling/importance_sampling_ratio/min": 0.8721359372138977,
      "sampling/sampling_logp_difference/max": 0.1863548755645752,
      "sampling/sampling_logp_difference/mean": 0.023926373571157455,
      "step": 189,
      "step_time": 10.47409372400034
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2576.0,
      "completions/max_terminated_length": 2576.0,
      "completions/mean_length": 737.75,
      "completions/mean_terminated_length": 726.6666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4299402832984924,
      "epoch": 0.0019,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9555301070213318,
      "kl": 1.0459455363452435,
      "learning_rate": 4.630507227978846e-05,
      "loss": -0.0788,
      "num_tokens": 4464168.0,
      "reward": 0.4167386591434479,
      "reward_std": 0.7268157005310059,
      "rewards/rollout_reward_func/mean": 0.4167386591434479,
      "rewards/rollout_reward_func/std": 0.7323936223983765,
      "sampling/importance_sampling_ratio/max": 1.219430923461914,
      "sampling/importance_sampling_ratio/mean": 1.023998498916626,
      "sampling/importance_sampling_ratio/min": 0.8336794376373291,
      "sampling/sampling_logp_difference/max": 0.30152225494384766,
      "sampling/sampling_logp_difference/mean": 0.024315200746059418,
      "step": 190,
      "step_time": 13.216604074000315
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2494.0,
      "completions/max_terminated_length": 2494.0,
      "completions/mean_length": 678.75,
      "completions/mean_terminated_length": 678.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0148128382861614,
      "epoch": 0.00191,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9180527925491333,
      "kl": 0.4954526424407959,
      "learning_rate": 4.626959194869825e-05,
      "loss": 0.0782,
      "num_tokens": 4490004.0,
      "reward": 0.5004785060882568,
      "reward_std": 0.7410280704498291,
      "rewards/rollout_reward_func/mean": 0.5004785060882568,
      "rewards/rollout_reward_func/std": 0.7159825563430786,
      "sampling/importance_sampling_ratio/max": 1.2647864818572998,
      "sampling/importance_sampling_ratio/mean": 0.992756187915802,
      "sampling/importance_sampling_ratio/min": 0.8491835594177246,
      "sampling/sampling_logp_difference/max": 0.2409353256225586,
      "sampling/sampling_logp_difference/mean": 0.02337854914367199,
      "step": 191,
      "step_time": 12.181147309999687
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1833.0,
      "completions/max_terminated_length": 1833.0,
      "completions/mean_length": 587.375,
      "completions/mean_terminated_length": 587.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.19494991004467,
      "epoch": 0.00192,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2387315332889557,
      "kl": 0.5244747791439295,
      "learning_rate": 4.623389595973223e-05,
      "loss": 0.0003,
      "num_tokens": 4514588.0,
      "reward": 0.6782480478286743,
      "reward_std": 0.6120697259902954,
      "rewards/rollout_reward_func/mean": 0.6782480478286743,
      "rewards/rollout_reward_func/std": 0.6031309366226196,
      "sampling/importance_sampling_ratio/max": 1.0420764684677124,
      "sampling/importance_sampling_ratio/mean": 0.9952988028526306,
      "sampling/importance_sampling_ratio/min": 0.9349057078361511,
      "sampling/sampling_logp_difference/max": 0.06543970108032227,
      "sampling/sampling_logp_difference/mean": 0.008370128460228443,
      "step": 192,
      "step_time": 9.953480612000021
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 898.0,
      "completions/max_terminated_length": 898.0,
      "completions/mean_length": 415.5,
      "completions/mean_terminated_length": 394.7333679199219,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1771658137440681,
      "epoch": 0.00193,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8394626379013062,
      "kl": 0.6432748921215534,
      "learning_rate": 4.619798473832807e-05,
      "loss": -0.0158,
      "num_tokens": 4536435.0,
      "reward": 0.1343187689781189,
      "reward_std": 0.6711304187774658,
      "rewards/rollout_reward_func/mean": 0.1343187689781189,
      "rewards/rollout_reward_func/std": 0.6686204075813293,
      "sampling/importance_sampling_ratio/max": 1.5519254207611084,
      "sampling/importance_sampling_ratio/mean": 1.0275399684906006,
      "sampling/importance_sampling_ratio/min": 0.9286418557167053,
      "sampling/sampling_logp_difference/max": 0.29319143295288086,
      "sampling/sampling_logp_difference/mean": 0.025683259591460228,
      "step": 193,
      "step_time": 7.548791956000059
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2683.0,
      "completions/max_terminated_length": 2683.0,
      "completions/mean_length": 635.9375,
      "completions/mean_terminated_length": 626.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0821861177682877,
      "epoch": 0.00194,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6343474984169006,
      "kl": 0.9304651170969009,
      "learning_rate": 4.6161858712488656e-05,
      "loss": -0.0592,
      "num_tokens": 4561696.0,
      "reward": 0.5910018682479858,
      "reward_std": 0.5650948286056519,
      "rewards/rollout_reward_func/mean": 0.5910018682479858,
      "rewards/rollout_reward_func/std": 0.5459673404693604,
      "sampling/importance_sampling_ratio/max": 1.332504153251648,
      "sampling/importance_sampling_ratio/mean": 1.0106383562088013,
      "sampling/importance_sampling_ratio/min": 0.8077538013458252,
      "sampling/sampling_logp_difference/max": 0.2015361785888672,
      "sampling/sampling_logp_difference/mean": 0.02656659670174122,
      "step": 194,
      "step_time": 12.77943405499991
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 916.0,
      "completions/max_terminated_length": 916.0,
      "completions/mean_length": 316.375,
      "completions/mean_terminated_length": 316.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8546969098970294,
      "epoch": 0.00195,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7728651762008667,
      "kl": 0.9103174768388271,
      "learning_rate": 4.612551831277701e-05,
      "loss": 0.0045,
      "num_tokens": 4580744.0,
      "reward": 0.34805238246917725,
      "reward_std": 0.7496694326400757,
      "rewards/rollout_reward_func/mean": 0.34805238246917725,
      "rewards/rollout_reward_func/std": 0.7242671847343445,
      "sampling/importance_sampling_ratio/max": 1.5922011137008667,
      "sampling/importance_sampling_ratio/mean": 1.033548355102539,
      "sampling/importance_sampling_ratio/min": 0.9201522469520569,
      "sampling/sampling_logp_difference/max": 0.39672255516052246,
      "sampling/sampling_logp_difference/mean": 0.022162994369864464,
      "step": 195,
      "step_time": 7.088162745999853
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 862.0,
      "completions/max_terminated_length": 862.0,
      "completions/mean_length": 348.1875,
      "completions/mean_terminated_length": 348.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6919427048414946,
      "epoch": 0.00196,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.43286627531051636,
      "kl": 0.5456393826752901,
      "learning_rate": 4.6088963972311144e-05,
      "loss": -0.0022,
      "num_tokens": 4601035.0,
      "reward": 0.5223503708839417,
      "reward_std": 0.52987140417099,
      "rewards/rollout_reward_func/mean": 0.5223503708839417,
      "rewards/rollout_reward_func/std": 0.559777557849884,
      "sampling/importance_sampling_ratio/max": 1.0698392391204834,
      "sampling/importance_sampling_ratio/mean": 1.0027172565460205,
      "sampling/importance_sampling_ratio/min": 0.9012739062309265,
      "sampling/sampling_logp_difference/max": 0.08850491046905518,
      "sampling/sampling_logp_difference/mean": 0.009562377817928791,
      "step": 196,
      "step_time": 7.570961438999802
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 915.0,
      "completions/max_terminated_length": 915.0,
      "completions/mean_length": 489.875,
      "completions/mean_terminated_length": 489.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1657178848981857,
      "epoch": 0.00197,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4391504228115082,
      "kl": 0.8017006972804666,
      "learning_rate": 4.6052196126758855e-05,
      "loss": -0.043,
      "num_tokens": 4624351.0,
      "reward": 0.5890946388244629,
      "reward_std": 0.5471168756484985,
      "rewards/rollout_reward_func/mean": 0.5890946388244629,
      "rewards/rollout_reward_func/std": 0.5481027364730835,
      "sampling/importance_sampling_ratio/max": 1.0958003997802734,
      "sampling/importance_sampling_ratio/mean": 0.9991846680641174,
      "sampling/importance_sampling_ratio/min": 0.8985453248023987,
      "sampling/sampling_logp_difference/max": 0.2806735038757324,
      "sampling/sampling_logp_difference/mean": 0.020023897290229797,
      "step": 197,
      "step_time": 7.746471892000045
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 826.0,
      "completions/max_terminated_length": 826.0,
      "completions/mean_length": 323.8125,
      "completions/mean_terminated_length": 323.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9488779027014971,
      "epoch": 0.00198,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2755127251148224,
      "kl": 0.6967457830905914,
      "learning_rate": 4.60152152143326e-05,
      "loss": -0.0471,
      "num_tokens": 4643929.0,
      "reward": 0.6654882431030273,
      "reward_std": 0.638024628162384,
      "rewards/rollout_reward_func/mean": 0.6654882431030273,
      "rewards/rollout_reward_func/std": 0.6298072934150696,
      "sampling/importance_sampling_ratio/max": 1.0442224740982056,
      "sampling/importance_sampling_ratio/mean": 0.9232609272003174,
      "sampling/importance_sampling_ratio/min": 0.6164547204971313,
      "sampling/sampling_logp_difference/max": 0.3905951976776123,
      "sampling/sampling_logp_difference/mean": 0.021257750689983368,
      "step": 198,
      "step_time": 7.160518301000138
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 880.0,
      "completions/max_terminated_length": 880.0,
      "completions/mean_length": 429.625,
      "completions/mean_terminated_length": 399.8000183105469,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1852735420688987,
      "epoch": 0.00199,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5488576889038086,
      "kl": 0.9028103277087212,
      "learning_rate": 4.5978021675784234e-05,
      "loss": -0.0541,
      "num_tokens": 4665714.0,
      "reward": 0.2135217785835266,
      "reward_std": 0.700448751449585,
      "rewards/rollout_reward_func/mean": 0.2135217785835266,
      "rewards/rollout_reward_func/std": 0.6921167373657227,
      "sampling/importance_sampling_ratio/max": 1.2268354892730713,
      "sampling/importance_sampling_ratio/mean": 0.9460150003433228,
      "sampling/importance_sampling_ratio/min": 0.40158459544181824,
      "sampling/sampling_logp_difference/max": 0.46643638610839844,
      "sampling/sampling_logp_difference/mean": 0.027755076065659523,
      "step": 199,
      "step_time": 7.428715269999884
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1907.0,
      "completions/max_terminated_length": 898.0,
      "completions/mean_length": 407.25,
      "completions/mean_terminated_length": 256.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3165307668969035,
      "epoch": 0.002,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.826464831829071,
      "kl": 0.4715082682669163,
      "learning_rate": 4.594061595439978e-05,
      "loss": -0.0879,
      "num_tokens": 4687046.0,
      "reward": 0.454461932182312,
      "reward_std": 0.583815336227417,
      "rewards/rollout_reward_func/mean": 0.454461932182312,
      "rewards/rollout_reward_func/std": 0.564020574092865,
      "sampling/importance_sampling_ratio/max": 1.1717621088027954,
      "sampling/importance_sampling_ratio/mean": 0.8549085855484009,
      "sampling/importance_sampling_ratio/min": 1.8726668032631408e-18,
      "sampling/sampling_logp_difference/max": 28.568645477294922,
      "sampling/sampling_logp_difference/mean": 0.2909993529319763,
      "step": 200,
      "step_time": 11.018483509999896
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 888.0,
      "completions/max_terminated_length": 888.0,
      "completions/mean_length": 331.625,
      "completions/mean_terminated_length": 286.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9558559320867062,
      "epoch": 0.00201,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5943700075149536,
      "kl": 0.9763778410851955,
      "learning_rate": 4.590299849599411e-05,
      "loss": -0.0763,
      "num_tokens": 4707055.0,
      "reward": 0.4703453779220581,
      "reward_std": 0.6095690727233887,
      "rewards/rollout_reward_func/mean": 0.4703453779220581,
      "rewards/rollout_reward_func/std": 0.6558030843734741,
      "sampling/importance_sampling_ratio/max": 1.0035293102264404,
      "sampling/importance_sampling_ratio/mean": 0.9082185626029968,
      "sampling/importance_sampling_ratio/min": 2.6619015325302087e-10,
      "sampling/sampling_logp_difference/max": 22.083099365234375,
      "sampling/sampling_logp_difference/mean": 0.1633288562297821,
      "step": 201,
      "step_time": 7.724533239000721
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1624.0,
      "completions/max_terminated_length": 813.0,
      "completions/mean_length": 431.375,
      "completions/mean_terminated_length": 351.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7870935741811991,
      "epoch": 0.00202,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2292656898498535,
      "kl": 0.4281471399590373,
      "learning_rate": 4.586516974890567e-05,
      "loss": 0.3244,
      "num_tokens": 4728921.0,
      "reward": 0.6055127382278442,
      "reward_std": 0.6521220207214355,
      "rewards/rollout_reward_func/mean": 0.6055127382278442,
      "rewards/rollout_reward_func/std": 0.6345822811126709,
      "sampling/importance_sampling_ratio/max": 1.0322706699371338,
      "sampling/importance_sampling_ratio/mean": 0.9794826507568359,
      "sampling/importance_sampling_ratio/min": 0.8485531210899353,
      "sampling/sampling_logp_difference/max": 0.1944732666015625,
      "sampling/sampling_logp_difference/mean": 0.02197406254708767,
      "step": 202,
      "step_time": 9.837652435999189
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1832.0,
      "completions/max_terminated_length": 1832.0,
      "completions/mean_length": 813.25,
      "completions/mean_terminated_length": 721.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4159990902990103,
      "epoch": 0.00203,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8663976192474365,
      "kl": 0.629713486880064,
      "learning_rate": 4.582713016399112e-05,
      "loss": -0.1973,
      "num_tokens": 4757597.0,
      "reward": 0.10228438675403595,
      "reward_std": 0.809098482131958,
      "rewards/rollout_reward_func/mean": 0.10228438675403595,
      "rewards/rollout_reward_func/std": 0.7958199977874756,
      "sampling/importance_sampling_ratio/max": 1.8511407375335693,
      "sampling/importance_sampling_ratio/mean": 0.945542573928833,
      "sampling/importance_sampling_ratio/min": 4.879155501434254e-23,
      "sampling/sampling_logp_difference/max": 20.442554473876953,
      "sampling/sampling_logp_difference/mean": 0.15386725962162018,
      "step": 203,
      "step_time": 13.498069165000288
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 973.0,
      "completions/max_terminated_length": 973.0,
      "completions/mean_length": 549.125,
      "completions/mean_terminated_length": 533.0667114257812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2822169326245785,
      "epoch": 0.00204,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8096746206283569,
      "kl": 0.739327397197485,
      "learning_rate": 4.578888019461996e-05,
      "loss": -0.0798,
      "num_tokens": 4781555.0,
      "reward": 0.008457034826278687,
      "reward_std": 0.5652322769165039,
      "rewards/rollout_reward_func/mean": 0.008457034826278687,
      "rewards/rollout_reward_func/std": 0.5747915506362915,
      "sampling/importance_sampling_ratio/max": 1.318725347518921,
      "sampling/importance_sampling_ratio/mean": 0.9977083206176758,
      "sampling/importance_sampling_ratio/min": 0.6982685923576355,
      "sampling/sampling_logp_difference/max": 0.2069845199584961,
      "sampling/sampling_logp_difference/mean": 0.02490164339542389,
      "step": 204,
      "step_time": 8.378189013999418
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 915.0,
      "completions/max_terminated_length": 915.0,
      "completions/mean_length": 338.4375,
      "completions/mean_terminated_length": 356.73333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.213806388899684,
      "epoch": 0.00205,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.42113035917282104,
      "kl": 0.9620688185095787,
      "learning_rate": 4.5750420296669125e-05,
      "loss": -0.0336,
      "num_tokens": 4801677.0,
      "reward": 0.3881481885910034,
      "reward_std": 0.5716249942779541,
      "rewards/rollout_reward_func/mean": 0.3881481885910034,
      "rewards/rollout_reward_func/std": 0.5574859976768494,
      "sampling/importance_sampling_ratio/max": 1.1503593921661377,
      "sampling/importance_sampling_ratio/mean": 0.8686246275901794,
      "sampling/importance_sampling_ratio/min": 1.1254853493260737e-36,
      "sampling/sampling_logp_difference/max": 22.905092239379883,
      "sampling/sampling_logp_difference/mean": 0.8094610571861267,
      "step": 205,
      "step_time": 7.8893698830001995
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 931.0,
      "completions/max_terminated_length": 907.0,
      "completions/mean_length": 512.4375,
      "completions/mean_terminated_length": 484.5333557128906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0558717176318169,
      "epoch": 0.00206,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5475472807884216,
      "kl": 0.7360203191637993,
      "learning_rate": 4.5711750928517545e-05,
      "loss": 0.0523,
      "num_tokens": 4824899.0,
      "reward": 0.46925240755081177,
      "reward_std": 0.6575711965560913,
      "rewards/rollout_reward_func/mean": 0.46925240755081177,
      "rewards/rollout_reward_func/std": 0.6524302959442139,
      "sampling/importance_sampling_ratio/max": 1.031027913093567,
      "sampling/importance_sampling_ratio/mean": 0.9474260807037354,
      "sampling/importance_sampling_ratio/min": 0.40702876448631287,
      "sampling/sampling_logp_difference/max": 0.37520134449005127,
      "sampling/sampling_logp_difference/mean": 0.02641621232032776,
      "step": 206,
      "step_time": 8.074118069000178
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 931.0,
      "completions/max_terminated_length": 931.0,
      "completions/mean_length": 351.0,
      "completions/mean_terminated_length": 337.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7407616544514894,
      "epoch": 0.00207,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1602873802185059,
      "kl": 0.5394846349954605,
      "learning_rate": 4.5672872551040724e-05,
      "loss": 0.0986,
      "num_tokens": 4844412.0,
      "reward": 0.8648368120193481,
      "reward_std": 0.3822992146015167,
      "rewards/rollout_reward_func/mean": 0.8648368120193481,
      "rewards/rollout_reward_func/std": 0.3693467378616333,
      "sampling/importance_sampling_ratio/max": 1.450052261352539,
      "sampling/importance_sampling_ratio/mean": 0.9787561893463135,
      "sampling/importance_sampling_ratio/min": 1.0377371110329281e-19,
      "sampling/sampling_logp_difference/max": 16.654592514038086,
      "sampling/sampling_logp_difference/mean": 0.3036476969718933,
      "step": 207,
      "step_time": 8.147521632000007
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1508.0,
      "completions/max_terminated_length": 862.0,
      "completions/mean_length": 436.375,
      "completions/mean_terminated_length": 310.7692565917969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0042121075093746,
      "epoch": 0.00208,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6607486009597778,
      "kl": 0.7787455879151821,
      "learning_rate": 4.5633785627605175e-05,
      "loss": -0.1504,
      "num_tokens": 4865723.0,
      "reward": 0.6561787128448486,
      "reward_std": 0.539013147354126,
      "rewards/rollout_reward_func/mean": 0.6561787128448486,
      "rewards/rollout_reward_func/std": 0.5268288254737854,
      "sampling/importance_sampling_ratio/max": 1.0459108352661133,
      "sampling/importance_sampling_ratio/mean": 0.9368011951446533,
      "sampling/importance_sampling_ratio/min": 0.6280043721199036,
      "sampling/sampling_logp_difference/max": 0.3047184944152832,
      "sampling/sampling_logp_difference/mean": 0.02794496901333332,
      "step": 208,
      "step_time": 10.22057830499898
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 871.0,
      "completions/max_terminated_length": 871.0,
      "completions/mean_length": 567.5625,
      "completions/mean_terminated_length": 548.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1490860637277365,
      "epoch": 0.00209,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2356979846954346,
      "kl": 0.6757755996659398,
      "learning_rate": 4.559449062406296e-05,
      "loss": 0.3777,
      "num_tokens": 4889773.0,
      "reward": 0.5444471836090088,
      "reward_std": 0.6200743913650513,
      "rewards/rollout_reward_func/mean": 0.5444471836090088,
      "rewards/rollout_reward_func/std": 0.640548825263977,
      "sampling/importance_sampling_ratio/max": 2.064633846282959,
      "sampling/importance_sampling_ratio/mean": 1.1082066297531128,
      "sampling/importance_sampling_ratio/min": 0.7443817257881165,
      "sampling/sampling_logp_difference/max": 0.18340063095092773,
      "sampling/sampling_logp_difference/mean": 0.026794370263814926,
      "step": 209,
      "step_time": 8.71277239300025
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 935.0,
      "completions/max_terminated_length": 935.0,
      "completions/mean_length": 496.125,
      "completions/mean_terminated_length": 483.66668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1122111836448312,
      "epoch": 0.0021,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6340797543525696,
      "kl": 0.8742159940302372,
      "learning_rate": 4.555498800874612e-05,
      "loss": -0.0967,
      "num_tokens": 4912302.0,
      "reward": 0.5935119390487671,
      "reward_std": 0.5403187274932861,
      "rewards/rollout_reward_func/mean": 0.5935119390487671,
      "rewards/rollout_reward_func/std": 0.5420464277267456,
      "sampling/importance_sampling_ratio/max": 1.0430314540863037,
      "sampling/importance_sampling_ratio/mean": 0.9180165529251099,
      "sampling/importance_sampling_ratio/min": 0.37723609805107117,
      "sampling/sampling_logp_difference/max": 0.16040563583374023,
      "sampling/sampling_logp_difference/mean": 0.026559725403785706,
      "step": 210,
      "step_time": 9.201866888000495
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 889.0,
      "completions/max_terminated_length": 889.0,
      "completions/mean_length": 336.25,
      "completions/mean_terminated_length": 336.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9952372275292873,
      "epoch": 0.00211,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.285069465637207,
      "kl": 0.5587481819093227,
      "learning_rate": 4.5515278252461064e-05,
      "loss": -0.1677,
      "num_tokens": 4932844.0,
      "reward": 0.4079897701740265,
      "reward_std": 0.7655115723609924,
      "rewards/rollout_reward_func/mean": 0.4079897701740265,
      "rewards/rollout_reward_func/std": 0.7435224652290344,
      "sampling/importance_sampling_ratio/max": 1.8575447797775269,
      "sampling/importance_sampling_ratio/mean": 1.0399607419967651,
      "sampling/importance_sampling_ratio/min": 0.7506864666938782,
      "sampling/sampling_logp_difference/max": 0.2605562210083008,
      "sampling/sampling_logp_difference/mean": 0.023328639566898346,
      "step": 211,
      "step_time": 7.781360815999506
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1702.0,
      "completions/max_terminated_length": 1702.0,
      "completions/mean_length": 632.25,
      "completions/mean_terminated_length": 664.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5272927284240723,
      "epoch": 0.00212,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.259375810623169,
      "kl": 0.5152924656867981,
      "learning_rate": 4.547536182848301e-05,
      "loss": -0.1355,
      "num_tokens": 4958686.0,
      "reward": 0.4295605421066284,
      "reward_std": 0.830947995185852,
      "rewards/rollout_reward_func/mean": 0.4295605421066284,
      "rewards/rollout_reward_func/std": 0.8079487085342407,
      "sampling/importance_sampling_ratio/max": 1.4796128273010254,
      "sampling/importance_sampling_ratio/mean": 0.96620774269104,
      "sampling/importance_sampling_ratio/min": 0.7583972215652466,
      "sampling/sampling_logp_difference/max": 0.24909257888793945,
      "sampling/sampling_logp_difference/mean": 0.024706436321139336,
      "step": 212,
      "step_time": 11.730272340000283
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 851.0,
      "completions/max_terminated_length": 851.0,
      "completions/mean_length": 589.625,
      "completions/mean_terminated_length": 553.5714721679688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0886486172676086,
      "epoch": 0.00213,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.334141731262207,
      "kl": 1.2604160569608212,
      "learning_rate": 4.543523921255028e-05,
      "loss": 0.3838,
      "num_tokens": 4983229.0,
      "reward": 0.47437000274658203,
      "reward_std": 0.6656765937805176,
      "rewards/rollout_reward_func/mean": 0.47437000274658203,
      "rewards/rollout_reward_func/std": 0.6485035419464111,
      "sampling/importance_sampling_ratio/max": 1.4240448474884033,
      "sampling/importance_sampling_ratio/mean": 1.0526514053344727,
      "sampling/importance_sampling_ratio/min": 0.7492220997810364,
      "sampling/sampling_logp_difference/max": 0.18714964389801025,
      "sampling/sampling_logp_difference/mean": 0.025999054312705994,
      "step": 213,
      "step_time": 8.442561626000042
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 906.0,
      "completions/max_terminated_length": 906.0,
      "completions/mean_length": 431.3125,
      "completions/mean_terminated_length": 416.6153869628906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9760564621537924,
      "epoch": 0.00214,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1954281330108643,
      "kl": 0.41587140411138535,
      "learning_rate": 4.53949108828587e-05,
      "loss": -0.0848,
      "num_tokens": 5004522.0,
      "reward": 0.5895793437957764,
      "reward_std": 0.5666608214378357,
      "rewards/rollout_reward_func/mean": 0.5895793437957764,
      "rewards/rollout_reward_func/std": 0.5475286841392517,
      "sampling/importance_sampling_ratio/max": 2.031956195831299,
      "sampling/importance_sampling_ratio/mean": 0.9960144758224487,
      "sampling/importance_sampling_ratio/min": 0.6071122884750366,
      "sampling/sampling_logp_difference/max": 0.296053409576416,
      "sampling/sampling_logp_difference/mean": 0.025808531790971756,
      "step": 214,
      "step_time": 8.446716614999787
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 931.0,
      "completions/max_terminated_length": 910.0,
      "completions/mean_length": 292.125,
      "completions/mean_terminated_length": 216.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0577773563563824,
      "epoch": 0.00215,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7676314115524292,
      "kl": 0.4695556201040745,
      "learning_rate": 4.535437732005586e-05,
      "loss": -0.0193,
      "num_tokens": 5023326.0,
      "reward": 0.4703860282897949,
      "reward_std": 0.6720806360244751,
      "rewards/rollout_reward_func/mean": 0.4703860282897949,
      "rewards/rollout_reward_func/std": 0.6558161377906799,
      "sampling/importance_sampling_ratio/max": 1.2158554792404175,
      "sampling/importance_sampling_ratio/mean": 1.0175249576568604,
      "sampling/importance_sampling_ratio/min": 0.9274688959121704,
      "sampling/sampling_logp_difference/max": 0.1921982765197754,
      "sampling/sampling_logp_difference/mean": 0.021725788712501526,
      "step": 215,
      "step_time": 8.407311658000253
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 947.0,
      "completions/max_terminated_length": 862.0,
      "completions/mean_length": 443.1875,
      "completions/mean_terminated_length": 346.5384826660156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.139294570311904,
      "epoch": 0.00216,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0520511865615845,
      "kl": 0.8662398979067802,
      "learning_rate": 4.531363900723538e-05,
      "loss": 0.0702,
      "num_tokens": 5044756.0,
      "reward": 0.5958002209663391,
      "reward_std": 0.5580307245254517,
      "rewards/rollout_reward_func/mean": 0.5958002209663391,
      "rewards/rollout_reward_func/std": 0.5394045114517212,
      "sampling/importance_sampling_ratio/max": 1.282467246055603,
      "sampling/importance_sampling_ratio/mean": 1.0156147480010986,
      "sampling/importance_sampling_ratio/min": 0.6182148456573486,
      "sampling/sampling_logp_difference/max": 0.17090678215026855,
      "sampling/sampling_logp_difference/mean": 0.02566569671034813,
      "step": 216,
      "step_time": 8.603162869999323
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1873.0,
      "completions/max_terminated_length": 1873.0,
      "completions/mean_length": 713.5,
      "completions/mean_terminated_length": 623.4285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1658797329291701,
      "epoch": 0.00217,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5471564531326294,
      "kl": 0.5695774964988232,
      "learning_rate": 4.527269642993119e-05,
      "loss": -0.2217,
      "num_tokens": 5071241.0,
      "reward": 0.27309665083885193,
      "reward_std": 0.6126882433891296,
      "rewards/rollout_reward_func/mean": 0.27309665083885193,
      "rewards/rollout_reward_func/std": 0.6245466470718384,
      "sampling/importance_sampling_ratio/max": 1.7616398334503174,
      "sampling/importance_sampling_ratio/mean": 1.0318571329116821,
      "sampling/importance_sampling_ratio/min": 0.28056129813194275,
      "sampling/sampling_logp_difference/max": 0.23431634902954102,
      "sampling/sampling_logp_difference/mean": 0.026151522994041443,
      "step": 217,
      "step_time": 12.924312091999127
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 941.0,
      "completions/max_terminated_length": 941.0,
      "completions/mean_length": 538.9375,
      "completions/mean_terminated_length": 473.0769348144531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3638052456080914,
      "epoch": 0.00218,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.526651382446289,
      "kl": 0.5376599580049515,
      "learning_rate": 4.523155007611167e-05,
      "loss": 0.2468,
      "num_tokens": 5094694.0,
      "reward": 0.5395807027816772,
      "reward_std": 0.6229984760284424,
      "rewards/rollout_reward_func/mean": 0.5395807027816772,
      "rewards/rollout_reward_func/std": 0.647162139415741,
      "sampling/importance_sampling_ratio/max": 1.1944396495819092,
      "sampling/importance_sampling_ratio/mean": 0.9221977591514587,
      "sampling/importance_sampling_ratio/min": 0.5443939566612244,
      "sampling/sampling_logp_difference/max": 0.22891902923583984,
      "sampling/sampling_logp_difference/mean": 0.02336951158940792,
      "step": 218,
      "step_time": 9.45477309900025
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 979.0,
      "completions/max_terminated_length": 979.0,
      "completions/mean_length": 493.625,
      "completions/mean_terminated_length": 465.60003662109375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.368257189169526,
      "epoch": 0.00219,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6859654784202576,
      "kl": 0.8049760535359383,
      "learning_rate": 4.519020043617392e-05,
      "loss": 0.1763,
      "num_tokens": 5117980.0,
      "reward": 0.3208381235599518,
      "reward_std": 0.4717456102371216,
      "rewards/rollout_reward_func/mean": 0.3208381235599518,
      "rewards/rollout_reward_func/std": 0.543516993522644,
      "sampling/importance_sampling_ratio/max": 1.0790318250656128,
      "sampling/importance_sampling_ratio/mean": 0.8446363210678101,
      "sampling/importance_sampling_ratio/min": 2.3144152491439947e-15,
      "sampling/sampling_logp_difference/max": 20.011640548706055,
      "sampling/sampling_logp_difference/mean": 0.20131468772888184,
      "step": 219,
      "step_time": 9.366635719000442
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 923.0,
      "completions/max_terminated_length": 906.0,
      "completions/mean_length": 620.9375,
      "completions/mean_terminated_length": 600.800048828125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1457417011260986,
      "epoch": 0.0022,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5617286562919617,
      "kl": 0.9413995742797852,
      "learning_rate": 4.514864800293788e-05,
      "loss": -0.1257,
      "num_tokens": 5143412.0,
      "reward": 0.6589969396591187,
      "reward_std": 0.48771631717681885,
      "rewards/rollout_reward_func/mean": 0.6589969396591187,
      "rewards/rollout_reward_func/std": 0.5224554538726807,
      "sampling/importance_sampling_ratio/max": 1.1834841966629028,
      "sampling/importance_sampling_ratio/mean": 0.9772457480430603,
      "sampling/importance_sampling_ratio/min": 0.7193782925605774,
      "sampling/sampling_logp_difference/max": 0.14826607704162598,
      "sampling/sampling_logp_difference/mean": 0.023722564801573753,
      "step": 220,
      "step_time": 8.600823096000113
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1012.0,
      "completions/max_terminated_length": 985.0,
      "completions/mean_length": 512.125,
      "completions/mean_terminated_length": 476.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4297135472297668,
      "epoch": 0.00221,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0620307922363281,
      "kl": 0.5995022617280483,
      "learning_rate": 4.510689327164043e-05,
      "loss": 0.1697,
      "num_tokens": 5166963.0,
      "reward": 0.2644069492816925,
      "reward_std": 0.643193244934082,
      "rewards/rollout_reward_func/mean": 0.2644069492816925,
      "rewards/rollout_reward_func/std": 0.6294947862625122,
      "sampling/importance_sampling_ratio/max": 1.8807768821716309,
      "sampling/importance_sampling_ratio/mean": 1.0269681215286255,
      "sampling/importance_sampling_ratio/min": 2.5360463070967874e-31,
      "sampling/sampling_logp_difference/max": 10.652950286865234,
      "sampling/sampling_logp_difference/mean": 0.2808268964290619,
      "step": 221,
      "step_time": 9.23234819900108
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 871.0,
      "completions/max_terminated_length": 871.0,
      "completions/mean_length": 398.0625,
      "completions/mean_terminated_length": 391.21429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2540645077824593,
      "epoch": 0.00222,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0840120315551758,
      "kl": 1.2498331256210804,
      "learning_rate": 4.5064936739929535e-05,
      "loss": 0.0489,
      "num_tokens": 5188053.0,
      "reward": 0.46559590101242065,
      "reward_std": 0.6141313910484314,
      "rewards/rollout_reward_func/mean": 0.46559590101242065,
      "rewards/rollout_reward_func/std": 0.6622478365898132,
      "sampling/importance_sampling_ratio/max": 1.2325034141540527,
      "sampling/importance_sampling_ratio/mean": 0.9719573259353638,
      "sampling/importance_sampling_ratio/min": 0.6611142158508301,
      "sampling/sampling_logp_difference/max": 0.21007537841796875,
      "sampling/sampling_logp_difference/mean": 0.028652174398303032,
      "step": 222,
      "step_time": 8.020312051000019
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1518.0,
      "completions/max_terminated_length": 1518.0,
      "completions/mean_length": 512.5,
      "completions/mean_terminated_length": 480.0666809082031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8665131870657206,
      "epoch": 0.00223,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.801103413105011,
      "kl": 0.7626403011381626,
      "learning_rate": 4.502277890785827e-05,
      "loss": 0.0731,
      "num_tokens": 5210988.0,
      "reward": 0.5987583994865417,
      "reward_std": 0.46657341718673706,
      "rewards/rollout_reward_func/mean": 0.5987583994865417,
      "rewards/rollout_reward_func/std": 0.5352535247802734,
      "sampling/importance_sampling_ratio/max": 1.127671480178833,
      "sampling/importance_sampling_ratio/mean": 0.8772291541099548,
      "sampling/importance_sampling_ratio/min": 8.862098754260368e-19,
      "sampling/sampling_logp_difference/max": 23.713029861450195,
      "sampling/sampling_logp_difference/mean": 0.3594873249530792,
      "step": 223,
      "step_time": 10.063292937999904
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 836.0,
      "completions/max_terminated_length": 817.0,
      "completions/mean_length": 332.75,
      "completions/mean_terminated_length": 299.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.913736735470593,
      "epoch": 0.00224,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.34954169392585754,
      "kl": 0.8346971645951271,
      "learning_rate": 4.49804202778789e-05,
      "loss": -0.0599,
      "num_tokens": 5230798.0,
      "reward": 0.5320512056350708,
      "reward_std": 0.637736439704895,
      "rewards/rollout_reward_func/mean": 0.5320512056350708,
      "rewards/rollout_reward_func/std": 0.6593586206436157,
      "sampling/importance_sampling_ratio/max": 1.353140115737915,
      "sampling/importance_sampling_ratio/mean": 1.0436794757843018,
      "sampling/importance_sampling_ratio/min": 0.8634694218635559,
      "sampling/sampling_logp_difference/max": 0.20123803615570068,
      "sampling/sampling_logp_difference/mean": 0.019940556958317757,
      "step": 224,
      "step_time": 7.902924487999371
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 938.0,
      "completions/max_terminated_length": 938.0,
      "completions/mean_length": 543.5625,
      "completions/mean_terminated_length": 543.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2152816876769066,
      "epoch": 0.00225,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1696171760559082,
      "kl": 0.7324325814843178,
      "learning_rate": 4.493786135483684e-05,
      "loss": 0.2626,
      "num_tokens": 5254256.0,
      "reward": 0.3902056813240051,
      "reward_std": 0.5294331312179565,
      "rewards/rollout_reward_func/mean": 0.3902056813240051,
      "rewards/rollout_reward_func/std": 0.5556839108467102,
      "sampling/importance_sampling_ratio/max": 1.9294787645339966,
      "sampling/importance_sampling_ratio/mean": 1.0921618938446045,
      "sampling/importance_sampling_ratio/min": 0.8497728109359741,
      "sampling/sampling_logp_difference/max": 0.18460988998413086,
      "sampling/sampling_logp_difference/mean": 0.02752380073070526,
      "step": 225,
      "step_time": 8.490182495998852
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 961.0,
      "completions/max_terminated_length": 961.0,
      "completions/mean_length": 398.625,
      "completions/mean_terminated_length": 398.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0897964015603065,
      "epoch": 0.00226,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8053662180900574,
      "kl": 0.6038487255573273,
      "learning_rate": 4.489510264596472e-05,
      "loss": 0.08,
      "num_tokens": 5275395.0,
      "reward": 0.46862074732780457,
      "reward_std": 0.6729058623313904,
      "rewards/rollout_reward_func/mean": 0.46862074732780457,
      "rewards/rollout_reward_func/std": 0.6555213332176208,
      "sampling/importance_sampling_ratio/max": 1.1589159965515137,
      "sampling/importance_sampling_ratio/mean": 0.9946470260620117,
      "sampling/importance_sampling_ratio/min": 0.7316780090332031,
      "sampling/sampling_logp_difference/max": 0.12359809875488281,
      "sampling/sampling_logp_difference/mean": 0.020784346386790276,
      "step": 226,
      "step_time": 7.910908963000111
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 937.0,
      "completions/max_terminated_length": 937.0,
      "completions/mean_length": 393.0,
      "completions/mean_terminated_length": 393.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9855397157371044,
      "epoch": 0.00227,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.42608848214149475,
      "kl": 0.7742233462631702,
      "learning_rate": 4.485214466087624e-05,
      "loss": -0.0457,
      "num_tokens": 5296652.0,
      "reward": 0.20501434803009033,
      "reward_std": 0.7086312770843506,
      "rewards/rollout_reward_func/mean": 0.20501434803009033,
      "rewards/rollout_reward_func/std": 0.7016813158988953,
      "sampling/importance_sampling_ratio/max": 1.129536509513855,
      "sampling/importance_sampling_ratio/mean": 0.9249289035797119,
      "sampling/importance_sampling_ratio/min": 6.950349185075311e-10,
      "sampling/sampling_logp_difference/max": 18.447025299072266,
      "sampling/sampling_logp_difference/mean": 0.13798242807388306,
      "step": 227,
      "step_time": 8.21730524800023
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1905.0,
      "completions/max_terminated_length": 1905.0,
      "completions/mean_length": 501.6875,
      "completions/mean_terminated_length": 467.0666809082031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.110623612999916,
      "epoch": 0.00228,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4825444519519806,
      "kl": 0.632828664034605,
      "learning_rate": 4.480898791156017e-05,
      "loss": -0.0617,
      "num_tokens": 5319391.0,
      "reward": 0.40322378277778625,
      "reward_std": 0.6249727606773376,
      "rewards/rollout_reward_func/mean": 0.40322378277778625,
      "rewards/rollout_reward_func/std": 0.6531172394752502,
      "sampling/importance_sampling_ratio/max": 1.0138459205627441,
      "sampling/importance_sampling_ratio/mean": 0.9528987407684326,
      "sampling/importance_sampling_ratio/min": 0.4600065052509308,
      "sampling/sampling_logp_difference/max": 0.2324671745300293,
      "sampling/sampling_logp_difference/mean": 0.026483751833438873,
      "step": 228,
      "step_time": 10.878119926000181
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 924.0,
      "completions/max_terminated_length": 924.0,
      "completions/mean_length": 585.3125,
      "completions/mean_terminated_length": 585.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.931101456284523,
      "epoch": 0.00229,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7464995384216309,
      "kl": 1.193366713821888,
      "learning_rate": 4.4765632912374226e-05,
      "loss": -0.0772,
      "num_tokens": 5344144.0,
      "reward": 0.38579362630844116,
      "reward_std": 0.5739731788635254,
      "rewards/rollout_reward_func/mean": 0.38579362630844116,
      "rewards/rollout_reward_func/std": 0.559607207775116,
      "sampling/importance_sampling_ratio/max": 1.230475902557373,
      "sampling/importance_sampling_ratio/mean": 1.0159547328948975,
      "sampling/importance_sampling_ratio/min": 0.9078204035758972,
      "sampling/sampling_logp_difference/max": 0.3598930835723877,
      "sampling/sampling_logp_difference/mean": 0.028103241696953773,
      "step": 229,
      "step_time": 7.8953943639999125
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 970.0,
      "completions/max_terminated_length": 970.0,
      "completions/mean_length": 483.1875,
      "completions/mean_terminated_length": 483.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8983368556946516,
      "epoch": 0.0023,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8319486379623413,
      "kl": 0.4659963436424732,
      "learning_rate": 4.472208018003893e-05,
      "loss": 0.2267,
      "num_tokens": 5366533.0,
      "reward": 0.5965509414672852,
      "reward_std": 0.5569446086883545,
      "rewards/rollout_reward_func/mean": 0.5965509414672852,
      "rewards/rollout_reward_func/std": 0.5381730794906616,
      "sampling/importance_sampling_ratio/max": 1.2191585302352905,
      "sampling/importance_sampling_ratio/mean": 1.043981909751892,
      "sampling/importance_sampling_ratio/min": 0.9766371846199036,
      "sampling/sampling_logp_difference/max": 0.12219858169555664,
      "sampling/sampling_logp_difference/mean": 0.019605712965130806,
      "step": 230,
      "step_time": 8.576491188000091
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1630.0,
      "completions/max_terminated_length": 834.0,
      "completions/mean_length": 398.4375,
      "completions/mean_terminated_length": 334.3571472167969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.781177690718323,
      "epoch": 0.00231,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6021113395690918,
      "kl": 0.5181132033467293,
      "learning_rate": 4.467833023363145e-05,
      "loss": -0.1509,
      "num_tokens": 5387533.0,
      "reward": 0.3963715434074402,
      "reward_std": 0.6769166588783264,
      "rewards/rollout_reward_func/mean": 0.3963715434074402,
      "rewards/rollout_reward_func/std": 0.6610711812973022,
      "sampling/importance_sampling_ratio/max": 1.1000505685806274,
      "sampling/importance_sampling_ratio/mean": 0.9439904093742371,
      "sampling/importance_sampling_ratio/min": 2.843778852075616e-26,
      "sampling/sampling_logp_difference/max": 13.649959564208984,
      "sampling/sampling_logp_difference/mean": 0.32839399576187134,
      "step": 231,
      "step_time": 10.190579914999944
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1639.0,
      "completions/max_terminated_length": 1639.0,
      "completions/mean_length": 562.9375,
      "completions/mean_terminated_length": 562.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8972563929855824,
      "epoch": 0.00232,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7003495693206787,
      "kl": 1.313760744407773,
      "learning_rate": 4.463438359457945e-05,
      "loss": 0.0504,
      "num_tokens": 5411446.0,
      "reward": 0.6093201041221619,
      "reward_std": 0.6245510578155518,
      "rewards/rollout_reward_func/mean": 0.6093201041221619,
      "rewards/rollout_reward_func/std": 0.6283040642738342,
      "sampling/importance_sampling_ratio/max": 1.1995805501937866,
      "sampling/importance_sampling_ratio/mean": 0.9119423627853394,
      "sampling/importance_sampling_ratio/min": 1.170047836396293e-09,
      "sampling/sampling_logp_difference/max": 18.00660514831543,
      "sampling/sampling_logp_difference/mean": 0.1333041936159134,
      "step": 232,
      "step_time": 10.327183393000723
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1903.0,
      "completions/max_terminated_length": 960.0,
      "completions/mean_length": 536.625,
      "completions/mean_terminated_length": 445.5333557128906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.003737648949027,
      "epoch": 0.00233,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.23107869923114777,
      "kl": 0.9193114191293716,
      "learning_rate": 4.459024078665482e-05,
      "loss": 0.0094,
      "num_tokens": 5435649.0,
      "reward": 0.2781023681163788,
      "reward_std": 0.708351731300354,
      "rewards/rollout_reward_func/mean": 0.2781023681163788,
      "rewards/rollout_reward_func/std": 0.7168239951133728,
      "sampling/importance_sampling_ratio/max": 1.103992223739624,
      "sampling/importance_sampling_ratio/mean": 0.9761530756950378,
      "sampling/importance_sampling_ratio/min": 0.7442144155502319,
      "sampling/sampling_logp_difference/max": 0.19966363906860352,
      "sampling/sampling_logp_difference/mean": 0.023586276918649673,
      "step": 233,
      "step_time": 10.603448010000193
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1839.0,
      "completions/max_terminated_length": 1839.0,
      "completions/mean_length": 541.1875,
      "completions/mean_terminated_length": 521.2000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9756260451395065,
      "epoch": 0.00234,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2524287700653076,
      "kl": 0.758141566067934,
      "learning_rate": 4.454590233596748e-05,
      "loss": -0.1177,
      "num_tokens": 5459449.0,
      "reward": 0.28354766964912415,
      "reward_std": 0.7010438442230225,
      "rewards/rollout_reward_func/mean": 0.28354766964912415,
      "rewards/rollout_reward_func/std": 0.7107442021369934,
      "sampling/importance_sampling_ratio/max": 2.4578559398651123,
      "sampling/importance_sampling_ratio/mean": 1.0394649505615234,
      "sampling/importance_sampling_ratio/min": 0.5983540415763855,
      "sampling/sampling_logp_difference/max": 0.3412814140319824,
      "sampling/sampling_logp_difference/mean": 0.027752645313739777,
      "step": 234,
      "step_time": 10.894418420000875
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 933.0,
      "completions/max_terminated_length": 933.0,
      "completions/mean_length": 385.625,
      "completions/mean_terminated_length": 385.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.834519068710506,
      "epoch": 0.00235,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19913850724697113,
      "kl": 0.7839771434664726,
      "learning_rate": 4.450136877095908e-05,
      "loss": -0.0107,
      "num_tokens": 5481056.0,
      "reward": 0.2752799093723297,
      "reward_std": 0.7420285940170288,
      "rewards/rollout_reward_func/mean": 0.2752799093723297,
      "rewards/rollout_reward_func/std": 0.7203606367111206,
      "sampling/importance_sampling_ratio/max": 1.0749231576919556,
      "sampling/importance_sampling_ratio/mean": 0.933664858341217,
      "sampling/importance_sampling_ratio/min": 1.1915181515054218e-11,
      "sampling/sampling_logp_difference/max": 18.508094787597656,
      "sampling/sampling_logp_difference/mean": 0.2985939085483551,
      "step": 235,
      "step_time": 7.616370602999723
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 933.0,
      "completions/max_terminated_length": 933.0,
      "completions/mean_length": 393.625,
      "completions/mean_terminated_length": 393.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6925576943904161,
      "epoch": 0.00236,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3080177307128906,
      "kl": 0.8103342317044735,
      "learning_rate": 4.445664062239671e-05,
      "loss": -0.0521,
      "num_tokens": 5501490.0,
      "reward": 0.7402769923210144,
      "reward_std": 0.6028134226799011,
      "rewards/rollout_reward_func/mean": 0.7402769923210144,
      "rewards/rollout_reward_func/std": 0.5909290313720703,
      "sampling/importance_sampling_ratio/max": 1.2013444900512695,
      "sampling/importance_sampling_ratio/mean": 1.0166347026824951,
      "sampling/importance_sampling_ratio/min": 0.9191626906394958,
      "sampling/sampling_logp_difference/max": 0.10291671752929688,
      "sampling/sampling_logp_difference/mean": 0.01443563587963581,
      "step": 236,
      "step_time": 8.165393935999873
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 942.0,
      "completions/max_terminated_length": 942.0,
      "completions/mean_length": 563.4375,
      "completions/mean_terminated_length": 563.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1230651019141078,
      "epoch": 0.00237,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8953040242195129,
      "kl": 0.8605775162577629,
      "learning_rate": 4.441171842336659e-05,
      "loss": 0.1175,
      "num_tokens": 5525546.0,
      "reward": 0.5305224061012268,
      "reward_std": 0.5271810293197632,
      "rewards/rollout_reward_func/mean": 0.5305224061012268,
      "rewards/rollout_reward_func/std": 0.5498719811439514,
      "sampling/importance_sampling_ratio/max": 1.2094954252243042,
      "sampling/importance_sampling_ratio/mean": 0.9195504188537598,
      "sampling/importance_sampling_ratio/min": 2.0493766406115554e-10,
      "sampling/sampling_logp_difference/max": 11.853711128234863,
      "sampling/sampling_logp_difference/mean": 0.15386644005775452,
      "step": 237,
      "step_time": 8.458173879999777
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 940.0,
      "completions/max_terminated_length": 940.0,
      "completions/mean_length": 499.25,
      "completions/mean_terminated_length": 499.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8600752931088209,
      "epoch": 0.00238,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2452870011329651,
      "kl": 0.8767250403761864,
      "learning_rate": 4.4366602709267716e-05,
      "loss": -0.0158,
      "num_tokens": 5548722.0,
      "reward": 0.4657072424888611,
      "reward_std": 0.5149057507514954,
      "rewards/rollout_reward_func/mean": 0.4657072424888611,
      "rewards/rollout_reward_func/std": 0.6597607731819153,
      "sampling/importance_sampling_ratio/max": 1.1175464391708374,
      "sampling/importance_sampling_ratio/mean": 1.0003557205200195,
      "sampling/importance_sampling_ratio/min": 0.8599601984024048,
      "sampling/sampling_logp_difference/max": 0.07350361347198486,
      "sampling/sampling_logp_difference/mean": 0.011112644337117672,
      "step": 238,
      "step_time": 7.850872266999886
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 896.0,
      "completions/max_terminated_length": 896.0,
      "completions/mean_length": 451.0625,
      "completions/mean_terminated_length": 451.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7700213957577944,
      "epoch": 0.00239,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4763701558113098,
      "kl": 0.7244811933487654,
      "learning_rate": 4.4321294017805414e-05,
      "loss": -0.0694,
      "num_tokens": 5571213.0,
      "reward": 0.45247530937194824,
      "reward_std": 0.56650710105896,
      "rewards/rollout_reward_func/mean": 0.45247530937194824,
      "rewards/rollout_reward_func/std": 0.5656501650810242,
      "sampling/importance_sampling_ratio/max": 1.0768369436264038,
      "sampling/importance_sampling_ratio/mean": 0.9904334545135498,
      "sampling/importance_sampling_ratio/min": 0.8975992798805237,
      "sampling/sampling_logp_difference/max": 0.09992647171020508,
      "sampling/sampling_logp_difference/mean": 0.0141985472291708,
      "step": 239,
      "step_time": 7.642857146001006
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1841.0,
      "completions/max_terminated_length": 1841.0,
      "completions/mean_length": 451.0,
      "completions/mean_terminated_length": 339.4285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7783236531540751,
      "epoch": 0.0024,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.017186164855957,
      "kl": 0.5773392170667648,
      "learning_rate": 4.427579288898503e-05,
      "loss": 0.094,
      "num_tokens": 5592565.0,
      "reward": 0.483623743057251,
      "reward_std": 0.7656315565109253,
      "rewards/rollout_reward_func/mean": 0.483623743057251,
      "rewards/rollout_reward_func/std": 0.7396750450134277,
      "sampling/importance_sampling_ratio/max": 1.0521489381790161,
      "sampling/importance_sampling_ratio/mean": 0.9816095232963562,
      "sampling/importance_sampling_ratio/min": 0.7399073839187622,
      "sampling/sampling_logp_difference/max": 0.1472322940826416,
      "sampling/sampling_logp_difference/mean": 0.022119387984275818,
      "step": 240,
      "step_time": 10.763385329000357
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 966.0,
      "completions/max_terminated_length": 862.0,
      "completions/mean_length": 419.5625,
      "completions/mean_terminated_length": 383.13336181640625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8476203144527972,
      "epoch": 0.00241,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9034574031829834,
      "kl": 0.49220225401222706,
      "learning_rate": 4.4230099865105445e-05,
      "loss": 0.1479,
      "num_tokens": 5614120.0,
      "reward": 0.281948447227478,
      "reward_std": 0.7007027864456177,
      "rewards/rollout_reward_func/mean": 0.281948447227478,
      "rewards/rollout_reward_func/std": 0.708615779876709,
      "sampling/importance_sampling_ratio/max": 1.1344877481460571,
      "sampling/importance_sampling_ratio/mean": 0.9954975247383118,
      "sampling/importance_sampling_ratio/min": 0.8767620921134949,
      "sampling/sampling_logp_difference/max": 0.10851573944091797,
      "sampling/sampling_logp_difference/mean": 0.01692826859652996,
      "step": 241,
      "step_time": 7.958531578000475
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1794.0,
      "completions/max_terminated_length": 1540.0,
      "completions/mean_length": 528.5625,
      "completions/mean_terminated_length": 461.3077087402344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3272969424724579,
      "epoch": 0.00242,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7926445603370667,
      "kl": 0.8193247318267822,
      "learning_rate": 4.418421549075259e-05,
      "loss": -0.2183,
      "num_tokens": 5637705.0,
      "reward": 0.36389774084091187,
      "reward_std": 0.8244630694389343,
      "rewards/rollout_reward_func/mean": 0.36389774084091187,
      "rewards/rollout_reward_func/std": 0.8073947429656982,
      "sampling/importance_sampling_ratio/max": 1.211629033088684,
      "sampling/importance_sampling_ratio/mean": 0.9344499111175537,
      "sampling/importance_sampling_ratio/min": 4.873431046024458e-36,
      "sampling/sampling_logp_difference/max": 18.679304122924805,
      "sampling/sampling_logp_difference/mean": 0.31549131870269775,
      "step": 242,
      "step_time": 11.112403756000276
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1827.0,
      "completions/max_terminated_length": 1827.0,
      "completions/mean_length": 787.6875,
      "completions/mean_terminated_length": 835.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1141440942883492,
      "epoch": 0.00243,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7769773006439209,
      "kl": 1.011199939996004,
      "learning_rate": 4.4138140312793004e-05,
      "loss": -0.0944,
      "num_tokens": 5665327.0,
      "reward": 0.7291566133499146,
      "reward_std": 0.5015420913696289,
      "rewards/rollout_reward_func/mean": 0.7291566133499146,
      "rewards/rollout_reward_func/std": 0.484738290309906,
      "sampling/importance_sampling_ratio/max": 1.195186972618103,
      "sampling/importance_sampling_ratio/mean": 0.9119952917098999,
      "sampling/importance_sampling_ratio/min": 7.994041101053812e-24,
      "sampling/sampling_logp_difference/max": 18.69123649597168,
      "sampling/sampling_logp_difference/mean": 0.29035574197769165,
      "step": 243,
      "step_time": 11.809196309999606
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 835.0,
      "completions/max_terminated_length": 835.0,
      "completions/mean_length": 301.125,
      "completions/mean_terminated_length": 301.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5975356036797166,
      "epoch": 0.00244,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.31140342354774475,
      "kl": 0.7715253531932831,
      "learning_rate": 4.409187488036728e-05,
      "loss": -0.037,
      "num_tokens": 5684154.0,
      "reward": 0.7924801111221313,
      "reward_std": 0.45215606689453125,
      "rewards/rollout_reward_func/mean": 0.7924801111221313,
      "rewards/rollout_reward_func/std": 0.4461672604084015,
      "sampling/importance_sampling_ratio/max": 1.1244202852249146,
      "sampling/importance_sampling_ratio/mean": 1.0188515186309814,
      "sampling/importance_sampling_ratio/min": 0.9443312287330627,
      "sampling/sampling_logp_difference/max": 0.12273883819580078,
      "sampling/sampling_logp_difference/mean": 0.008947338908910751,
      "step": 244,
      "step_time": 7.0351158980001856
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 859.0,
      "completions/max_terminated_length": 859.0,
      "completions/mean_length": 374.875,
      "completions/mean_terminated_length": 374.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3502087146043777,
      "epoch": 0.00245,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9480643272399902,
      "kl": 1.1149943871423602,
      "learning_rate": 4.404541974488354e-05,
      "loss": 0.0553,
      "num_tokens": 5705599.0,
      "reward": 0.23689892888069153,
      "reward_std": 0.8858444690704346,
      "rewards/rollout_reward_func/mean": 0.23689892888069153,
      "rewards/rollout_reward_func/std": 0.8558156490325928,
      "sampling/importance_sampling_ratio/max": 1.1335853338241577,
      "sampling/importance_sampling_ratio/mean": 0.8930599093437195,
      "sampling/importance_sampling_ratio/min": 7.546894815190855e-17,
      "sampling/sampling_logp_difference/max": 18.573646545410156,
      "sampling/sampling_logp_difference/mean": 0.44605833292007446,
      "step": 245,
      "step_time": 7.9555497349997495
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1860.0,
      "completions/max_terminated_length": 1797.0,
      "completions/mean_length": 746.125,
      "completions/mean_terminated_length": 671.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2873683497309685,
      "epoch": 0.00246,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8359131813049316,
      "kl": 0.8617205321788788,
      "learning_rate": 4.399877546001084e-05,
      "loss": 0.351,
      "num_tokens": 5732176.0,
      "reward": 0.4803197681903839,
      "reward_std": 0.6004764437675476,
      "rewards/rollout_reward_func/mean": 0.4803197681903839,
      "rewards/rollout_reward_func/std": 0.6486254334449768,
      "sampling/importance_sampling_ratio/max": 1.4925283193588257,
      "sampling/importance_sampling_ratio/mean": 0.9707176685333252,
      "sampling/importance_sampling_ratio/min": 0.7682955265045166,
      "sampling/sampling_logp_difference/max": 0.4158172607421875,
      "sampling/sampling_logp_difference/mean": 0.02549522928893566,
      "step": 246,
      "step_time": 11.50296246400103
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1639.0,
      "completions/max_terminated_length": 1639.0,
      "completions/mean_length": 523.625,
      "completions/mean_terminated_length": 523.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7926940247416496,
      "epoch": 0.00247,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9040848016738892,
      "kl": 2.4324029944837093,
      "learning_rate": 4.395194258167262e-05,
      "loss": -0.0005,
      "num_tokens": 5754900.0,
      "reward": 0.5468004941940308,
      "reward_std": 0.6404365301132202,
      "rewards/rollout_reward_func/mean": 0.5468004941940308,
      "rewards/rollout_reward_func/std": 0.7410563230514526,
      "sampling/importance_sampling_ratio/max": 1.2185994386672974,
      "sampling/importance_sampling_ratio/mean": 1.0604960918426514,
      "sampling/importance_sampling_ratio/min": 0.9461562633514404,
      "sampling/sampling_logp_difference/max": 0.19933271408081055,
      "sampling/sampling_logp_difference/mean": 0.018742281943559647,
      "step": 247,
      "step_time": 9.550182773999495
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1594.0,
      "completions/max_terminated_length": 1594.0,
      "completions/mean_length": 669.8125,
      "completions/mean_terminated_length": 712.3333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1281968355178833,
      "epoch": 0.00248,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8956437110900879,
      "kl": 1.1555654928088188,
      "learning_rate": 4.390492166803999e-05,
      "loss": 0.0784,
      "num_tokens": 5780880.0,
      "reward": 0.5353405475616455,
      "reward_std": 0.5583289861679077,
      "rewards/rollout_reward_func/mean": 0.5353405475616455,
      "rewards/rollout_reward_func/std": 0.5445261597633362,
      "sampling/importance_sampling_ratio/max": 1.1906750202178955,
      "sampling/importance_sampling_ratio/mean": 0.9256417751312256,
      "sampling/importance_sampling_ratio/min": 4.004564979465194e-17,
      "sampling/sampling_logp_difference/max": 19.08795928955078,
      "sampling/sampling_logp_difference/mean": 0.21916836500167847,
      "step": 248,
      "step_time": 10.43147640800089
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 902.0,
      "completions/max_terminated_length": 902.0,
      "completions/mean_length": 577.125,
      "completions/mean_terminated_length": 577.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.124546155333519,
      "epoch": 0.00249,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8596081137657166,
      "kl": 0.842559453099966,
      "learning_rate": 4.385771327952519e-05,
      "loss": 0.0105,
      "num_tokens": 5805055.0,
      "reward": 0.47832703590393066,
      "reward_std": 0.640225350856781,
      "rewards/rollout_reward_func/mean": 0.47832703590393066,
      "rewards/rollout_reward_func/std": 0.6425536870956421,
      "sampling/importance_sampling_ratio/max": 1.2900115251541138,
      "sampling/importance_sampling_ratio/mean": 0.9335260987281799,
      "sampling/importance_sampling_ratio/min": 4.876488204941521e-18,
      "sampling/sampling_logp_difference/max": 18.584585189819336,
      "sampling/sampling_logp_difference/mean": 0.2700645327568054,
      "step": 249,
      "step_time": 8.857141459000104
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1799.0,
      "completions/max_terminated_length": 1799.0,
      "completions/mean_length": 741.8125,
      "completions/mean_terminated_length": 741.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1187607944011688,
      "epoch": 0.0025,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8761639595031738,
      "kl": 0.9474686682224274,
      "learning_rate": 4.3810317978774824e-05,
      "loss": -0.069,
      "num_tokens": 5832388.0,
      "reward": 0.43525514006614685,
      "reward_std": 0.8205856084823608,
      "rewards/rollout_reward_func/mean": 0.43525514006614685,
      "rewards/rollout_reward_func/std": 0.8012270331382751,
      "sampling/importance_sampling_ratio/max": 1.24571692943573,
      "sampling/importance_sampling_ratio/mean": 0.9892392158508301,
      "sampling/importance_sampling_ratio/min": 0.6029766798019409,
      "sampling/sampling_logp_difference/max": 0.22023630142211914,
      "sampling/sampling_logp_difference/mean": 0.022171856835484505,
      "step": 250,
      "step_time": 10.611450620000141
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1639.0,
      "completions/max_terminated_length": 1639.0,
      "completions/mean_length": 509.75,
      "completions/mean_terminated_length": 452.21429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.70200414955616,
      "epoch": 0.00251,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9837809205055237,
      "kl": 1.4609587527811527,
      "learning_rate": 4.376273633066317e-05,
      "loss": 0.0738,
      "num_tokens": 5854793.0,
      "reward": 0.376473605632782,
      "reward_std": 0.9054715633392334,
      "rewards/rollout_reward_func/mean": 0.376473605632782,
      "rewards/rollout_reward_func/std": 0.8748100399971008,
      "sampling/importance_sampling_ratio/max": 1.0690057277679443,
      "sampling/importance_sampling_ratio/mean": 0.9719717502593994,
      "sampling/importance_sampling_ratio/min": 0.6051976084709167,
      "sampling/sampling_logp_difference/max": 0.17638850212097168,
      "sampling/sampling_logp_difference/mean": 0.02877272292971611,
      "step": 251,
      "step_time": 10.347122590998424
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1721.0,
      "completions/max_terminated_length": 1721.0,
      "completions/mean_length": 769.875,
      "completions/mean_terminated_length": 703.7857666015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1339565366506577,
      "epoch": 0.00252,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1899490356445312,
      "kl": 0.6175190573558211,
      "learning_rate": 4.3714968902285506e-05,
      "loss": 0.0993,
      "num_tokens": 5882828.0,
      "reward": 0.2953169345855713,
      "reward_std": 0.7345230579376221,
      "rewards/rollout_reward_func/mean": 0.2953169345855713,
      "rewards/rollout_reward_func/std": 0.7997487187385559,
      "sampling/importance_sampling_ratio/max": 1.3931310176849365,
      "sampling/importance_sampling_ratio/mean": 0.9651715755462646,
      "sampling/importance_sampling_ratio/min": 0.36129751801490784,
      "sampling/sampling_logp_difference/max": 0.35842132568359375,
      "sampling/sampling_logp_difference/mean": 0.0257875919342041,
      "step": 252,
      "step_time": 11.118446080000012
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1630.0,
      "completions/max_terminated_length": 1630.0,
      "completions/mean_length": 625.375,
      "completions/mean_terminated_length": 625.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.193921834230423,
      "epoch": 0.00253,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0619233846664429,
      "kl": 0.7487713731825352,
      "learning_rate": 4.366701626295126e-05,
      "loss": 0.1929,
      "num_tokens": 5907100.0,
      "reward": 0.6809347867965698,
      "reward_std": 0.6214317679405212,
      "rewards/rollout_reward_func/mean": 0.6809347867965698,
      "rewards/rollout_reward_func/std": 0.6102222204208374,
      "sampling/importance_sampling_ratio/max": 1.145721673965454,
      "sampling/importance_sampling_ratio/mean": 1.0074217319488525,
      "sampling/importance_sampling_ratio/min": 0.7249683141708374,
      "sampling/sampling_logp_difference/max": 0.2864511013031006,
      "sampling/sampling_logp_difference/mean": 0.027552668005228043,
      "step": 253,
      "step_time": 10.243445440999949
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1631.0,
      "completions/max_terminated_length": 1631.0,
      "completions/mean_length": 611.3125,
      "completions/mean_terminated_length": 597.1333618164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0940792709589005,
      "epoch": 0.00254,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5412718653678894,
      "kl": 1.1381728649139404,
      "learning_rate": 4.3618878984177295e-05,
      "loss": -0.0816,
      "num_tokens": 5931669.0,
      "reward": 0.4841098189353943,
      "reward_std": 0.7678534388542175,
      "rewards/rollout_reward_func/mean": 0.4841098189353943,
      "rewards/rollout_reward_func/std": 0.7418191432952881,
      "sampling/importance_sampling_ratio/max": 1.0737940073013306,
      "sampling/importance_sampling_ratio/mean": 0.9647127389907837,
      "sampling/importance_sampling_ratio/min": 0.8136951923370361,
      "sampling/sampling_logp_difference/max": 0.1712970733642578,
      "sampling/sampling_logp_difference/mean": 0.019129259511828423,
      "step": 254,
      "step_time": 10.080878009000116
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1745.0,
      "completions/max_terminated_length": 1745.0,
      "completions/mean_length": 533.5625,
      "completions/mean_terminated_length": 533.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.026877336204052,
      "epoch": 0.00255,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.37893056869506836,
      "kl": 1.1426907405257225,
      "learning_rate": 4.3570557639681066e-05,
      "loss": -0.0117,
      "num_tokens": 5954679.0,
      "reward": 0.37099993228912354,
      "reward_std": 0.9022575616836548,
      "rewards/rollout_reward_func/mean": 0.37099993228912354,
      "rewards/rollout_reward_func/std": 0.882391631603241,
      "sampling/importance_sampling_ratio/max": 1.0869694948196411,
      "sampling/importance_sampling_ratio/mean": 0.9532216787338257,
      "sampling/importance_sampling_ratio/min": 0.8200361728668213,
      "sampling/sampling_logp_difference/max": 0.13117218017578125,
      "sampling/sampling_logp_difference/mean": 0.020381305366754532,
      "step": 255,
      "step_time": 9.824980517999848
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1752.0,
      "completions/max_terminated_length": 1752.0,
      "completions/mean_length": 721.8125,
      "completions/mean_terminated_length": 719.5333862304688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.856919452548027,
      "epoch": 0.00256,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.41818922758102417,
      "kl": 0.8664454221725464,
      "learning_rate": 4.3522052805373803e-05,
      "loss": 0.0114,
      "num_tokens": 5981255.0,
      "reward": 0.47432249784469604,
      "reward_std": 0.6699216961860657,
      "rewards/rollout_reward_func/mean": 0.47432249784469604,
      "rewards/rollout_reward_func/std": 0.6541674137115479,
      "sampling/importance_sampling_ratio/max": 1.1098538637161255,
      "sampling/importance_sampling_ratio/mean": 0.9272921681404114,
      "sampling/importance_sampling_ratio/min": 1.083010587896917e-33,
      "sampling/sampling_logp_difference/max": 21.27334976196289,
      "sampling/sampling_logp_difference/mean": 0.5165280103683472,
      "step": 256,
      "step_time": 11.153525717999855
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2636.0,
      "completions/max_terminated_length": 2636.0,
      "completions/mean_length": 867.6875,
      "completions/mean_terminated_length": 867.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9302045255899429,
      "epoch": 0.00257,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.38214847445487976,
      "kl": 0.9230584558099508,
      "learning_rate": 4.347336505935362e-05,
      "loss": -0.0215,
      "num_tokens": 6010048.0,
      "reward": 0.5568475127220154,
      "reward_std": 0.7533355951309204,
      "rewards/rollout_reward_func/mean": 0.5568475127220154,
      "rewards/rollout_reward_func/std": 0.7317767143249512,
      "sampling/importance_sampling_ratio/max": 1.148937463760376,
      "sampling/importance_sampling_ratio/mean": 1.021296501159668,
      "sampling/importance_sampling_ratio/min": 0.8769312500953674,
      "sampling/sampling_logp_difference/max": 0.1563129425048828,
      "sampling/sampling_logp_difference/mean": 0.02176932990550995,
      "step": 257,
      "step_time": 12.596304661999966
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2501.0,
      "completions/max_terminated_length": 2501.0,
      "completions/mean_length": 771.375,
      "completions/mean_terminated_length": 771.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1515762507915497,
      "epoch": 0.00258,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.84161376953125,
      "kl": 1.2400906309485435,
      "learning_rate": 4.3424494981898625e-05,
      "loss": 0.0711,
      "num_tokens": 6037305.0,
      "reward": 0.45087483525276184,
      "reward_std": 0.892526388168335,
      "rewards/rollout_reward_func/mean": 0.45087483525276184,
      "rewards/rollout_reward_func/std": 0.8645705580711365,
      "sampling/importance_sampling_ratio/max": 1.4599974155426025,
      "sampling/importance_sampling_ratio/mean": 1.028012752532959,
      "sampling/importance_sampling_ratio/min": 0.9082810878753662,
      "sampling/sampling_logp_difference/max": 0.2443981170654297,
      "sampling/sampling_logp_difference/mean": 0.02391788549721241,
      "step": 258,
      "step_time": 12.401279674999842
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1746.0,
      "completions/max_terminated_length": 1746.0,
      "completions/mean_length": 643.6875,
      "completions/mean_terminated_length": 643.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8483598157763481,
      "epoch": 0.00259,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5162983536720276,
      "kl": 0.8489071279764175,
      "learning_rate": 4.337544315546005e-05,
      "loss": 0.0178,
      "num_tokens": 6062131.0,
      "reward": 0.7375189661979675,
      "reward_std": 0.4571816921234131,
      "rewards/rollout_reward_func/mean": 0.7375189661979675,
      "rewards/rollout_reward_func/std": 0.4695627689361572,
      "sampling/importance_sampling_ratio/max": 1.205651879310608,
      "sampling/importance_sampling_ratio/mean": 0.9618090391159058,
      "sampling/importance_sampling_ratio/min": 6.89448220736466e-11,
      "sampling/sampling_logp_difference/max": 14.276086807250977,
      "sampling/sampling_logp_difference/mean": 0.284139484167099,
      "step": 259,
      "step_time": 10.101909052999872
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2790.0,
      "completions/max_terminated_length": 2790.0,
      "completions/mean_length": 1061.625,
      "completions/mean_terminated_length": 1075.666748046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9168810620903969,
      "epoch": 0.0026,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1383447647094727,
      "kl": 0.6789077911525965,
      "learning_rate": 4.332621016465526e-05,
      "loss": 0.1215,
      "num_tokens": 6094758.0,
      "reward": 0.49679505825042725,
      "reward_std": 0.7292205095291138,
      "rewards/rollout_reward_func/mean": 0.49679505825042725,
      "rewards/rollout_reward_func/std": 0.7176946401596069,
      "sampling/importance_sampling_ratio/max": 1.295798420906067,
      "sampling/importance_sampling_ratio/mean": 1.005345344543457,
      "sampling/importance_sampling_ratio/min": 0.7086573839187622,
      "sampling/sampling_logp_difference/max": 0.26503753662109375,
      "sampling/sampling_logp_difference/mean": 0.02309502102434635,
      "step": 260,
      "step_time": 14.311220839999805
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 871.0,
      "completions/max_terminated_length": 871.0,
      "completions/mean_length": 393.0,
      "completions/mean_terminated_length": 393.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9962976798415184,
      "epoch": 0.00261,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.22136549651622772,
      "kl": 1.1970823239535093,
      "learning_rate": 4.327679659626081e-05,
      "loss": -0.0089,
      "num_tokens": 6115236.0,
      "reward": 0.9356029033660889,
      "reward_std": 0.18214240670204163,
      "rewards/rollout_reward_func/mean": 0.9356029033660889,
      "rewards/rollout_reward_func/std": 0.25758829712867737,
      "sampling/importance_sampling_ratio/max": 1.1158819198608398,
      "sampling/importance_sampling_ratio/mean": 0.991309404373169,
      "sampling/importance_sampling_ratio/min": 0.8363450169563293,
      "sampling/sampling_logp_difference/max": 0.10999774932861328,
      "sampling/sampling_logp_difference/mean": 0.019071895629167557,
      "step": 261,
      "step_time": 7.264111895000042
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1719.0,
      "completions/max_terminated_length": 1719.0,
      "completions/mean_length": 592.4375,
      "completions/mean_terminated_length": 592.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8955540060997009,
      "epoch": 0.00262,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3172333836555481,
      "kl": 1.2400476317852736,
      "learning_rate": 4.322720303920542e-05,
      "loss": -0.0105,
      "num_tokens": 6139296.0,
      "reward": 0.43175411224365234,
      "reward_std": 0.8106938600540161,
      "rewards/rollout_reward_func/mean": 0.43175411224365234,
      "rewards/rollout_reward_func/std": 0.8117294907569885,
      "sampling/importance_sampling_ratio/max": 1.1566948890686035,
      "sampling/importance_sampling_ratio/mean": 1.0205154418945312,
      "sampling/importance_sampling_ratio/min": 0.9186461567878723,
      "sampling/sampling_logp_difference/max": 0.12795031070709229,
      "sampling/sampling_logp_difference/mean": 0.014334880746901035,
      "step": 262,
      "step_time": 9.69695111100009
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1791.0,
      "completions/max_terminated_length": 1791.0,
      "completions/mean_length": 609.125,
      "completions/mean_terminated_length": 609.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9167831763625145,
      "epoch": 0.00263,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3431251645088196,
      "kl": 1.1303085070103407,
      "learning_rate": 4.317743008456301e-05,
      "loss": 0.0246,
      "num_tokens": 6163110.0,
      "reward": 0.6668534874916077,
      "reward_std": 0.5218194723129272,
      "rewards/rollout_reward_func/mean": 0.6668534874916077,
      "rewards/rollout_reward_func/std": 0.510585606098175,
      "sampling/importance_sampling_ratio/max": 1.064048409461975,
      "sampling/importance_sampling_ratio/mean": 0.9747271537780762,
      "sampling/importance_sampling_ratio/min": 0.8565858006477356,
      "sampling/sampling_logp_difference/max": 0.09105372428894043,
      "sampling/sampling_logp_difference/mean": 0.015161939896643162,
      "step": 263,
      "step_time": 10.407778881999548
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1736.0,
      "completions/max_terminated_length": 1736.0,
      "completions/mean_length": 707.8125,
      "completions/mean_terminated_length": 707.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8945140615105629,
      "epoch": 0.00264,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4415166676044464,
      "kl": 0.7037576138973236,
      "learning_rate": 4.312747832554562e-05,
      "loss": 0.0091,
      "num_tokens": 6189623.0,
      "reward": 0.5456765294075012,
      "reward_std": 0.6565244197845459,
      "rewards/rollout_reward_func/mean": 0.5456765294075012,
      "rewards/rollout_reward_func/std": 0.6449940800666809,
      "sampling/importance_sampling_ratio/max": 1.2679638862609863,
      "sampling/importance_sampling_ratio/mean": 0.997174859046936,
      "sampling/importance_sampling_ratio/min": 0.8935409188270569,
      "sampling/sampling_logp_difference/max": 0.18553638458251953,
      "sampling/sampling_logp_difference/mean": 0.020262332633137703,
      "step": 264,
      "step_time": 10.131026253000982
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1725.0,
      "completions/max_terminated_length": 1725.0,
      "completions/mean_length": 801.5,
      "completions/mean_terminated_length": 801.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1792384684085846,
      "epoch": 0.00265,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7790778875350952,
      "kl": 1.1865205019712448,
      "learning_rate": 4.307734835749631e-05,
      "loss": -0.0351,
      "num_tokens": 6217700.0,
      "reward": 0.4444606900215149,
      "reward_std": 0.8790686130523682,
      "rewards/rollout_reward_func/mean": 0.4444606900215149,
      "rewards/rollout_reward_func/std": 0.877281665802002,
      "sampling/importance_sampling_ratio/max": 1.0910106897354126,
      "sampling/importance_sampling_ratio/mean": 1.0109610557556152,
      "sampling/importance_sampling_ratio/min": 0.8844749331474304,
      "sampling/sampling_logp_difference/max": 0.11898517608642578,
      "sampling/sampling_logp_difference/mean": 0.016416290774941444,
      "step": 265,
      "step_time": 10.264415396000913
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1698.0,
      "completions/max_terminated_length": 1698.0,
      "completions/mean_length": 474.25,
      "completions/mean_terminated_length": 474.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3764661401510239,
      "epoch": 0.00266,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4570809602737427,
      "kl": 0.4826454548165202,
      "learning_rate": 4.3027040777882156e-05,
      "loss": -0.0211,
      "num_tokens": 6239590.0,
      "reward": 0.18609410524368286,
      "reward_std": 0.9927402138710022,
      "rewards/rollout_reward_func/mean": 0.18609410524368286,
      "rewards/rollout_reward_func/std": 0.9804118871688843,
      "sampling/importance_sampling_ratio/max": 1.1076358556747437,
      "sampling/importance_sampling_ratio/mean": 0.9880493879318237,
      "sampling/importance_sampling_ratio/min": 0.8306839466094971,
      "sampling/sampling_logp_difference/max": 0.18552446365356445,
      "sampling/sampling_logp_difference/mean": 0.019810939207673073,
      "step": 266,
      "step_time": 9.451324613001361
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1620.0,
      "completions/max_terminated_length": 1620.0,
      "completions/mean_length": 533.125,
      "completions/mean_terminated_length": 533.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4234584718942642,
      "epoch": 0.00267,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.475644052028656,
      "kl": 0.8858182318508625,
      "learning_rate": 4.297655618628703e-05,
      "loss": 0.0349,
      "num_tokens": 6262256.0,
      "reward": 0.17790496349334717,
      "reward_std": 0.9165230989456177,
      "rewards/rollout_reward_func/mean": 0.17790496349334717,
      "rewards/rollout_reward_func/std": 0.906785786151886,
      "sampling/importance_sampling_ratio/max": 1.1908619403839111,
      "sampling/importance_sampling_ratio/mean": 0.9969258308410645,
      "sampling/importance_sampling_ratio/min": 0.8317633867263794,
      "sampling/sampling_logp_difference/max": 0.20115447044372559,
      "sampling/sampling_logp_difference/mean": 0.023691074922680855,
      "step": 267,
      "step_time": 9.494857738000064
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1917.0,
      "completions/max_terminated_length": 1917.0,
      "completions/mean_length": 602.875,
      "completions/mean_terminated_length": 597.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5196491181850433,
      "epoch": 0.00268,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8829346299171448,
      "kl": 3.7285299692302942,
      "learning_rate": 4.292589518440453e-05,
      "loss": -0.0958,
      "num_tokens": 6287024.0,
      "reward": 0.24737665057182312,
      "reward_std": 0.92098468542099,
      "rewards/rollout_reward_func/mean": 0.24737665057182312,
      "rewards/rollout_reward_func/std": 0.9287647008895874,
      "sampling/importance_sampling_ratio/max": 1.1089532375335693,
      "sampling/importance_sampling_ratio/mean": 0.9419678449630737,
      "sampling/importance_sampling_ratio/min": 0.5938289165496826,
      "sampling/sampling_logp_difference/max": 0.35628247261047363,
      "sampling/sampling_logp_difference/mean": 0.02830437570810318,
      "step": 268,
      "step_time": 11.29907045300024
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1790.0,
      "completions/max_terminated_length": 1790.0,
      "completions/mean_length": 455.1875,
      "completions/mean_terminated_length": 455.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6339410841464996,
      "epoch": 0.00269,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.731733500957489,
      "kl": 0.5654373485594988,
      "learning_rate": 4.2875058376030726e-05,
      "loss": 0.0144,
      "num_tokens": 6308335.0,
      "reward": -0.3082740306854248,
      "reward_std": 0.9004747867584229,
      "rewards/rollout_reward_func/mean": -0.3082740306854248,
      "rewards/rollout_reward_func/std": 0.9408336877822876,
      "sampling/importance_sampling_ratio/max": 1.0958460569381714,
      "sampling/importance_sampling_ratio/mean": 0.9280480146408081,
      "sampling/importance_sampling_ratio/min": 6.741621936053486e-26,
      "sampling/sampling_logp_difference/max": 22.979904174804688,
      "sampling/sampling_logp_difference/mean": 0.5108173489570618,
      "step": 269,
      "step_time": 10.56883871800028
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1681.0,
      "completions/max_terminated_length": 1681.0,
      "completions/mean_length": 860.25,
      "completions/mean_terminated_length": 860.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.567468449473381,
      "epoch": 0.0027,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.537196159362793,
      "kl": 0.2658327044919133,
      "learning_rate": 4.282404636705706e-05,
      "loss": -0.0277,
      "num_tokens": 6337982.0,
      "reward": 0.12622223794460297,
      "reward_std": 0.9701927304267883,
      "rewards/rollout_reward_func/mean": 0.12622223794460297,
      "rewards/rollout_reward_func/std": 0.9482596516609192,
      "sampling/importance_sampling_ratio/max": 1.068157434463501,
      "sampling/importance_sampling_ratio/mean": 1.0045019388198853,
      "sampling/importance_sampling_ratio/min": 0.9238503575325012,
      "sampling/sampling_logp_difference/max": 0.16462326049804688,
      "sampling/sampling_logp_difference/mean": 0.016912516206502914,
      "step": 270,
      "step_time": 10.195179439999265
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1735.0,
      "completions/max_terminated_length": 1735.0,
      "completions/mean_length": 677.1875,
      "completions/mean_terminated_length": 677.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6736897230148315,
      "epoch": 0.00271,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6313352584838867,
      "kl": 0.75811374373734,
      "learning_rate": 4.277285976546306e-05,
      "loss": -0.0953,
      "num_tokens": 6363881.0,
      "reward": -0.23702675104141235,
      "reward_std": 1.0245860815048218,
      "rewards/rollout_reward_func/mean": -0.23702675104141235,
      "rewards/rollout_reward_func/std": 0.9898552298545837,
      "sampling/importance_sampling_ratio/max": 1.2117241621017456,
      "sampling/importance_sampling_ratio/mean": 1.0199806690216064,
      "sampling/importance_sampling_ratio/min": 0.6992930173873901,
      "sampling/sampling_logp_difference/max": 0.2772336006164551,
      "sampling/sampling_logp_difference/mean": 0.03194426745176315,
      "step": 271,
      "step_time": 10.172978740999042
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1764.0,
      "completions/max_terminated_length": 1764.0,
      "completions/mean_length": 628.3125,
      "completions/mean_terminated_length": 628.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.004609063267708,
      "epoch": 0.00272,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7571040391921997,
      "kl": 0.5623381708282977,
      "learning_rate": 4.272149918130911e-05,
      "loss": 0.0206,
      "num_tokens": 6389218.0,
      "reward": -0.11778096109628677,
      "reward_std": 1.0451838970184326,
      "rewards/rollout_reward_func/mean": -0.11778096109628677,
      "rewards/rollout_reward_func/std": 1.0182597637176514,
      "sampling/importance_sampling_ratio/max": 1.0845599174499512,
      "sampling/importance_sampling_ratio/mean": 0.9219378232955933,
      "sampling/importance_sampling_ratio/min": 6.967706966953813e-11,
      "sampling/sampling_logp_difference/max": 14.616127967834473,
      "sampling/sampling_logp_difference/mean": 0.22767707705497742,
      "step": 272,
      "step_time": 10.340028228999017
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2590.0,
      "completions/max_terminated_length": 2590.0,
      "completions/mean_length": 449.0,
      "completions/mean_terminated_length": 478.5384826660156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8260973691940308,
      "epoch": 0.00273,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.772098183631897,
      "kl": 1.0525305173359811,
      "learning_rate": 4.266996522672918e-05,
      "loss": -0.0545,
      "num_tokens": 6410704.0,
      "reward": 0.31941354274749756,
      "reward_std": 0.9625244140625,
      "rewards/rollout_reward_func/mean": 0.31941354274749756,
      "rewards/rollout_reward_func/std": 0.9327266812324524,
      "sampling/importance_sampling_ratio/max": 1.5007805824279785,
      "sampling/importance_sampling_ratio/mean": 0.8939164876937866,
      "sampling/importance_sampling_ratio/min": 1.4615648771171078e-25,
      "sampling/sampling_logp_difference/max": 14.02663516998291,
      "sampling/sampling_logp_difference/mean": 0.30914267897605896,
      "step": 273,
      "step_time": 12.817627200999596
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1758.0,
      "completions/max_terminated_length": 1758.0,
      "completions/mean_length": 689.1875,
      "completions/mean_terminated_length": 689.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8067148923873901,
      "epoch": 0.00274,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6380752325057983,
      "kl": 0.5224787127226591,
      "learning_rate": 4.261825851592355e-05,
      "loss": 0.1253,
      "num_tokens": 6436218.0,
      "reward": 0.31405600905418396,
      "reward_std": 0.8549341559410095,
      "rewards/rollout_reward_func/mean": 0.31405600905418396,
      "rewards/rollout_reward_func/std": 0.8563924431800842,
      "sampling/importance_sampling_ratio/max": 1.1462148427963257,
      "sampling/importance_sampling_ratio/mean": 0.9336868524551392,
      "sampling/importance_sampling_ratio/min": 2.968650897988126e-16,
      "sampling/sampling_logp_difference/max": 14.509407043457031,
      "sampling/sampling_logp_difference/mean": 0.29347026348114014,
      "step": 274,
      "step_time": 10.600788508999358
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1639.0,
      "completions/max_terminated_length": 1639.0,
      "completions/mean_length": 787.3125,
      "completions/mean_terminated_length": 704.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5206098854541779,
      "epoch": 0.00275,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.031853437423706,
      "kl": 0.5994912621099502,
      "learning_rate": 4.2566379665151455e-05,
      "loss": 0.1579,
      "num_tokens": 6464000.0,
      "reward": 0.4483470618724823,
      "reward_std": 0.9000924825668335,
      "rewards/rollout_reward_func/mean": 0.4483470618724823,
      "rewards/rollout_reward_func/std": 0.8718446493148804,
      "sampling/importance_sampling_ratio/max": 1.5985711812973022,
      "sampling/importance_sampling_ratio/mean": 1.0849186182022095,
      "sampling/importance_sampling_ratio/min": 0.787323534488678,
      "sampling/sampling_logp_difference/max": 0.21123695373535156,
      "sampling/sampling_logp_difference/mean": 0.02896137535572052,
      "step": 275,
      "step_time": 12.001759131999279
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2513.0,
      "completions/max_terminated_length": 2513.0,
      "completions/mean_length": 926.125,
      "completions/mean_terminated_length": 926.4000244140625,
      "completions/min_length": 579.0,
      "completions/min_terminated_length": 579.0,
      "entropy": 1.356849081814289,
      "epoch": 0.00276,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1688052415847778,
      "kl": 0.6871709562838078,
      "learning_rate": 4.251432929272374e-05,
      "loss": 0.1325,
      "num_tokens": 6494674.0,
      "reward": 0.7570531368255615,
      "reward_std": 0.6871575713157654,
      "rewards/rollout_reward_func/mean": 0.7570531368255615,
      "rewards/rollout_reward_func/std": 0.6641770005226135,
      "sampling/importance_sampling_ratio/max": 1.3993521928787231,
      "sampling/importance_sampling_ratio/mean": 0.9967699646949768,
      "sampling/importance_sampling_ratio/min": 0.7159397602081299,
      "sampling/sampling_logp_difference/max": 0.2202610969543457,
      "sampling/sampling_logp_difference/mean": 0.024819733574986458,
      "step": 276,
      "step_time": 13.423720379000315
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1640.0,
      "completions/max_terminated_length": 1640.0,
      "completions/mean_length": 465.25,
      "completions/mean_terminated_length": 459.5384826660156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4518947377800941,
      "epoch": 0.00277,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9973200559616089,
      "kl": 0.5484465211629868,
      "learning_rate": 4.246210801899554e-05,
      "loss": 0.0219,
      "num_tokens": 6516856.0,
      "reward": 0.06439366191625595,
      "reward_std": 0.9234141111373901,
      "rewards/rollout_reward_func/mean": 0.06439366191625595,
      "rewards/rollout_reward_func/std": 0.9124034643173218,
      "sampling/importance_sampling_ratio/max": 1.1482360363006592,
      "sampling/importance_sampling_ratio/mean": 1.0107014179229736,
      "sampling/importance_sampling_ratio/min": 0.8776127696037292,
      "sampling/sampling_logp_difference/max": 0.13393402099609375,
      "sampling/sampling_logp_difference/mean": 0.023973794654011726,
      "step": 277,
      "step_time": 10.229707908999899
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1862.0,
      "completions/max_terminated_length": 1862.0,
      "completions/mean_length": 784.375,
      "completions/mean_terminated_length": 822.3846435546875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8751859962940216,
      "epoch": 0.00278,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6488038301467896,
      "kl": 0.8657292164862156,
      "learning_rate": 4.2409716466358836e-05,
      "loss": -0.1226,
      "num_tokens": 6544614.0,
      "reward": 0.41190123558044434,
      "reward_std": 0.7645725011825562,
      "rewards/rollout_reward_func/mean": 0.41190123558044434,
      "rewards/rollout_reward_func/std": 0.7423791289329529,
      "sampling/importance_sampling_ratio/max": 1.8657721281051636,
      "sampling/importance_sampling_ratio/mean": 1.01755952835083,
      "sampling/importance_sampling_ratio/min": 3.8421006852849197e-16,
      "sampling/sampling_logp_difference/max": 16.142135620117188,
      "sampling/sampling_logp_difference/mean": 0.21422770619392395,
      "step": 278,
      "step_time": 11.414973793000172
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1544.0,
      "completions/max_terminated_length": 1544.0,
      "completions/mean_length": 487.0625,
      "completions/mean_terminated_length": 429.5714416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.458553746342659,
      "epoch": 0.00279,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8970247507095337,
      "kl": 0.8246567882597446,
      "learning_rate": 4.235715525923504e-05,
      "loss": 0.1973,
      "num_tokens": 6566610.0,
      "reward": 0.18120554089546204,
      "reward_std": 0.9311363697052002,
      "rewards/rollout_reward_func/mean": 0.18120554089546204,
      "rewards/rollout_reward_func/std": 0.9018646478652954,
      "sampling/importance_sampling_ratio/max": 1.8735318183898926,
      "sampling/importance_sampling_ratio/mean": 1.1026455163955688,
      "sampling/importance_sampling_ratio/min": 0.9872448444366455,
      "sampling/sampling_logp_difference/max": 0.23799991607666016,
      "sampling/sampling_logp_difference/mean": 0.0346040353178978,
      "step": 279,
      "step_time": 9.68041051300088
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1682.0,
      "completions/max_terminated_length": 1682.0,
      "completions/mean_length": 693.25,
      "completions/mean_terminated_length": 697.2667236328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.491800770163536,
      "epoch": 0.0028,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5379760265350342,
      "kl": 1.0108853783458471,
      "learning_rate": 4.2304425024067604e-05,
      "loss": -0.0702,
      "num_tokens": 6592828.0,
      "reward": 0.24063408374786377,
      "reward_std": 0.8673262596130371,
      "rewards/rollout_reward_func/mean": 0.24063408374786377,
      "rewards/rollout_reward_func/std": 0.8532533645629883,
      "sampling/importance_sampling_ratio/max": 1.188344120979309,
      "sampling/importance_sampling_ratio/mean": 0.9878222942352295,
      "sampling/importance_sampling_ratio/min": 0.8072313070297241,
      "sampling/sampling_logp_difference/max": 0.16680622100830078,
      "sampling/sampling_logp_difference/mean": 0.02134426310658455,
      "step": 280,
      "step_time": 10.039953646999493
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1586.0,
      "completions/max_terminated_length": 1586.0,
      "completions/mean_length": 590.5625,
      "completions/mean_terminated_length": 521.1666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7201723605394363,
      "epoch": 0.00281,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.804480791091919,
      "kl": 0.8160188123583794,
      "learning_rate": 4.22515263893145e-05,
      "loss": 0.107,
      "num_tokens": 6617228.0,
      "reward": 0.6191225647926331,
      "reward_std": 0.740719199180603,
      "rewards/rollout_reward_func/mean": 0.6191225647926331,
      "rewards/rollout_reward_func/std": 0.715672492980957,
      "sampling/importance_sampling_ratio/max": 1.267541766166687,
      "sampling/importance_sampling_ratio/mean": 0.9333329796791077,
      "sampling/importance_sampling_ratio/min": 0.34985223412513733,
      "sampling/sampling_logp_difference/max": 0.24875164031982422,
      "sampling/sampling_logp_difference/mean": 0.03089229017496109,
      "step": 281,
      "step_time": 11.172894341000301
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1725.0,
      "completions/max_terminated_length": 1725.0,
      "completions/mean_length": 596.25,
      "completions/mean_terminated_length": 596.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3775381445884705,
      "epoch": 0.00282,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7770137786865234,
      "kl": 0.533398638246581,
      "learning_rate": 4.219845998544073e-05,
      "loss": -0.0088,
      "num_tokens": 6641809.0,
      "reward": 0.1891385167837143,
      "reward_std": 1.0025659799575806,
      "rewards/rollout_reward_func/mean": 0.1891385167837143,
      "rewards/rollout_reward_func/std": 0.9715558290481567,
      "sampling/importance_sampling_ratio/max": 1.2730321884155273,
      "sampling/importance_sampling_ratio/mean": 0.9751561284065247,
      "sampling/importance_sampling_ratio/min": 0.5980283617973328,
      "sampling/sampling_logp_difference/max": 0.19460749626159668,
      "sampling/sampling_logp_difference/mean": 0.028620382770895958,
      "step": 282,
      "step_time": 10.626714337000067
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1566.0,
      "completions/max_terminated_length": 1566.0,
      "completions/mean_length": 522.625,
      "completions/mean_terminated_length": 472.5384826660156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4163020104169846,
      "epoch": 0.00283,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3842575550079346,
      "kl": 0.4417529050260782,
      "learning_rate": 4.2145226444910854e-05,
      "loss": 0.0492,
      "num_tokens": 6665347.0,
      "reward": 0.22986209392547607,
      "reward_std": 0.6894406676292419,
      "rewards/rollout_reward_func/mean": 0.22986209392547607,
      "rewards/rollout_reward_func/std": 0.8575712442398071,
      "sampling/importance_sampling_ratio/max": 1.6188013553619385,
      "sampling/importance_sampling_ratio/mean": 0.9833325147628784,
      "sampling/importance_sampling_ratio/min": 0.6819217205047607,
      "sampling/sampling_logp_difference/max": 0.29618895053863525,
      "sampling/sampling_logp_difference/mean": 0.03102373518049717,
      "step": 283,
      "step_time": 10.13834147200032
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1785.0,
      "completions/max_terminated_length": 1785.0,
      "completions/mean_length": 702.875,
      "completions/mean_terminated_length": 741.6364135742188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5061514899134636,
      "epoch": 0.00284,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2244038581848145,
      "kl": 0.7244793884456158,
      "learning_rate": 4.20918264021814e-05,
      "loss": -0.1112,
      "num_tokens": 6691080.0,
      "reward": 0.7323398590087891,
      "reward_std": 0.4672985076904297,
      "rewards/rollout_reward_func/mean": 0.7323398590087891,
      "rewards/rollout_reward_func/std": 0.47915545105934143,
      "sampling/importance_sampling_ratio/max": 1.2660059928894043,
      "sampling/importance_sampling_ratio/mean": 1.0154671669006348,
      "sampling/importance_sampling_ratio/min": 0.8069363832473755,
      "sampling/sampling_logp_difference/max": 0.3127514123916626,
      "sampling/sampling_logp_difference/mean": 0.031900856643915176,
      "step": 284,
      "step_time": 11.390132341000935
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1838.0,
      "completions/max_terminated_length": 1838.0,
      "completions/mean_length": 568.5,
      "completions/mean_terminated_length": 553.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1096307672560215,
      "epoch": 0.00285,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6206812858581543,
      "kl": 0.5332447569817305,
      "learning_rate": 4.203826049369336e-05,
      "loss": 0.2859,
      "num_tokens": 6714713.0,
      "reward": 0.7422334551811218,
      "reward_std": 0.5579151511192322,
      "rewards/rollout_reward_func/mean": 0.7422334551811218,
      "rewards/rollout_reward_func/std": 0.5859925150871277,
      "sampling/importance_sampling_ratio/max": 1.176902174949646,
      "sampling/importance_sampling_ratio/mean": 0.9314146041870117,
      "sampling/importance_sampling_ratio/min": 4.555638366543491e-10,
      "sampling/sampling_logp_difference/max": 18.15966033935547,
      "sampling/sampling_logp_difference/mean": 0.1495424211025238,
      "step": 285,
      "step_time": 10.639348501000313
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1642.0,
      "completions/max_terminated_length": 1589.0,
      "completions/mean_length": 752.4375,
      "completions/mean_terminated_length": 676.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2160603627562523,
      "epoch": 0.00286,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.770919919013977,
      "kl": 0.4673234336078167,
      "learning_rate": 4.198452935786455e-05,
      "loss": -0.1536,
      "num_tokens": 6742194.0,
      "reward": 0.7424569129943848,
      "reward_std": 0.38253363966941833,
      "rewards/rollout_reward_func/mean": 0.7424569129943848,
      "rewards/rollout_reward_func/std": 0.5864327549934387,
      "sampling/importance_sampling_ratio/max": 1.3480457067489624,
      "sampling/importance_sampling_ratio/mean": 1.0121431350708008,
      "sampling/importance_sampling_ratio/min": 0.7312909960746765,
      "sampling/sampling_logp_difference/max": 0.3287431001663208,
      "sampling/sampling_logp_difference/mean": 0.028768833726644516,
      "step": 286,
      "step_time": 10.730055971998809
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1690.0,
      "completions/max_terminated_length": 1690.0,
      "completions/mean_length": 474.0,
      "completions/mean_terminated_length": 464.7692565917969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0933174192905426,
      "epoch": 0.00287,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9697467088699341,
      "kl": 0.7540105991065502,
      "learning_rate": 4.1930633635082046e-05,
      "loss": -0.0381,
      "num_tokens": 6764403.0,
      "reward": 0.42351973056793213,
      "reward_std": 0.7138786911964417,
      "rewards/rollout_reward_func/mean": 0.42351973056793213,
      "rewards/rollout_reward_func/std": 0.7318563461303711,
      "sampling/importance_sampling_ratio/max": 1.1244045495986938,
      "sampling/importance_sampling_ratio/mean": 1.0110259056091309,
      "sampling/importance_sampling_ratio/min": 0.8496595621109009,
      "sampling/sampling_logp_difference/max": 0.17789673805236816,
      "sampling/sampling_logp_difference/mean": 0.02454022690653801,
      "step": 287,
      "step_time": 10.271376266000516
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1688.0,
      "completions/max_terminated_length": 1688.0,
      "completions/mean_length": 656.0,
      "completions/mean_terminated_length": 570.7857666015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1432236125692725,
      "epoch": 0.00288,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.068013072013855,
      "kl": 0.4179973639547825,
      "learning_rate": 4.187657396769451e-05,
      "loss": 0.0413,
      "num_tokens": 6789535.0,
      "reward": 0.6918344497680664,
      "reward_std": 0.7101826071739197,
      "rewards/rollout_reward_func/mean": 0.6918344497680664,
      "rewards/rollout_reward_func/std": 0.6901659369468689,
      "sampling/importance_sampling_ratio/max": 1.1229156255722046,
      "sampling/importance_sampling_ratio/mean": 0.9470362663269043,
      "sampling/importance_sampling_ratio/min": 0.5942179560661316,
      "sampling/sampling_logp_difference/max": 0.16556310653686523,
      "sampling/sampling_logp_difference/mean": 0.028500277549028397,
      "step": 288,
      "step_time": 11.266294812001433
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1630.0,
      "completions/max_terminated_length": 1630.0,
      "completions/mean_length": 712.0625,
      "completions/mean_terminated_length": 665.2307739257812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3837142288684845,
      "epoch": 0.00289,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4808611869812012,
      "kl": 0.3845157641917467,
      "learning_rate": 4.182235100000456e-05,
      "loss": 0.2329,
      "num_tokens": 6815366.0,
      "reward": 0.3620920777320862,
      "reward_std": 0.8083854913711548,
      "rewards/rollout_reward_func/mean": 0.3620920777320862,
      "rewards/rollout_reward_func/std": 0.8015711903572083,
      "sampling/importance_sampling_ratio/max": 1.4845472574234009,
      "sampling/importance_sampling_ratio/mean": 1.00673508644104,
      "sampling/importance_sampling_ratio/min": 0.6460894346237183,
      "sampling/sampling_logp_difference/max": 0.12518930435180664,
      "sampling/sampling_logp_difference/mean": 0.023210225626826286,
      "step": 289,
      "step_time": 10.456702796000172
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1913.0,
      "completions/max_terminated_length": 1913.0,
      "completions/mean_length": 423.1875,
      "completions/mean_terminated_length": 368.0000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.044211259111762,
      "epoch": 0.0029,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.7048425674438477,
      "kl": 0.46899842843413353,
      "learning_rate": 4.1767965378261104e-05,
      "loss": 0.2439,
      "num_tokens": 6836047.0,
      "reward": 0.48628470301628113,
      "reward_std": 0.7633606195449829,
      "rewards/rollout_reward_func/mean": 0.48628470301628113,
      "rewards/rollout_reward_func/std": 0.7375158667564392,
      "sampling/importance_sampling_ratio/max": 2.614287853240967,
      "sampling/importance_sampling_ratio/mean": 1.1271467208862305,
      "sampling/importance_sampling_ratio/min": 0.8171318769454956,
      "sampling/sampling_logp_difference/max": 0.2377932071685791,
      "sampling/sampling_logp_difference/mean": 0.02831108681857586,
      "step": 290,
      "step_time": 10.789495520000855
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1536.0,
      "completions/max_terminated_length": 1536.0,
      "completions/mean_length": 576.1875,
      "completions/mean_terminated_length": 554.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.316634401679039,
      "epoch": 0.00291,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6567497849464417,
      "kl": 0.4646656606346369,
      "learning_rate": 4.171341775065159e-05,
      "loss": -0.0014,
      "num_tokens": 6860243.0,
      "reward": 0.3532358705997467,
      "reward_std": 0.7323111295700073,
      "rewards/rollout_reward_func/mean": 0.3532358705997467,
      "rewards/rollout_reward_func/std": 0.721943736076355,
      "sampling/importance_sampling_ratio/max": 1.268521785736084,
      "sampling/importance_sampling_ratio/mean": 1.0238635540008545,
      "sampling/importance_sampling_ratio/min": 0.912903368473053,
      "sampling/sampling_logp_difference/max": 0.17859840393066406,
      "sampling/sampling_logp_difference/mean": 0.01762966811656952,
      "step": 291,
      "step_time": 9.698903849998715
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2730.0,
      "completions/max_terminated_length": 1931.0,
      "completions/mean_length": 1163.75,
      "completions/mean_terminated_length": 1024.75,
      "completions/min_length": 675.0,
      "completions/min_terminated_length": 675.0,
      "entropy": 1.3790894523262978,
      "epoch": 0.00292,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6809593439102173,
      "kl": 0.5489352028816938,
      "learning_rate": 4.165870876729433e-05,
      "loss": 0.1898,
      "num_tokens": 6894823.0,
      "reward": 0.1197890192270279,
      "reward_std": 0.8206636905670166,
      "rewards/rollout_reward_func/mean": 0.1197890192270279,
      "rewards/rollout_reward_func/std": 0.7928484082221985,
      "sampling/importance_sampling_ratio/max": 1.1146150827407837,
      "sampling/importance_sampling_ratio/mean": 0.9189530611038208,
      "sampling/importance_sampling_ratio/min": 0.18057717382907867,
      "sampling/sampling_logp_difference/max": 0.27490234375,
      "sampling/sampling_logp_difference/mean": 0.031966160982847214,
      "step": 292,
      "step_time": 15.359549530001459
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2665.0,
      "completions/max_terminated_length": 2665.0,
      "completions/mean_length": 692.4375,
      "completions/mean_terminated_length": 685.2000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1859916970133781,
      "epoch": 0.00293,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9601539969444275,
      "kl": 1.115074459463358,
      "learning_rate": 4.160383908023075e-05,
      "loss": 0.0415,
      "num_tokens": 6921002.0,
      "reward": 0.400182843208313,
      "reward_std": 0.5757719874382019,
      "rewards/rollout_reward_func/mean": 0.400182843208313,
      "rewards/rollout_reward_func/std": 0.6578404903411865,
      "sampling/importance_sampling_ratio/max": 1.5049571990966797,
      "sampling/importance_sampling_ratio/mean": 1.0624030828475952,
      "sampling/importance_sampling_ratio/min": 0.9119511842727661,
      "sampling/sampling_logp_difference/max": 0.18671798706054688,
      "sampling/sampling_logp_difference/mean": 0.028399590402841568,
      "step": 293,
      "step_time": 12.636456962000011
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2449.0,
      "completions/max_terminated_length": 2449.0,
      "completions/mean_length": 775.6875,
      "completions/mean_terminated_length": 729.5385131835938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.2760003805160522,
      "epoch": 0.00294,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8497793078422546,
      "kl": 0.566764872521162,
      "learning_rate": 4.1548809343417544e-05,
      "loss": -0.2808,
      "num_tokens": 6948950.0,
      "reward": 0.4381363093852997,
      "reward_std": 0.8864172697067261,
      "rewards/rollout_reward_func/mean": 0.4381363093852997,
      "rewards/rollout_reward_func/std": 0.8834690451622009,
      "sampling/importance_sampling_ratio/max": 1.1939761638641357,
      "sampling/importance_sampling_ratio/mean": 0.8835992217063904,
      "sampling/importance_sampling_ratio/min": 0.3249095380306244,
      "sampling/sampling_logp_difference/max": 0.3173034191131592,
      "sampling/sampling_logp_difference/mean": 0.03410656377673149,
      "step": 294,
      "step_time": 14.4132196139999
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1734.0,
      "completions/max_terminated_length": 1711.0,
      "completions/mean_length": 570.3125,
      "completions/mean_terminated_length": 469.0714416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9668552181683481,
      "epoch": 0.00295,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4579090178012848,
      "kl": 0.3164177406579256,
      "learning_rate": 4.149362021271899e-05,
      "loss": -0.0894,
      "num_tokens": 6972066.0,
      "reward": 0.6266427040100098,
      "reward_std": 0.7288117408752441,
      "rewards/rollout_reward_func/mean": 0.6266427040100098,
      "rewards/rollout_reward_func/std": 0.7041630744934082,
      "sampling/importance_sampling_ratio/max": 1.0251575708389282,
      "sampling/importance_sampling_ratio/mean": 0.9719858169555664,
      "sampling/importance_sampling_ratio/min": 0.8447113037109375,
      "sampling/sampling_logp_difference/max": 0.14354991912841797,
      "sampling/sampling_logp_difference/mean": 0.024534102529287338,
      "step": 295,
      "step_time": 10.404367481001827
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1555.0,
      "completions/max_terminated_length": 1555.0,
      "completions/mean_length": 482.0625,
      "completions/mean_terminated_length": 482.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6986074494197965,
      "epoch": 0.00296,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5438892245292664,
      "kl": 0.6094178892672062,
      "learning_rate": 4.143827234589906e-05,
      "loss": 0.0179,
      "num_tokens": 6994034.0,
      "reward": 0.4238467514514923,
      "reward_std": 0.8401288986206055,
      "rewards/rollout_reward_func/mean": 0.4238467514514923,
      "rewards/rollout_reward_func/std": 0.8170391917228699,
      "sampling/importance_sampling_ratio/max": 1.0450351238250732,
      "sampling/importance_sampling_ratio/mean": 0.9979848861694336,
      "sampling/importance_sampling_ratio/min": 0.9462175369262695,
      "sampling/sampling_logp_difference/max": 0.10193824768066406,
      "sampling/sampling_logp_difference/mean": 0.010347738862037659,
      "step": 296,
      "step_time": 9.198868602998118
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 3389.0,
      "completions/max_terminated_length": 797.0,
      "completions/mean_length": 669.8125,
      "completions/mean_terminated_length": 230.10000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3021010579541326,
      "epoch": 0.00297,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5611211657524109,
      "kl": 0.4514406844973564,
      "learning_rate": 4.1382766402613605e-05,
      "loss": -0.2139,
      "num_tokens": 7018833.0,
      "reward": 0.6760094165802002,
      "reward_std": 0.5737863183021545,
      "rewards/rollout_reward_func/mean": 0.6760094165802002,
      "rewards/rollout_reward_func/std": 0.6164786219596863,
      "sampling/importance_sampling_ratio/max": 1.4416183233261108,
      "sampling/importance_sampling_ratio/mean": 0.8797194957733154,
      "sampling/importance_sampling_ratio/min": 4.9445498616665415e-22,
      "sampling/sampling_logp_difference/max": 23.53373908996582,
      "sampling/sampling_logp_difference/mean": 0.138726606965065,
      "step": 297,
      "step_time": 16.775515943000755
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2406.0,
      "completions/max_terminated_length": 2406.0,
      "completions/mean_length": 674.125,
      "completions/mean_terminated_length": 687.3333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3558335760608315,
      "epoch": 0.00298,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0072002410888672,
      "kl": 0.6021840199828148,
      "learning_rate": 4.132710304440247e-05,
      "loss": -0.0901,
      "num_tokens": 7043956.0,
      "reward": 0.36479446291923523,
      "reward_std": 0.8178280591964722,
      "rewards/rollout_reward_func/mean": 0.36479446291923523,
      "rewards/rollout_reward_func/std": 0.800805389881134,
      "sampling/importance_sampling_ratio/max": 1.2202361822128296,
      "sampling/importance_sampling_ratio/mean": 0.9307957887649536,
      "sampling/importance_sampling_ratio/min": 2.6998798761928626e-20,
      "sampling/sampling_logp_difference/max": 21.5030574798584,
      "sampling/sampling_logp_difference/mean": 0.26254671812057495,
      "step": 298,
      "step_time": 12.699216324999725
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1691.0,
      "completions/max_terminated_length": 861.0,
      "completions/mean_length": 495.5625,
      "completions/mean_terminated_length": 343.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4090211987495422,
      "epoch": 0.00299,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0774248838424683,
      "kl": 0.4959158357232809,
      "learning_rate": 4.1271282934681657e-05,
      "loss": 0.1687,
      "num_tokens": 7066278.0,
      "reward": 0.24133460223674774,
      "reward_std": 0.8613111972808838,
      "rewards/rollout_reward_func/mean": 0.24133460223674774,
      "rewards/rollout_reward_func/std": 0.8484061360359192,
      "sampling/importance_sampling_ratio/max": 1.121416687965393,
      "sampling/importance_sampling_ratio/mean": 0.9610395431518555,
      "sampling/importance_sampling_ratio/min": 0.6882920861244202,
      "sampling/sampling_logp_difference/max": 0.16701412200927734,
      "sampling/sampling_logp_difference/mean": 0.028784601017832756,
      "step": 299,
      "step_time": 10.38288599400039
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 860.0,
      "completions/max_terminated_length": 835.0,
      "completions/mean_length": 340.9375,
      "completions/mean_terminated_length": 197.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.453514076769352,
      "epoch": 0.003,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4231995344161987,
      "kl": 0.6338005429133773,
      "learning_rate": 4.1215306738735375e-05,
      "loss": 0.236,
      "num_tokens": 7085678.0,
      "reward": 0.6789179444313049,
      "reward_std": 0.5598816275596619,
      "rewards/rollout_reward_func/mean": 0.6789179444313049,
      "rewards/rollout_reward_func/std": 0.60263592004776,
      "sampling/importance_sampling_ratio/max": 1.3225539922714233,
      "sampling/importance_sampling_ratio/mean": 0.9712811708450317,
      "sampling/importance_sampling_ratio/min": 6.189185465808921e-15,
      "sampling/sampling_logp_difference/max": 20.95208168029785,
      "sampling/sampling_logp_difference/mean": 0.19215109944343567,
      "step": 300,
      "step_time": 8.263143628998478
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1815.0,
      "completions/max_terminated_length": 1630.0,
      "completions/mean_length": 666.625,
      "completions/mean_terminated_length": 479.3999938964844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8677552789449692,
      "epoch": 0.00301,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1353565454483032,
      "kl": 0.5296441381797194,
      "learning_rate": 4.115917512370812e-05,
      "loss": -0.1141,
      "num_tokens": 7111368.0,
      "reward": 0.4996926784515381,
      "reward_std": 0.7792251110076904,
      "rewards/rollout_reward_func/mean": 0.4996926784515381,
      "rewards/rollout_reward_func/std": 0.8050742745399475,
      "sampling/importance_sampling_ratio/max": 1.3729815483093262,
      "sampling/importance_sampling_ratio/mean": 0.9825010299682617,
      "sampling/importance_sampling_ratio/min": 0.7431766390800476,
      "sampling/sampling_logp_difference/max": 0.18523240089416504,
      "sampling/sampling_logp_difference/mean": 0.030439937487244606,
      "step": 301,
      "step_time": 12.235758081998938
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2734.0,
      "completions/max_terminated_length": 2734.0,
      "completions/mean_length": 881.625,
      "completions/mean_terminated_length": 794.0833740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9194130823016167,
      "epoch": 0.00302,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.546552300453186,
      "kl": 0.6615241579711437,
      "learning_rate": 4.1102888758596734e-05,
      "loss": 0.1904,
      "num_tokens": 7140520.0,
      "reward": 0.32331186532974243,
      "reward_std": 0.8816493153572083,
      "rewards/rollout_reward_func/mean": 0.32331186532974243,
      "rewards/rollout_reward_func/std": 0.8551563620567322,
      "sampling/importance_sampling_ratio/max": 1.6112374067306519,
      "sampling/importance_sampling_ratio/mean": 1.0511621236801147,
      "sampling/importance_sampling_ratio/min": 0.6218595504760742,
      "sampling/sampling_logp_difference/max": 0.17823290824890137,
      "sampling/sampling_logp_difference/mean": 0.028645440936088562,
      "step": 302,
      "step_time": 14.895068175999768
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 1737.0,
      "completions/max_terminated_length": 1737.0,
      "completions/mean_length": 670.6875,
      "completions/mean_terminated_length": 705.5555419921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.166866049170494,
      "epoch": 0.00303,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6529778242111206,
      "kl": 0.6049652509391308,
      "learning_rate": 4.1046448314242425e-05,
      "loss": 0.0752,
      "num_tokens": 7166211.0,
      "reward": 0.7450451850891113,
      "reward_std": 0.5583691000938416,
      "rewards/rollout_reward_func/mean": 0.7450451850891113,
      "rewards/rollout_reward_func/std": 0.5791568160057068,
      "sampling/importance_sampling_ratio/max": 1.5599608421325684,
      "sampling/importance_sampling_ratio/mean": 1.0063940286636353,
      "sampling/importance_sampling_ratio/min": 0.4360499978065491,
      "sampling/sampling_logp_difference/max": 0.24377524852752686,
      "sampling/sampling_logp_difference/mean": 0.03143956884741783,
      "step": 303,
      "step_time": 11.782199765000769
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1750.0,
      "completions/max_terminated_length": 1750.0,
      "completions/mean_length": 703.25,
      "completions/mean_terminated_length": 640.2999877929688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.1009800136089325,
      "epoch": 0.00304,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.736402988433838,
      "kl": 0.5755608901381493,
      "learning_rate": 4.098985446332277e-05,
      "loss": 0.0801,
      "num_tokens": 7192176.0,
      "reward": 0.4364893436431885,
      "reward_std": 0.7982026934623718,
      "rewards/rollout_reward_func/mean": 0.4364893436431885,
      "rewards/rollout_reward_func/std": 0.800060510635376,
      "sampling/importance_sampling_ratio/max": 1.5903726816177368,
      "sampling/importance_sampling_ratio/mean": 1.0032038688659668,
      "sampling/importance_sampling_ratio/min": 0.7119848132133484,
      "sampling/sampling_logp_difference/max": 0.3623121976852417,
      "sampling/sampling_logp_difference/mean": 0.032202787697315216,
      "step": 304,
      "step_time": 12.276548426998488
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1718.0,
      "completions/max_terminated_length": 851.0,
      "completions/mean_length": 456.125,
      "completions/mean_terminated_length": 208.08334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5035693664103746,
      "epoch": 0.00305,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.987403392791748,
      "kl": 0.3882460854947567,
      "learning_rate": 4.093310788034371e-05,
      "loss": -0.1615,
      "num_tokens": 7213514.0,
      "reward": 0.504336416721344,
      "reward_std": 0.9177907705307007,
      "rewards/rollout_reward_func/mean": 0.504336416721344,
      "rewards/rollout_reward_func/std": 0.886707603931427,
      "sampling/importance_sampling_ratio/max": 1.3175257444381714,
      "sampling/importance_sampling_ratio/mean": 0.9734377861022949,
      "sampling/importance_sampling_ratio/min": 0.6443927884101868,
      "sampling/sampling_logp_difference/max": 0.2866687774658203,
      "sampling/sampling_logp_difference/mean": 0.029632296413183212,
      "step": 305,
      "step_time": 11.022451423999883
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 2575.0,
      "completions/max_terminated_length": 2575.0,
      "completions/mean_length": 714.625,
      "completions/mean_terminated_length": 644.2000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7655046470463276,
      "epoch": 0.00306,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.395002841949463,
      "kl": 0.5654574781656265,
      "learning_rate": 4.087620924163151e-05,
      "loss": -0.0804,
      "num_tokens": 7239980.0,
      "reward": 0.48878395557403564,
      "reward_std": 0.7247069478034973,
      "rewards/rollout_reward_func/mean": 0.48878395557403564,
      "rewards/rollout_reward_func/std": 0.7244811058044434,
      "sampling/importance_sampling_ratio/max": 1.7885756492614746,
      "sampling/importance_sampling_ratio/mean": 0.9223492741584778,
      "sampling/importance_sampling_ratio/min": 0.2425624430179596,
      "sampling/sampling_logp_difference/max": 0.28858089447021484,
      "sampling/sampling_logp_difference/mean": 0.03584450110793114,
      "step": 306,
      "step_time": 14.09513756799879
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1044.0,
      "completions/max_terminated_length": 763.0,
      "completions/mean_length": 457.3125,
      "completions/mean_terminated_length": 374.9090881347656,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6612088158726692,
      "epoch": 0.00307,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9980422854423523,
      "kl": 0.6143200173974037,
      "learning_rate": 4.0819159225324654e-05,
      "loss": 0.0308,
      "num_tokens": 7262297.0,
      "reward": 0.4383153021335602,
      "reward_std": 0.9123969078063965,
      "rewards/rollout_reward_func/mean": 0.4383153021335602,
      "rewards/rollout_reward_func/std": 0.8840301632881165,
      "sampling/importance_sampling_ratio/max": 1.2819560766220093,
      "sampling/importance_sampling_ratio/mean": 0.9288212656974792,
      "sampling/importance_sampling_ratio/min": 6.391491780277647e-10,
      "sampling/sampling_logp_difference/max": 19.8245849609375,
      "sampling/sampling_logp_difference/mean": 0.10888148844242096,
      "step": 307,
      "step_time": 9.169477184999778
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 2588.0,
      "completions/max_terminated_length": 736.0,
      "completions/mean_length": 549.625,
      "completions/mean_terminated_length": 235.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7296932088211179,
      "epoch": 0.00308,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.697035014629364,
      "kl": 0.4671720452606678,
      "learning_rate": 4.076195851136583e-05,
      "loss": -0.2048,
      "num_tokens": 7285700.0,
      "reward": 0.806312084197998,
      "reward_std": 0.377208948135376,
      "rewards/rollout_reward_func/mean": 0.806312084197998,
      "rewards/rollout_reward_func/std": 0.5528269410133362,
      "sampling/importance_sampling_ratio/max": 1.369546890258789,
      "sampling/importance_sampling_ratio/mean": 1.0155565738677979,
      "sampling/importance_sampling_ratio/min": 0.6148578524589539,
      "sampling/sampling_logp_difference/max": 0.22695374488830566,
      "sampling/sampling_logp_difference/mean": 0.03363551199436188,
      "step": 308,
      "step_time": 14.65557849599918
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1901.0,
      "completions/max_terminated_length": 1614.0,
      "completions/mean_length": 645.0625,
      "completions/mean_terminated_length": 544.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.708402855321765,
      "epoch": 0.00309,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4225167036056519,
      "kl": 0.5765058882534504,
      "learning_rate": 4.070460778149379e-05,
      "loss": -0.2317,
      "num_tokens": 7310990.0,
      "reward": 0.48043644428253174,
      "reward_std": 0.7690227031707764,
      "rewards/rollout_reward_func/mean": 0.48043644428253174,
      "rewards/rollout_reward_func/std": 0.7429682016372681,
      "sampling/importance_sampling_ratio/max": 1.663087010383606,
      "sampling/importance_sampling_ratio/mean": 0.9951388835906982,
      "sampling/importance_sampling_ratio/min": 0.4753367602825165,
      "sampling/sampling_logp_difference/max": 0.2955164909362793,
      "sampling/sampling_logp_difference/mean": 0.033305346965789795,
      "step": 309,
      "step_time": 12.465147654000248
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1745.0,
      "completions/max_terminated_length": 1745.0,
      "completions/mean_length": 632.375,
      "completions/mean_terminated_length": 500.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5463018654845655,
      "epoch": 0.0031,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2354611158370972,
      "kl": 0.5721954666078091,
      "learning_rate": 4.064710771923522e-05,
      "loss": 0.2505,
      "num_tokens": 7335785.0,
      "reward": 0.34739360213279724,
      "reward_std": 0.7568869590759277,
      "rewards/rollout_reward_func/mean": 0.34739360213279724,
      "rewards/rollout_reward_func/std": 0.7312448024749756,
      "sampling/importance_sampling_ratio/max": 1.3644371032714844,
      "sampling/importance_sampling_ratio/mean": 0.9733877182006836,
      "sampling/importance_sampling_ratio/min": 0.6660746932029724,
      "sampling/sampling_logp_difference/max": 0.21880745887756348,
      "sampling/sampling_logp_difference/mean": 0.02927979826927185,
      "step": 310,
      "step_time": 11.62270393200015
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 797.0,
      "completions/max_terminated_length": 797.0,
      "completions/mean_length": 354.4375,
      "completions/mean_terminated_length": 301.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0871569029986858,
      "epoch": 0.00311,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0645114183425903,
      "kl": 0.35575393307954073,
      "learning_rate": 4.0589459009896614e-05,
      "loss": 0.0925,
      "num_tokens": 7355695.0,
      "reward": 0.6214690208435059,
      "reward_std": 0.634225606918335,
      "rewards/rollout_reward_func/mean": 0.6214690208435059,
      "rewards/rollout_reward_func/std": 0.715661346912384,
      "sampling/importance_sampling_ratio/max": 1.0837198495864868,
      "sampling/importance_sampling_ratio/mean": 0.988154947757721,
      "sampling/importance_sampling_ratio/min": 0.9284036159515381,
      "sampling/sampling_logp_difference/max": 0.12284970283508301,
      "sampling/sampling_logp_difference/mean": 0.021563949063420296,
      "step": 311,
      "step_time": 7.147130427998491
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1776.0,
      "completions/max_terminated_length": 1776.0,
      "completions/mean_length": 369.375,
      "completions/mean_terminated_length": 255.08334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6458533708937466,
      "epoch": 0.00312,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5769093036651611,
      "kl": 0.48059387039393187,
      "learning_rate": 4.0531662340556094e-05,
      "loss": 0.0783,
      "num_tokens": 7375916.0,
      "reward": 0.22860027849674225,
      "reward_std": 0.842038631439209,
      "rewards/rollout_reward_func/mean": 0.22860027849674225,
      "rewards/rollout_reward_func/std": 0.8557219505310059,
      "sampling/importance_sampling_ratio/max": 1.4634160995483398,
      "sampling/importance_sampling_ratio/mean": 1.0338101387023926,
      "sampling/importance_sampling_ratio/min": 0.590807318687439,
      "sampling/sampling_logp_difference/max": 0.32657718658447266,
      "sampling/sampling_logp_difference/mean": 0.034351617097854614,
      "step": 312,
      "step_time": 10.419314272999145
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1810.0,
      "completions/max_terminated_length": 1780.0,
      "completions/mean_length": 614.9375,
      "completions/mean_terminated_length": 535.2667236328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9886329921428114,
      "epoch": 0.00313,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9239383935928345,
      "kl": 0.4316693488508463,
      "learning_rate": 4.04737184000552e-05,
      "loss": 0.0583,
      "num_tokens": 7400043.0,
      "reward": 0.6080436706542969,
      "reward_std": 0.6294056177139282,
      "rewards/rollout_reward_func/mean": 0.6080436706542969,
      "rewards/rollout_reward_func/std": 0.6337771415710449,
      "sampling/importance_sampling_ratio/max": 1.2796276807785034,
      "sampling/importance_sampling_ratio/mean": 0.9952926635742188,
      "sampling/importance_sampling_ratio/min": 0.504689633846283,
      "sampling/sampling_logp_difference/max": 0.1964414119720459,
      "sampling/sampling_logp_difference/mean": 0.026631465181708336,
      "step": 313,
      "step_time": 10.622524937998605
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1675.0,
      "completions/max_terminated_length": 1675.0,
      "completions/mean_length": 574.375,
      "completions/mean_terminated_length": 483.7857360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6736167806666344,
      "epoch": 0.00314,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.4934581220149994,
      "kl": 0.6465176464989781,
      "learning_rate": 4.041562787899075e-05,
      "loss": -0.0509,
      "num_tokens": 7423557.0,
      "reward": 0.6784943342208862,
      "reward_std": 0.3706943988800049,
      "rewards/rollout_reward_func/mean": 0.6784943342208862,
      "rewards/rollout_reward_func/std": 0.6056102514266968,
      "sampling/importance_sampling_ratio/max": 1.6376314163208008,
      "sampling/importance_sampling_ratio/mean": 1.0487158298492432,
      "sampling/importance_sampling_ratio/min": 0.9057880640029907,
      "sampling/sampling_logp_difference/max": 0.1678084135055542,
      "sampling/sampling_logp_difference/mean": 0.01973680779337883,
      "step": 314,
      "step_time": 10.860849057999076
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 950.0,
      "completions/max_terminated_length": 871.0,
      "completions/mean_length": 533.4375,
      "completions/mean_terminated_length": 505.66668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.025212049484253,
      "epoch": 0.00315,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.36111539602279663,
      "kl": 0.40081563778221607,
      "learning_rate": 4.035739146970652e-05,
      "loss": -0.0166,
      "num_tokens": 7447345.0,
      "reward": 0.17987212538719177,
      "reward_std": 0.8366234302520752,
      "rewards/rollout_reward_func/mean": 0.17987212538719177,
      "rewards/rollout_reward_func/std": 0.8210421800613403,
      "sampling/importance_sampling_ratio/max": 1.0478744506835938,
      "sampling/importance_sampling_ratio/mean": 0.906028687953949,
      "sampling/importance_sampling_ratio/min": 1.3481260376418534e-19,
      "sampling/sampling_logp_difference/max": 16.184261322021484,
      "sampling/sampling_logp_difference/mean": 0.4321822226047516,
      "step": 315,
      "step_time": 8.006404808001207
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1666.0,
      "completions/max_terminated_length": 1666.0,
      "completions/mean_length": 599.9375,
      "completions/mean_terminated_length": 577.5333862304688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0806834562681615,
      "epoch": 0.00316,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4035702049732208,
      "kl": 0.726960910949856,
      "learning_rate": 4.029900986628505e-05,
      "loss": 0.0323,
      "num_tokens": 7471585.0,
      "reward": 0.4164583086967468,
      "reward_std": 0.7610534429550171,
      "rewards/rollout_reward_func/mean": 0.4164583086967468,
      "rewards/rollout_reward_func/std": 0.7385413646697998,
      "sampling/importance_sampling_ratio/max": 1.0363030433654785,
      "sampling/importance_sampling_ratio/mean": 0.9731125235557556,
      "sampling/importance_sampling_ratio/min": 0.790616512298584,
      "sampling/sampling_logp_difference/max": 0.11883378028869629,
      "sampling/sampling_logp_difference/mean": 0.02122570388019085,
      "step": 316,
      "step_time": 9.71340660900114
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1801.0,
      "completions/max_terminated_length": 1630.0,
      "completions/mean_length": 557.4375,
      "completions/mean_terminated_length": 474.5333557128906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7051825802773237,
      "epoch": 0.00317,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.39807647466659546,
      "kl": 0.7334528677165508,
      "learning_rate": 4.024048376453935e-05,
      "loss": -0.0519,
      "num_tokens": 7494887.0,
      "reward": 0.5372729897499084,
      "reward_std": 0.5505901575088501,
      "rewards/rollout_reward_func/mean": 0.5372729897499084,
      "rewards/rollout_reward_func/std": 0.6514042615890503,
      "sampling/importance_sampling_ratio/max": 1.0577863454818726,
      "sampling/importance_sampling_ratio/mean": 0.9953131675720215,
      "sampling/importance_sampling_ratio/min": 0.8157786130905151,
      "sampling/sampling_logp_difference/max": 0.20591068267822266,
      "sampling/sampling_logp_difference/mean": 0.01774250902235508,
      "step": 317,
      "step_time": 10.433643788999689
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1690.0,
      "completions/max_terminated_length": 1690.0,
      "completions/mean_length": 597.9375,
      "completions/mean_terminated_length": 551.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2485958002507687,
      "epoch": 0.00318,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2312023639678955,
      "kl": 0.5156844537705183,
      "learning_rate": 4.0181813862004625e-05,
      "loss": 0.3838,
      "num_tokens": 7518784.0,
      "reward": 0.5983943343162537,
      "reward_std": 0.5334715843200684,
      "rewards/rollout_reward_func/mean": 0.5983943343162537,
      "rewards/rollout_reward_func/std": 0.5357301235198975,
      "sampling/importance_sampling_ratio/max": 1.6614121198654175,
      "sampling/importance_sampling_ratio/mean": 1.026392936706543,
      "sampling/importance_sampling_ratio/min": 0.8322691321372986,
      "sampling/sampling_logp_difference/max": 0.21708297729492188,
      "sampling/sampling_logp_difference/mean": 0.02720274031162262,
      "step": 318,
      "step_time": 10.470667468001011
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2556.0,
      "completions/max_terminated_length": 2556.0,
      "completions/mean_length": 695.875,
      "completions/mean_terminated_length": 678.0667114257812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.743917983956635,
      "epoch": 0.00319,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.48074206709861755,
      "kl": 0.53515350446105,
      "learning_rate": 4.012300085792995e-05,
      "loss": 0.0664,
      "num_tokens": 7543918.0,
      "reward": 0.280802845954895,
      "reward_std": 0.6215623021125793,
      "rewards/rollout_reward_func/mean": 0.280802845954895,
      "rewards/rollout_reward_func/std": 0.6184473633766174,
      "sampling/importance_sampling_ratio/max": 1.721224308013916,
      "sampling/importance_sampling_ratio/mean": 1.0773353576660156,
      "sampling/importance_sampling_ratio/min": 0.9337410926818848,
      "sampling/sampling_logp_difference/max": 0.24039554595947266,
      "sampling/sampling_logp_difference/mean": 0.021086886525154114,
      "step": 319,
      "step_time": 12.264919328998985
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1818.0,
      "completions/max_terminated_length": 1818.0,
      "completions/mean_length": 635.625,
      "completions/mean_terminated_length": 635.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.897801504470408,
      "epoch": 0.0032,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7078927755355835,
      "kl": 0.9254992119967937,
      "learning_rate": 4.006404545326992e-05,
      "loss": -0.0527,
      "num_tokens": 7568669.0,
      "reward": 0.42931127548217773,
      "reward_std": 0.6533875465393066,
      "rewards/rollout_reward_func/mean": 0.42931127548217773,
      "rewards/rollout_reward_func/std": 0.7261368632316589,
      "sampling/importance_sampling_ratio/max": 1.0947198867797852,
      "sampling/importance_sampling_ratio/mean": 0.9891688823699951,
      "sampling/importance_sampling_ratio/min": 0.8410474061965942,
      "sampling/sampling_logp_difference/max": 0.1614079475402832,
      "sampling/sampling_logp_difference/mean": 0.026779988780617714,
      "step": 320,
      "step_time": 10.282759597999757
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1665.0,
      "completions/max_terminated_length": 1665.0,
      "completions/mean_length": 596.125,
      "completions/mean_terminated_length": 573.1333618164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6674159653484821,
      "epoch": 0.00321,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7114633917808533,
      "kl": 0.4163746815174818,
      "learning_rate": 4.000494835067631e-05,
      "loss": 0.1404,
      "num_tokens": 7592904.0,
      "reward": 0.5589179992675781,
      "reward_std": 0.739634096622467,
      "rewards/rollout_reward_func/mean": 0.5589179992675781,
      "rewards/rollout_reward_func/std": 0.7178892493247986,
      "sampling/importance_sampling_ratio/max": 1.0876083374023438,
      "sampling/importance_sampling_ratio/mean": 0.971351146697998,
      "sampling/importance_sampling_ratio/min": 0.6514509320259094,
      "sampling/sampling_logp_difference/max": 0.09516501426696777,
      "sampling/sampling_logp_difference/mean": 0.01392505131661892,
      "step": 321,
      "step_time": 10.384528030000183
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1591.0,
      "completions/max_terminated_length": 1591.0,
      "completions/mean_length": 391.6875,
      "completions/mean_terminated_length": 391.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4735610643401742,
      "epoch": 0.00322,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.22529664635658264,
      "kl": 0.4139766599982977,
      "learning_rate": 3.9945710254489735e-05,
      "loss": 0.0131,
      "num_tokens": 7612864.0,
      "reward": 0.435030996799469,
      "reward_std": 0.8300023078918457,
      "rewards/rollout_reward_func/mean": 0.435030996799469,
      "rewards/rollout_reward_func/std": 0.8073007464408875,
      "sampling/importance_sampling_ratio/max": 1.0983426570892334,
      "sampling/importance_sampling_ratio/mean": 0.9995020627975464,
      "sampling/importance_sampling_ratio/min": 0.9113068580627441,
      "sampling/sampling_logp_difference/max": 0.10978507995605469,
      "sampling/sampling_logp_difference/mean": 0.01227408368140459,
      "step": 322,
      "step_time": 9.194226657998115
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 4324.0,
      "completions/max_terminated_length": 4324.0,
      "completions/mean_length": 861.4375,
      "completions/mean_terminated_length": 861.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6258032363839447,
      "epoch": 0.00323,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.3482849597930908,
      "kl": 0.4725131783634424,
      "learning_rate": 3.9886331870731184e-05,
      "loss": -0.0088,
      "num_tokens": 7640792.0,
      "reward": 0.933499276638031,
      "reward_std": 0.18809248507022858,
      "rewards/rollout_reward_func/mean": 0.933499276638031,
      "rewards/rollout_reward_func/std": 0.26600295305252075,
      "sampling/importance_sampling_ratio/max": 1.0502811670303345,
      "sampling/importance_sampling_ratio/mean": 1.0029804706573486,
      "sampling/importance_sampling_ratio/min": 0.9317668676376343,
      "sampling/sampling_logp_difference/max": 0.07304203510284424,
      "sampling/sampling_logp_difference/mean": 0.009507612325251102,
      "step": 323,
      "step_time": 17.270819631999984
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1729.0,
      "completions/max_terminated_length": 1729.0,
      "completions/mean_length": 699.4375,
      "completions/mean_terminated_length": 686.0000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9887384846806526,
      "epoch": 0.00324,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.62843918800354,
      "kl": 0.3733692057430744,
      "learning_rate": 3.982681390709366e-05,
      "loss": 0.071,
      "num_tokens": 7667388.0,
      "reward": 0.004609048366546631,
      "reward_std": 0.8609265089035034,
      "rewards/rollout_reward_func/mean": 0.004609048366546631,
      "rewards/rollout_reward_func/std": 0.8710720539093018,
      "sampling/importance_sampling_ratio/max": 1.1306939125061035,
      "sampling/importance_sampling_ratio/mean": 0.984635591506958,
      "sampling/importance_sampling_ratio/min": 0.8297060132026672,
      "sampling/sampling_logp_difference/max": 0.1483612060546875,
      "sampling/sampling_logp_difference/mean": 0.022126516327261925,
      "step": 324,
      "step_time": 10.105395060001683
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1727.0,
      "completions/max_terminated_length": 1727.0,
      "completions/mean_length": 650.0,
      "completions/mean_terminated_length": 650.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7535242671146989,
      "epoch": 0.00325,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.2858944237232208,
      "kl": 0.42277564108371735,
      "learning_rate": 3.976715707293375e-05,
      "loss": 0.0042,
      "num_tokens": 7692573.0,
      "reward": 0.6179530620574951,
      "reward_std": 0.4398980736732483,
      "rewards/rollout_reward_func/mean": 0.6179530620574951,
      "rewards/rollout_reward_func/std": 0.7189643979072571,
      "sampling/importance_sampling_ratio/max": 1.0968858003616333,
      "sampling/importance_sampling_ratio/mean": 0.997590959072113,
      "sampling/importance_sampling_ratio/min": 0.8693801164627075,
      "sampling/sampling_logp_difference/max": 0.15086400508880615,
      "sampling/sampling_logp_difference/mean": 0.015261409804224968,
      "step": 325,
      "step_time": 9.871496803998525
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 797.0,
      "completions/max_terminated_length": 797.0,
      "completions/mean_length": 458.8125,
      "completions/mean_terminated_length": 458.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49799441429786384,
      "epoch": 0.00326,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.3966423571109772,
      "kl": 0.33806504122912884,
      "learning_rate": 3.9707362079263095e-05,
      "loss": 0.0102,
      "num_tokens": 7714131.0,
      "reward": 0.7964687347412109,
      "reward_std": 0.28096744418144226,
      "rewards/rollout_reward_func/mean": 0.7964687347412109,
      "rewards/rollout_reward_func/std": 0.4376600980758667,
      "sampling/importance_sampling_ratio/max": 1.0372142791748047,
      "sampling/importance_sampling_ratio/mean": 0.9975132942199707,
      "sampling/importance_sampling_ratio/min": 0.9390671849250793,
      "sampling/sampling_logp_difference/max": 0.09310150146484375,
      "sampling/sampling_logp_difference/mean": 0.008770523592829704,
      "step": 326,
      "step_time": 7.108374710001954
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2608.0,
      "completions/max_terminated_length": 2608.0,
      "completions/mean_length": 1020.5625,
      "completions/mean_terminated_length": 1020.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7228435128927231,
      "epoch": 0.00327,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.25829148292541504,
      "kl": 0.4342989157885313,
      "learning_rate": 3.9647429638740026e-05,
      "loss": 0.005,
      "num_tokens": 7746137.0,
      "reward": 0.7984557747840881,
      "reward_std": 0.2783929407596588,
      "rewards/rollout_reward_func/mean": 0.7984557747840881,
      "rewards/rollout_reward_func/std": 0.43358924984931946,
      "sampling/importance_sampling_ratio/max": 1.1127039194107056,
      "sampling/importance_sampling_ratio/mean": 0.9927463531494141,
      "sampling/importance_sampling_ratio/min": 0.8919141888618469,
      "sampling/sampling_logp_difference/max": 0.16916847229003906,
      "sampling/sampling_logp_difference/mean": 0.017531078308820724,
      "step": 327,
      "step_time": 13.302569420998225
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1707.0,
      "completions/max_terminated_length": 1707.0,
      "completions/mean_length": 543.875,
      "completions/mean_terminated_length": 543.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4906462816288695,
      "epoch": 0.00328,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.17438924312591553,
      "kl": 0.33628518879413605,
      "learning_rate": 3.9587360465661e-05,
      "loss": 0.0006,
      "num_tokens": 7768871.0,
      "reward": 0.6853141784667969,
      "reward_std": 0.5575411915779114,
      "rewards/rollout_reward_func/mean": 0.6853141784667969,
      "rewards/rollout_reward_func/std": 0.5954696536064148,
      "sampling/importance_sampling_ratio/max": 1.0235995054244995,
      "sampling/importance_sampling_ratio/mean": 0.9733259677886963,
      "sampling/importance_sampling_ratio/min": 0.8150136470794678,
      "sampling/sampling_logp_difference/max": 0.12148666381835938,
      "sampling/sampling_logp_difference/mean": 0.010706808418035507,
      "step": 328,
      "step_time": 9.740118057999098
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1844.0,
      "completions/max_terminated_length": 1844.0,
      "completions/mean_length": 742.3125,
      "completions/mean_terminated_length": 742.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.877697117626667,
      "epoch": 0.00329,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5202599763870239,
      "kl": 0.44560779072344303,
      "learning_rate": 3.952715527595208e-05,
      "loss": 0.0705,
      "num_tokens": 7796024.0,
      "reward": 0.1972154974937439,
      "reward_std": 0.9924910664558411,
      "rewards/rollout_reward_func/mean": 0.1972154974937439,
      "rewards/rollout_reward_func/std": 0.9616768956184387,
      "sampling/importance_sampling_ratio/max": 1.130056619644165,
      "sampling/importance_sampling_ratio/mean": 0.9957617521286011,
      "sampling/importance_sampling_ratio/min": 0.8931996822357178,
      "sampling/sampling_logp_difference/max": 0.16840887069702148,
      "sampling/sampling_logp_difference/mean": 0.02132139354944229,
      "step": 329,
      "step_time": 10.490949333000572
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1644.0,
      "completions/max_terminated_length": 1644.0,
      "completions/mean_length": 406.5,
      "completions/mean_terminated_length": 406.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5787901219446212,
      "epoch": 0.0033,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.26725342869758606,
      "kl": 0.47268530167639256,
      "learning_rate": 3.946681478716044e-05,
      "loss": 0.0022,
      "num_tokens": 7816411.0,
      "reward": 0.8113607168197632,
      "reward_std": 0.37311798334121704,
      "rewards/rollout_reward_func/mean": 0.8113607168197632,
      "rewards/rollout_reward_func/std": 0.5457370281219482,
      "sampling/importance_sampling_ratio/max": 1.1023671627044678,
      "sampling/importance_sampling_ratio/mean": 1.0085952281951904,
      "sampling/importance_sampling_ratio/min": 0.9057478308677673,
      "sampling/sampling_logp_difference/max": 0.10855042934417725,
      "sampling/sampling_logp_difference/mean": 0.01230402197688818,
      "step": 330,
      "step_time": 9.352015644999483
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1707.0,
      "completions/max_terminated_length": 1707.0,
      "completions/mean_length": 418.375,
      "completions/mean_terminated_length": 418.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4943773776758462,
      "epoch": 0.00331,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.22881725430488586,
      "kl": 0.6844453550875187,
      "learning_rate": 3.940633971844582e-05,
      "loss": -0.0048,
      "num_tokens": 7837407.0,
      "reward": 0.7288408279418945,
      "reward_std": 0.5024174451828003,
      "rewards/rollout_reward_func/mean": 0.7288408279418945,
      "rewards/rollout_reward_func/std": 0.48548007011413574,
      "sampling/importance_sampling_ratio/max": 1.215875506401062,
      "sampling/importance_sampling_ratio/mean": 1.0232782363891602,
      "sampling/importance_sampling_ratio/min": 0.9515767693519592,
      "sampling/sampling_logp_difference/max": 0.11864662170410156,
      "sampling/sampling_logp_difference/mean": 0.012958215549588203,
      "step": 331,
      "step_time": 9.44907676999992
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2572.0,
      "completions/max_terminated_length": 2572.0,
      "completions/mean_length": 583.6875,
      "completions/mean_terminated_length": 583.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5091544734314084,
      "epoch": 0.00332,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.22338618338108063,
      "kl": 1.1355015095323324,
      "learning_rate": 3.9345730790571886e-05,
      "loss": 0.0118,
      "num_tokens": 7860953.0,
      "reward": 0.4363594055175781,
      "reward_std": 0.7355594635009766,
      "rewards/rollout_reward_func/mean": 0.4363594055175781,
      "rewards/rollout_reward_func/std": 0.7992892265319824,
      "sampling/importance_sampling_ratio/max": 1.017189383506775,
      "sampling/importance_sampling_ratio/mean": 0.9919140934944153,
      "sampling/importance_sampling_ratio/min": 0.943011999130249,
      "sampling/sampling_logp_difference/max": 0.0729970932006836,
      "sampling/sampling_logp_difference/mean": 0.0060599688440561295,
      "step": 332,
      "step_time": 11.963827852001486
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1875.0,
      "completions/max_terminated_length": 1875.0,
      "completions/mean_length": 728.9375,
      "completions/mean_terminated_length": 728.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9395545795559883,
      "epoch": 0.00333,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5399436354637146,
      "kl": 0.8143777102231979,
      "learning_rate": 3.928498872589772e-05,
      "loss": 0.0771,
      "num_tokens": 7888292.0,
      "reward": 0.2570275366306305,
      "reward_std": 0.9382089376449585,
      "rewards/rollout_reward_func/mean": 0.2570275366306305,
      "rewards/rollout_reward_func/std": 0.9083855152130127,
      "sampling/importance_sampling_ratio/max": 1.1160852909088135,
      "sampling/importance_sampling_ratio/mean": 1.0106580257415771,
      "sampling/importance_sampling_ratio/min": 0.8246405124664307,
      "sampling/sampling_logp_difference/max": 0.1846628189086914,
      "sampling/sampling_logp_difference/mean": 0.020361673086881638,
      "step": 333,
      "step_time": 10.649925763001193
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1745.0,
      "completions/max_terminated_length": 1745.0,
      "completions/mean_length": 524.0,
      "completions/mean_terminated_length": 524.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5251712303142995,
      "epoch": 0.00334,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.16349908709526062,
      "kl": 0.47380052134394646,
      "learning_rate": 3.9224114248369163e-05,
      "loss": -0.0106,
      "num_tokens": 7911033.0,
      "reward": 0.6813573837280273,
      "reward_std": 0.3644201457500458,
      "rewards/rollout_reward_func/mean": 0.6813573837280273,
      "rewards/rollout_reward_func/std": 0.5968242287635803,
      "sampling/importance_sampling_ratio/max": 1.1008557081222534,
      "sampling/importance_sampling_ratio/mean": 0.950384259223938,
      "sampling/importance_sampling_ratio/min": 2.911644415967609e-15,
      "sampling/sampling_logp_difference/max": 16.4836368560791,
      "sampling/sampling_logp_difference/mean": 0.5149277448654175,
      "step": 334,
      "step_time": 10.319685454999671
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1643.0,
      "completions/max_terminated_length": 1643.0,
      "completions/mean_length": 432.4375,
      "completions/mean_terminated_length": 432.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.45635355915874243,
      "epoch": 0.00335,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3017308712005615,
      "kl": 0.3677996387705207,
      "learning_rate": 3.916310808351021e-05,
      "loss": -0.0056,
      "num_tokens": 7932410.0,
      "reward": 0.488298237323761,
      "reward_std": 0.7048558592796326,
      "rewards/rollout_reward_func/mean": 0.488298237323761,
      "rewards/rollout_reward_func/std": 0.735072910785675,
      "sampling/importance_sampling_ratio/max": 1.0864568948745728,
      "sampling/importance_sampling_ratio/mean": 1.000978708267212,
      "sampling/importance_sampling_ratio/min": 0.9357432723045349,
      "sampling/sampling_logp_difference/max": 0.06651580333709717,
      "sampling/sampling_logp_difference/mean": 0.0053183650597929955,
      "step": 335,
      "step_time": 9.358613901000354
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1727.0,
      "completions/max_terminated_length": 1727.0,
      "completions/mean_length": 623.6875,
      "completions/mean_terminated_length": 623.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5627524782903492,
      "epoch": 0.00336,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2543277442455292,
      "kl": 0.4787181280553341,
      "learning_rate": 3.9101970958414335e-05,
      "loss": 0.0181,
      "num_tokens": 7957610.0,
      "reward": 0.6020542979240417,
      "reward_std": 0.5260183811187744,
      "rewards/rollout_reward_func/mean": 0.6020542979240417,
      "rewards/rollout_reward_func/std": 0.531453013420105,
      "sampling/importance_sampling_ratio/max": 1.0376124382019043,
      "sampling/importance_sampling_ratio/mean": 0.9800416231155396,
      "sampling/importance_sampling_ratio/min": 0.7365758419036865,
      "sampling/sampling_logp_difference/max": 0.25128746032714844,
      "sampling/sampling_logp_difference/mean": 0.011163630522787571,
      "step": 336,
      "step_time": 10.195600498998829
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1736.0,
      "completions/max_terminated_length": 1736.0,
      "completions/mean_length": 776.125,
      "completions/mean_terminated_length": 776.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6340261362493038,
      "epoch": 0.00337,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2782031297683716,
      "kl": 0.4343600161373615,
      "learning_rate": 3.904070360173587e-05,
      "loss": -0.0122,
      "num_tokens": 7985534.0,
      "reward": 0.7424309253692627,
      "reward_std": 0.5586893558502197,
      "rewards/rollout_reward_func/mean": 0.7424309253692627,
      "rewards/rollout_reward_func/std": 0.5811406373977661,
      "sampling/importance_sampling_ratio/max": 1.1717805862426758,
      "sampling/importance_sampling_ratio/mean": 1.0279302597045898,
      "sampling/importance_sampling_ratio/min": 0.9407347440719604,
      "sampling/sampling_logp_difference/max": 0.14946651458740234,
      "sampling/sampling_logp_difference/mean": 0.011258311569690704,
      "step": 337,
      "step_time": 10.186261139000635
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1682.0,
      "completions/max_terminated_length": 1682.0,
      "completions/mean_length": 522.4375,
      "completions/mean_terminated_length": 522.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3445935854688287,
      "epoch": 0.00338,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4238836467266083,
      "kl": 0.32376451045274734,
      "learning_rate": 3.8979306743681264e-05,
      "loss": 0.0009,
      "num_tokens": 8008158.0,
      "reward": 0.7954655885696411,
      "reward_std": 0.4442320466041565,
      "rewards/rollout_reward_func/mean": 0.7954655885696411,
      "rewards/rollout_reward_func/std": 0.4398350119590759,
      "sampling/importance_sampling_ratio/max": 1.1254851818084717,
      "sampling/importance_sampling_ratio/mean": 1.0015358924865723,
      "sampling/importance_sampling_ratio/min": 0.9134187698364258,
      "sampling/sampling_logp_difference/max": 0.09670019149780273,
      "sampling/sampling_logp_difference/mean": 0.009540950879454613,
      "step": 338,
      "step_time": 9.521580385000561
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1576.0,
      "completions/max_terminated_length": 1576.0,
      "completions/mean_length": 571.1875,
      "completions/mean_terminated_length": 571.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5861630614381284,
      "epoch": 0.00339,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.18055106699466705,
      "kl": 0.27533342130482197,
      "learning_rate": 3.891778111600042e-05,
      "loss": 0.0233,
      "num_tokens": 8032433.0,
      "reward": 0.40421053767204285,
      "reward_std": 0.5702589750289917,
      "rewards/rollout_reward_func/mean": 0.40421053767204285,
      "rewards/rollout_reward_func/std": 0.6520279049873352,
      "sampling/importance_sampling_ratio/max": 1.3837807178497314,
      "sampling/importance_sampling_ratio/mean": 1.0125213861465454,
      "sampling/importance_sampling_ratio/min": 0.8785367608070374,
      "sampling/sampling_logp_difference/max": 0.1217794418334961,
      "sampling/sampling_logp_difference/mean": 0.013998031616210938,
      "step": 339,
      "step_time": 9.636177960999703
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1772.0,
      "completions/max_terminated_length": 1772.0,
      "completions/mean_length": 241.5,
      "completions/mean_terminated_length": 241.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.23956177732907236,
      "epoch": 0.0034,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13965469598770142,
      "kl": 1.1323642786592245,
      "learning_rate": 3.885612745197799e-05,
      "loss": 0.0001,
      "num_tokens": 8049720.0,
      "reward": 0.6682012677192688,
      "reward_std": 0.5810472965240479,
      "rewards/rollout_reward_func/mean": 0.6682012677192688,
      "rewards/rollout_reward_func/std": 0.6226438879966736,
      "sampling/importance_sampling_ratio/max": 1.0625202655792236,
      "sampling/importance_sampling_ratio/mean": 0.9851184487342834,
      "sampling/importance_sampling_ratio/min": 0.8132885098457336,
      "sampling/sampling_logp_difference/max": 0.12370848655700684,
      "sampling/sampling_logp_difference/mean": 0.0097251171246171,
      "step": 340,
      "step_time": 9.276817324999683
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1658.0,
      "completions/max_terminated_length": 1658.0,
      "completions/mean_length": 339.1875,
      "completions/mean_terminated_length": 339.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.42034190497361124,
      "epoch": 0.00341,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3130892217159271,
      "kl": 0.3934767674654722,
      "learning_rate": 3.8794346486424575e-05,
      "loss": -0.0263,
      "num_tokens": 8069340.0,
      "reward": 0.46441781520843506,
      "reward_std": 0.6750683784484863,
      "rewards/rollout_reward_func/mean": 0.46441781520843506,
      "rewards/rollout_reward_func/std": 0.6577343940734863,
      "sampling/importance_sampling_ratio/max": 1.1031142473220825,
      "sampling/importance_sampling_ratio/mean": 0.984606921672821,
      "sampling/importance_sampling_ratio/min": 0.5923222303390503,
      "sampling/sampling_logp_difference/max": 0.2279263138771057,
      "sampling/sampling_logp_difference/mean": 0.017653031274676323,
      "step": 341,
      "step_time": 9.863112756000191
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 808.0,
      "completions/max_terminated_length": 808.0,
      "completions/mean_length": 361.6875,
      "completions/mean_terminated_length": 361.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3990389526588842,
      "epoch": 0.00342,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1822684407234192,
      "kl": 0.4846653360873461,
      "learning_rate": 3.873243895566801e-05,
      "loss": -0.0102,
      "num_tokens": 8089632.0,
      "reward": 0.6567689180374146,
      "reward_std": 0.5397557020187378,
      "rewards/rollout_reward_func/mean": 0.6567689180374146,
      "rewards/rollout_reward_func/std": 0.5260075926780701,
      "sampling/importance_sampling_ratio/max": 1.0806992053985596,
      "sampling/importance_sampling_ratio/mean": 1.0110888481140137,
      "sampling/importance_sampling_ratio/min": 0.9268079996109009,
      "sampling/sampling_logp_difference/max": 0.07773852348327637,
      "sampling/sampling_logp_difference/mean": 0.007030848413705826,
      "step": 342,
      "step_time": 6.871826032000172
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 870.0,
      "completions/max_terminated_length": 870.0,
      "completions/mean_length": 223.25,
      "completions/mean_terminated_length": 223.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.37867807736620307,
      "epoch": 0.00343,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2538404166698456,
      "kl": 0.313096284866333,
      "learning_rate": 3.8670405597544606e-05,
      "loss": 0.0026,
      "num_tokens": 8107240.0,
      "reward": 0.6008927822113037,
      "reward_std": 0.6418352723121643,
      "rewards/rollout_reward_func/mean": 0.6008927822113037,
      "rewards/rollout_reward_func/std": 0.6461029648780823,
      "sampling/importance_sampling_ratio/max": 1.0427806377410889,
      "sampling/importance_sampling_ratio/mean": 0.9905444979667664,
      "sampling/importance_sampling_ratio/min": 0.9192530512809753,
      "sampling/sampling_logp_difference/max": 0.10956335067749023,
      "sampling/sampling_logp_difference/mean": 0.00709434412419796,
      "step": 343,
      "step_time": 6.820053398999335
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2690.0,
      "completions/max_terminated_length": 2690.0,
      "completions/mean_length": 498.1875,
      "completions/mean_terminated_length": 498.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4330964583205059,
      "epoch": 0.00344,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2171831727027893,
      "kl": 1.148713380098343,
      "learning_rate": 3.860824715139028e-05,
      "loss": 0.0044,
      "num_tokens": 8130116.0,
      "reward": 0.39807385206222534,
      "reward_std": 0.6416223049163818,
      "rewards/rollout_reward_func/mean": 0.39807385206222534,
      "rewards/rollout_reward_func/std": 0.6548625826835632,
      "sampling/importance_sampling_ratio/max": 1.0556583404541016,
      "sampling/importance_sampling_ratio/mean": 0.9981417059898376,
      "sampling/importance_sampling_ratio/min": 0.8974504470825195,
      "sampling/sampling_logp_difference/max": 0.12099075317382812,
      "sampling/sampling_logp_difference/mean": 0.011370684020221233,
      "step": 344,
      "step_time": 12.170457171000635
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 835.0,
      "completions/max_terminated_length": 835.0,
      "completions/mean_length": 355.4375,
      "completions/mean_terminated_length": 355.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.325194253702648,
      "epoch": 0.00345,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.056172966957092285,
      "kl": 0.3104660827666521,
      "learning_rate": 3.8545964358031836e-05,
      "loss": -0.0111,
      "num_tokens": 8150803.0,
      "reward": 0.5173795223236084,
      "reward_std": 0.5801167488098145,
      "rewards/rollout_reward_func/mean": 0.5173795223236084,
      "rewards/rollout_reward_func/std": 0.565341055393219,
      "sampling/importance_sampling_ratio/max": 1.0420836210250854,
      "sampling/importance_sampling_ratio/mean": 1.0000230073928833,
      "sampling/importance_sampling_ratio/min": 0.9563314318656921,
      "sampling/sampling_logp_difference/max": 0.04470658302307129,
      "sampling/sampling_logp_difference/mean": 0.003984528593719006,
      "step": 345,
      "step_time": 6.957018660001268
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1835.0,
      "completions/max_terminated_length": 1835.0,
      "completions/mean_length": 299.5625,
      "completions/mean_terminated_length": 299.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.35260581667535007,
      "epoch": 0.00346,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8317896127700806,
      "kl": 12.043093563523144,
      "learning_rate": 3.848355795977808e-05,
      "loss": 0.0141,
      "num_tokens": 8170478.0,
      "reward": 0.34007400274276733,
      "reward_std": 0.7398191690444946,
      "rewards/rollout_reward_func/mean": 0.34007400274276733,
      "rewards/rollout_reward_func/std": 0.7349085211753845,
      "sampling/importance_sampling_ratio/max": 1.0646957159042358,
      "sampling/importance_sampling_ratio/mean": 0.9969290494918823,
      "sampling/importance_sampling_ratio/min": 0.9309963583946228,
      "sampling/sampling_logp_difference/max": 0.06946945190429688,
      "sampling/sampling_logp_difference/mean": 0.00682662520557642,
      "step": 346,
      "step_time": 9.440439715999673
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1675.0,
      "completions/max_terminated_length": 1675.0,
      "completions/mean_length": 236.0,
      "completions/mean_terminated_length": 236.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.27365352492779493,
      "epoch": 0.00347,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9760653376579285,
      "kl": 3.437317419797182,
      "learning_rate": 3.842102870041097e-05,
      "loss": 0.0059,
      "num_tokens": 8187844.0,
      "reward": 0.5295370817184448,
      "reward_std": 0.6723544597625732,
      "rewards/rollout_reward_func/mean": 0.5295370817184448,
      "rewards/rollout_reward_func/std": 0.65923011302948,
      "sampling/importance_sampling_ratio/max": 1.0121904611587524,
      "sampling/importance_sampling_ratio/mean": 0.9963549375534058,
      "sampling/importance_sampling_ratio/min": 0.9188332557678223,
      "sampling/sampling_logp_difference/max": 0.062445640563964844,
      "sampling/sampling_logp_difference/mean": 0.003005451522767544,
      "step": 347,
      "step_time": 9.030807641998763
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1709.0,
      "completions/max_terminated_length": 1709.0,
      "completions/mean_length": 572.1875,
      "completions/mean_terminated_length": 572.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4468996562063694,
      "epoch": 0.00348,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.20247116684913635,
      "kl": 0.8803260647691786,
      "learning_rate": 3.835837732517681e-05,
      "loss": -0.0176,
      "num_tokens": 8212000.0,
      "reward": 0.6653975248336792,
      "reward_std": 0.5877760052680969,
      "rewards/rollout_reward_func/mean": 0.6653975248336792,
      "rewards/rollout_reward_func/std": 0.6302196979522705,
      "sampling/importance_sampling_ratio/max": 1.0502454042434692,
      "sampling/importance_sampling_ratio/mean": 0.9922816753387451,
      "sampling/importance_sampling_ratio/min": 0.9084140062332153,
      "sampling/sampling_logp_difference/max": 0.08576345443725586,
      "sampling/sampling_logp_difference/mean": 0.006662571337074041,
      "step": 348,
      "step_time": 10.168187559001126
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1773.0,
      "completions/max_terminated_length": 1773.0,
      "completions/mean_length": 611.1875,
      "completions/mean_terminated_length": 611.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.38527590152807534,
      "epoch": 0.00349,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.36860257387161255,
      "kl": 0.5151476077735424,
      "learning_rate": 3.8295604580777265e-05,
      "loss": 0.0208,
      "num_tokens": 8236671.0,
      "reward": 0.492379754781723,
      "reward_std": 0.7449498176574707,
      "rewards/rollout_reward_func/mean": 0.492379754781723,
      "rewards/rollout_reward_func/std": 0.7197232246398926,
      "sampling/importance_sampling_ratio/max": 1.132118582725525,
      "sampling/importance_sampling_ratio/mean": 0.991021454334259,
      "sampling/importance_sampling_ratio/min": 0.849826455116272,
      "sampling/sampling_logp_difference/max": 0.11880040168762207,
      "sampling/sampling_logp_difference/mean": 0.014037315733730793,
      "step": 349,
      "step_time": 9.863825682999959
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1782.0,
      "completions/max_terminated_length": 1782.0,
      "completions/mean_length": 536.1875,
      "completions/mean_terminated_length": 536.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.46153751760721207,
      "epoch": 0.0035,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2063780426979065,
      "kl": 0.41183730214834213,
      "learning_rate": 3.8232711215360574e-05,
      "loss": -0.0033,
      "num_tokens": 8260305.0,
      "reward": 0.7467820048332214,
      "reward_std": 0.588252067565918,
      "rewards/rollout_reward_func/mean": 0.7467820048332214,
      "rewards/rollout_reward_func/std": 0.5767629742622375,
      "sampling/importance_sampling_ratio/max": 1.0796579122543335,
      "sampling/importance_sampling_ratio/mean": 0.9991644620895386,
      "sampling/importance_sampling_ratio/min": 0.922033965587616,
      "sampling/sampling_logp_difference/max": 0.07531356811523438,
      "sampling/sampling_logp_difference/mean": 0.009347020648419857,
      "step": 350,
      "step_time": 9.89848823200009
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 844.0,
      "completions/max_terminated_length": 844.0,
      "completions/mean_length": 511.5625,
      "completions/mean_terminated_length": 511.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4041527220979333,
      "epoch": 0.00351,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0706329420208931,
      "kl": 0.31291118264198303,
      "learning_rate": 3.8169697978512545e-05,
      "loss": -0.0025,
      "num_tokens": 8283689.0,
      "reward": 0.5419058203697205,
      "reward_std": 0.6596132516860962,
      "rewards/rollout_reward_func/mean": 0.5419058203697205,
      "rewards/rollout_reward_func/std": 0.6466596126556396,
      "sampling/importance_sampling_ratio/max": 1.0791584253311157,
      "sampling/importance_sampling_ratio/mean": 0.9986833333969116,
      "sampling/importance_sampling_ratio/min": 0.9116337299346924,
      "sampling/sampling_logp_difference/max": 0.09734749794006348,
      "sampling/sampling_logp_difference/mean": 0.012124604545533657,
      "step": 351,
      "step_time": 7.255577456001447
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2612.0,
      "completions/max_terminated_length": 2612.0,
      "completions/mean_length": 625.6875,
      "completions/mean_terminated_length": 625.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.46896477160044014,
      "epoch": 0.00352,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.30630266666412354,
      "kl": 0.5867375270463526,
      "learning_rate": 3.810656562124768e-05,
      "loss": -0.0205,
      "num_tokens": 8308751.0,
      "reward": 0.4186413884162903,
      "reward_std": 0.845854640007019,
      "rewards/rollout_reward_func/mean": 0.4186413884162903,
      "rewards/rollout_reward_func/std": 0.8227803707122803,
      "sampling/importance_sampling_ratio/max": 1.1206138134002686,
      "sampling/importance_sampling_ratio/mean": 0.9969134330749512,
      "sampling/importance_sampling_ratio/min": 0.8946788907051086,
      "sampling/sampling_logp_difference/max": 0.11336421966552734,
      "sampling/sampling_logp_difference/mean": 0.009463435970246792,
      "step": 352,
      "step_time": 12.062813318000735
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1674.0,
      "completions/max_terminated_length": 1674.0,
      "completions/mean_length": 385.75,
      "completions/mean_terminated_length": 385.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.42383645637892187,
      "epoch": 0.00353,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.31108808517456055,
      "kl": 0.8242284171283245,
      "learning_rate": 3.8043314896000193e-05,
      "loss": -0.0064,
      "num_tokens": 8329198.0,
      "reward": 0.4227539896965027,
      "reward_std": 0.841111421585083,
      "rewards/rollout_reward_func/mean": 0.4227539896965027,
      "rewards/rollout_reward_func/std": 0.818843424320221,
      "sampling/importance_sampling_ratio/max": 1.1384921073913574,
      "sampling/importance_sampling_ratio/mean": 1.0104315280914307,
      "sampling/importance_sampling_ratio/min": 0.939271092414856,
      "sampling/sampling_logp_difference/max": 0.12975692749023438,
      "sampling/sampling_logp_difference/mean": 0.01390169095247984,
      "step": 353,
      "step_time": 9.262979028998416
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2523.0,
      "completions/max_terminated_length": 2523.0,
      "completions/mean_length": 881.5,
      "completions/mean_terminated_length": 881.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5669943019747734,
      "epoch": 0.00354,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4063970148563385,
      "kl": 0.32521184254437685,
      "learning_rate": 3.797994655661504e-05,
      "loss": -0.011,
      "num_tokens": 8358712.0,
      "reward": 0.7965946197509766,
      "reward_std": 0.4449338912963867,
      "rewards/rollout_reward_func/mean": 0.7965946197509766,
      "rewards/rollout_reward_func/std": 0.43745940923690796,
      "sampling/importance_sampling_ratio/max": 1.182422399520874,
      "sampling/importance_sampling_ratio/mean": 1.0172944068908691,
      "sampling/importance_sampling_ratio/min": 0.8590685129165649,
      "sampling/sampling_logp_difference/max": 0.15608274936676025,
      "sampling/sampling_logp_difference/mean": 0.017089663073420525,
      "step": 354,
      "step_time": 12.544767278999643
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3377.0,
      "completions/max_terminated_length": 3377.0,
      "completions/mean_length": 629.5,
      "completions/mean_terminated_length": 629.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4821637123823166,
      "epoch": 0.00355,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.16618916392326355,
      "kl": 0.3410778194665909,
      "learning_rate": 3.791646135833897e-05,
      "loss": 0.003,
      "num_tokens": 8383658.0,
      "reward": 0.2503877580165863,
      "reward_std": 0.7595770359039307,
      "rewards/rollout_reward_func/mean": 0.2503877580165863,
      "rewards/rollout_reward_func/std": 0.8297961354255676,
      "sampling/importance_sampling_ratio/max": 1.1179763078689575,
      "sampling/importance_sampling_ratio/mean": 1.0026963949203491,
      "sampling/importance_sampling_ratio/min": 0.9328464269638062,
      "sampling/sampling_logp_difference/max": 0.1098167896270752,
      "sampling/sampling_logp_difference/mean": 0.011368000879883766,
      "step": 355,
      "step_time": 15.130114087000948
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1763.0,
      "completions/max_terminated_length": 1763.0,
      "completions/mean_length": 961.125,
      "completions/mean_terminated_length": 961.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.624457348138094,
      "epoch": 0.00356,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.20255939662456512,
      "kl": 0.47820152528584003,
      "learning_rate": 3.785286005781148e-05,
      "loss": 0.0174,
      "num_tokens": 8414078.0,
      "reward": 0.5100035667419434,
      "reward_std": 0.8059042096138,
      "rewards/rollout_reward_func/mean": 0.5100035667419434,
      "rewards/rollout_reward_func/std": 0.7926029562950134,
      "sampling/importance_sampling_ratio/max": 1.136832594871521,
      "sampling/importance_sampling_ratio/mean": 0.9944028258323669,
      "sampling/importance_sampling_ratio/min": 0.8708053827285767,
      "sampling/sampling_logp_difference/max": 0.11561822891235352,
      "sampling/sampling_logp_difference/mean": 0.01727217622101307,
      "step": 356,
      "step_time": 10.552969280999605
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2405.0,
      "completions/max_terminated_length": 2405.0,
      "completions/mean_length": 768.875,
      "completions/mean_terminated_length": 768.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6166292075067759,
      "epoch": 0.00357,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.29325777292251587,
      "kl": 0.2899240078404546,
      "learning_rate": 3.7789143413055804e-05,
      "loss": -0.0139,
      "num_tokens": 8441681.0,
      "reward": 0.24859139323234558,
      "reward_std": 0.818993330001831,
      "rewards/rollout_reward_func/mean": 0.24859139323234558,
      "rewards/rollout_reward_func/std": 0.8291009664535522,
      "sampling/importance_sampling_ratio/max": 1.246134877204895,
      "sampling/importance_sampling_ratio/mean": 1.0403032302856445,
      "sampling/importance_sampling_ratio/min": 0.8706207275390625,
      "sampling/sampling_logp_difference/max": 0.15140748023986816,
      "sampling/sampling_logp_difference/mean": 0.01822720468044281,
      "step": 357,
      "step_time": 11.924545535998732
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1673.0,
      "completions/max_terminated_length": 1673.0,
      "completions/mean_length": 516.9375,
      "completions/mean_terminated_length": 516.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5698439106345177,
      "epoch": 0.00358,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21289221942424774,
      "kl": 0.569066233932972,
      "learning_rate": 3.7725312183469914e-05,
      "loss": -0.021,
      "num_tokens": 8464413.0,
      "reward": 0.5663602352142334,
      "reward_std": 0.8169868588447571,
      "rewards/rollout_reward_func/mean": 0.5663602352142334,
      "rewards/rollout_reward_func/std": 0.7974401116371155,
      "sampling/importance_sampling_ratio/max": 1.1575640439987183,
      "sampling/importance_sampling_ratio/mean": 0.95854651927948,
      "sampling/importance_sampling_ratio/min": 1.7483353431657633e-08,
      "sampling/sampling_logp_difference/max": 17.746641159057617,
      "sampling/sampling_logp_difference/mean": 0.33351510763168335,
      "step": 358,
      "step_time": 9.579998625998996
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3273.0,
      "completions/max_terminated_length": 3273.0,
      "completions/mean_length": 901.0625,
      "completions/mean_terminated_length": 901.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5132092339918017,
      "epoch": 0.00359,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.235223650932312,
      "kl": 0.4439892880618572,
      "learning_rate": 3.766136712981744e-05,
      "loss": 0.0273,
      "num_tokens": 8493474.0,
      "reward": 0.5295015573501587,
      "reward_std": 0.871507465839386,
      "rewards/rollout_reward_func/mean": 0.5295015573501587,
      "rewards/rollout_reward_func/std": 0.8419718742370605,
      "sampling/importance_sampling_ratio/max": 1.098468542098999,
      "sampling/importance_sampling_ratio/mean": 1.0106658935546875,
      "sampling/importance_sampling_ratio/min": 0.9072843790054321,
      "sampling/sampling_logp_difference/max": 0.10634946823120117,
      "sampling/sampling_logp_difference/mean": 0.014443673193454742,
      "step": 359,
      "step_time": 14.463453913000194
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1709.0,
      "completions/max_terminated_length": 1709.0,
      "completions/mean_length": 422.1875,
      "completions/mean_terminated_length": 422.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6521322415210307,
      "epoch": 0.0036,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2522599697113037,
      "kl": 0.4005614686757326,
      "learning_rate": 3.75973090142186e-05,
      "loss": 0.0086,
      "num_tokens": 8514558.0,
      "reward": 0.4469483196735382,
      "reward_std": 0.8956449627876282,
      "rewards/rollout_reward_func/mean": 0.4469483196735382,
      "rewards/rollout_reward_func/std": 0.8678773045539856,
      "sampling/importance_sampling_ratio/max": 1.0640645027160645,
      "sampling/importance_sampling_ratio/mean": 0.9895248413085938,
      "sampling/importance_sampling_ratio/min": 0.8274592757225037,
      "sampling/sampling_logp_difference/max": 0.16183018684387207,
      "sampling/sampling_logp_difference/mean": 0.016488218680024147,
      "step": 360,
      "step_time": 9.321199838997927
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1925.0,
      "completions/max_terminated_length": 1925.0,
      "completions/mean_length": 786.8125,
      "completions/mean_terminated_length": 786.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5899844216182828,
      "epoch": 0.00361,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3910177946090698,
      "kl": 0.4043824076652527,
      "learning_rate": 3.753313860014113e-05,
      "loss": 0.014,
      "num_tokens": 8542054.0,
      "reward": 0.21909713745117188,
      "reward_std": 0.8624383807182312,
      "rewards/rollout_reward_func/mean": 0.21909713745117188,
      "rewards/rollout_reward_func/std": 0.9343637824058533,
      "sampling/importance_sampling_ratio/max": 1.2449153661727905,
      "sampling/importance_sampling_ratio/mean": 1.0049563646316528,
      "sampling/importance_sampling_ratio/min": 0.8756469488143921,
      "sampling/sampling_logp_difference/max": 0.2084672451019287,
      "sampling/sampling_logp_difference/mean": 0.016720358282327652,
      "step": 361,
      "step_time": 10.660532955001145
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2403.0,
      "completions/max_terminated_length": 2403.0,
      "completions/mean_length": 749.8125,
      "completions/mean_terminated_length": 749.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6013033464550972,
      "epoch": 0.00362,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.15779803693294525,
      "kl": 0.7114616446197033,
      "learning_rate": 3.7468856652391175e-05,
      "loss": 0.0018,
      "num_tokens": 8569312.0,
      "reward": 0.25072744488716125,
      "reward_std": 0.8506348133087158,
      "rewards/rollout_reward_func/mean": 0.25072744488716125,
      "rewards/rollout_reward_func/std": 0.8347976207733154,
      "sampling/importance_sampling_ratio/max": 1.0793864727020264,
      "sampling/importance_sampling_ratio/mean": 0.9797043204307556,
      "sampling/importance_sampling_ratio/min": 0.8780698776245117,
      "sampling/sampling_logp_difference/max": 0.12278604507446289,
      "sampling/sampling_logp_difference/mean": 0.0138132618740201,
      "step": 362,
      "step_time": 12.397467146000054
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3271.0,
      "completions/max_terminated_length": 3271.0,
      "completions/mean_length": 701.5,
      "completions/mean_terminated_length": 701.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6253841575235128,
      "epoch": 0.00363,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.23277893662452698,
      "kl": 0.4302115626633167,
      "learning_rate": 3.74044639371042e-05,
      "loss": -0.0023,
      "num_tokens": 8595329.0,
      "reward": 0.37593621015548706,
      "reward_std": 0.8889084458351135,
      "rewards/rollout_reward_func/mean": 0.37593621015548706,
      "rewards/rollout_reward_func/std": 0.8759858012199402,
      "sampling/importance_sampling_ratio/max": 1.1045668125152588,
      "sampling/importance_sampling_ratio/mean": 1.0022172927856445,
      "sampling/importance_sampling_ratio/min": 0.8140279054641724,
      "sampling/sampling_logp_difference/max": 0.20800018310546875,
      "sampling/sampling_logp_difference/mean": 0.016864268109202385,
      "step": 363,
      "step_time": 13.99654722899777
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 3288.0,
      "completions/max_terminated_length": 3288.0,
      "completions/mean_length": 669.1875,
      "completions/mean_terminated_length": 669.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5219712257385254,
      "epoch": 0.00364,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.35901305079460144,
      "kl": 0.9232600256800652,
      "learning_rate": 3.7339961221735795e-05,
      "loss": 0.0243,
      "num_tokens": 8620811.0,
      "reward": 0.4561062455177307,
      "reward_std": 0.8785320520401001,
      "rewards/rollout_reward_func/mean": 0.4561062455177307,
      "rewards/rollout_reward_func/std": 0.8530017137527466,
      "sampling/importance_sampling_ratio/max": 1.069465160369873,
      "sampling/importance_sampling_ratio/mean": 0.9810104370117188,
      "sampling/importance_sampling_ratio/min": 0.8408908843994141,
      "sampling/sampling_logp_difference/max": 0.1416926383972168,
      "sampling/sampling_logp_difference/mean": 0.018831513822078705,
      "step": 364,
      "step_time": 14.22964037300153
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2536.0,
      "completions/max_terminated_length": 2536.0,
      "completions/mean_length": 427.0,
      "completions/mean_terminated_length": 427.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.38412028597667813,
      "epoch": 0.00365,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06580790877342224,
      "kl": 0.3488181158900261,
      "learning_rate": 3.727534927505261e-05,
      "loss": -0.005,
      "num_tokens": 8641616.0,
      "reward": 0.7926955223083496,
      "reward_std": 0.45219358801841736,
      "rewards/rollout_reward_func/mean": 0.7926955223083496,
      "rewards/rollout_reward_func/std": 0.4457743167877197,
      "sampling/importance_sampling_ratio/max": 1.1312230825424194,
      "sampling/importance_sampling_ratio/mean": 1.0149493217468262,
      "sampling/importance_sampling_ratio/min": 0.908879280090332,
      "sampling/sampling_logp_difference/max": 0.13949191570281982,
      "sampling/sampling_logp_difference/mean": 0.012770065106451511,
      "step": 365,
      "step_time": 11.83208016699973
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1566.0,
      "completions/max_terminated_length": 1566.0,
      "completions/mean_length": 541.3125,
      "completions/mean_terminated_length": 541.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3991567865014076,
      "epoch": 0.00366,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12804849445819855,
      "kl": 0.5032951068133116,
      "learning_rate": 3.721062886712314e-05,
      "loss": 0.0108,
      "num_tokens": 8664566.0,
      "reward": 0.3646901547908783,
      "reward_std": 0.8147523999214172,
      "rewards/rollout_reward_func/mean": 0.3646901547908783,
      "rewards/rollout_reward_func/std": 0.798312246799469,
      "sampling/importance_sampling_ratio/max": 1.1295384168624878,
      "sampling/importance_sampling_ratio/mean": 0.9908213019371033,
      "sampling/importance_sampling_ratio/min": 0.9056549072265625,
      "sampling/sampling_logp_difference/max": 0.09414207935333252,
      "sampling/sampling_logp_difference/mean": 0.012713532894849777,
      "step": 366,
      "step_time": 9.347810321999532
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 862.0,
      "completions/max_terminated_length": 862.0,
      "completions/mean_length": 364.875,
      "completions/mean_terminated_length": 364.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3177734110504389,
      "epoch": 0.00367,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.14533373713493347,
      "kl": 0.5411685416474938,
      "learning_rate": 3.714580076930856e-05,
      "loss": -0.0122,
      "num_tokens": 8685341.0,
      "reward": 0.4809131622314453,
      "reward_std": 0.7508883476257324,
      "rewards/rollout_reward_func/mean": 0.4809131622314453,
      "rewards/rollout_reward_func/std": 0.7387793064117432,
      "sampling/importance_sampling_ratio/max": 1.1265138387680054,
      "sampling/importance_sampling_ratio/mean": 1.0037517547607422,
      "sampling/importance_sampling_ratio/min": 0.9100947976112366,
      "sampling/sampling_logp_difference/max": 0.09987318515777588,
      "sampling/sampling_logp_difference/mean": 0.009353925473988056,
      "step": 367,
      "step_time": 6.960064205999515
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1816.0,
      "completions/max_terminated_length": 1816.0,
      "completions/mean_length": 546.625,
      "completions/mean_terminated_length": 546.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.35974299162626266,
      "epoch": 0.00368,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.227874755859375,
      "kl": 0.39015507884323597,
      "learning_rate": 3.708086575425352e-05,
      "loss": 0.0022,
      "num_tokens": 8708740.0,
      "reward": 0.5960956811904907,
      "reward_std": 0.538443386554718,
      "rewards/rollout_reward_func/mean": 0.5960956811904907,
      "rewards/rollout_reward_func/std": 0.53940749168396,
      "sampling/importance_sampling_ratio/max": 1.1687045097351074,
      "sampling/importance_sampling_ratio/mean": 1.011523962020874,
      "sampling/importance_sampling_ratio/min": 0.910703718662262,
      "sampling/sampling_logp_difference/max": 0.12650370597839355,
      "sampling/sampling_logp_difference/mean": 0.010520839132368565,
      "step": 368,
      "step_time": 9.872557447000872
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 871.0,
      "completions/max_terminated_length": 871.0,
      "completions/mean_length": 433.25,
      "completions/mean_terminated_length": 410.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.562132528051734,
      "epoch": 0.00369,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4127052426338196,
      "kl": 1.0499663054943085,
      "learning_rate": 3.701582459587698e-05,
      "loss": -0.0874,
      "num_tokens": 8730892.0,
      "reward": 0.4919072687625885,
      "reward_std": 0.8306519985198975,
      "rewards/rollout_reward_func/mean": 0.4919072687625885,
      "rewards/rollout_reward_func/std": 0.8175029754638672,
      "sampling/importance_sampling_ratio/max": 1.0433714389801025,
      "sampling/importance_sampling_ratio/mean": 0.9944390058517456,
      "sampling/importance_sampling_ratio/min": 0.8522179126739502,
      "sampling/sampling_logp_difference/max": 0.1644735336303711,
      "sampling/sampling_logp_difference/mean": 0.026726102456450462,
      "step": 369,
      "step_time": 7.622723090000363
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1806.0,
      "completions/max_terminated_length": 1806.0,
      "completions/mean_length": 481.875,
      "completions/mean_terminated_length": 481.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.268276983872056,
      "epoch": 0.0037,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.20278573036193848,
      "kl": 1.385072972625494,
      "learning_rate": 3.6950678069362916e-05,
      "loss": 0.0052,
      "num_tokens": 8753224.0,
      "reward": 0.729630708694458,
      "reward_std": 0.47146958112716675,
      "rewards/rollout_reward_func/mean": 0.729630708694458,
      "rewards/rollout_reward_func/std": 0.4842284321784973,
      "sampling/importance_sampling_ratio/max": 1.0281199216842651,
      "sampling/importance_sampling_ratio/mean": 1.0004172325134277,
      "sampling/importance_sampling_ratio/min": 0.9256728887557983,
      "sampling/sampling_logp_difference/max": 0.0817556381225586,
      "sampling/sampling_logp_difference/mean": 0.00638881279155612,
      "step": 370,
      "step_time": 10.213053719000527
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1718.0,
      "completions/max_terminated_length": 1718.0,
      "completions/mean_length": 492.125,
      "completions/mean_terminated_length": 492.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.27341209538280964,
      "epoch": 0.00371,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.15940643846988678,
      "kl": 0.6964212097227573,
      "learning_rate": 3.688542695115114e-05,
      "loss": -0.0055,
      "num_tokens": 8775959.0,
      "reward": 0.5368804931640625,
      "reward_std": 0.6635482311248779,
      "rewards/rollout_reward_func/mean": 0.5368804931640625,
      "rewards/rollout_reward_func/std": 0.6517050266265869,
      "sampling/importance_sampling_ratio/max": 1.1356431245803833,
      "sampling/importance_sampling_ratio/mean": 1.0046961307525635,
      "sampling/importance_sampling_ratio/min": 0.9162760972976685,
      "sampling/sampling_logp_difference/max": 0.12364685535430908,
      "sampling/sampling_logp_difference/mean": 0.0057099745608866215,
      "step": 371,
      "step_time": 9.81269704799979
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 879.0,
      "completions/max_terminated_length": 879.0,
      "completions/mean_length": 381.25,
      "completions/mean_terminated_length": 381.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.22589009907096624,
      "epoch": 0.00372,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21354655921459198,
      "kl": 0.8987773582339287,
      "learning_rate": 3.6820072018928036e-05,
      "loss": 0.0012,
      "num_tokens": 8796356.0,
      "reward": 0.4527733325958252,
      "reward_std": 0.5669265985488892,
      "rewards/rollout_reward_func/mean": 0.4527733325958252,
      "rewards/rollout_reward_func/std": 0.56532883644104,
      "sampling/importance_sampling_ratio/max": 1.0440309047698975,
      "sampling/importance_sampling_ratio/mean": 0.9872015714645386,
      "sampling/importance_sampling_ratio/min": 0.7980369329452515,
      "sampling/sampling_logp_difference/max": 0.22727203369140625,
      "sampling/sampling_logp_difference/mean": 0.010273849591612816,
      "step": 372,
      "step_time": 7.0968246789998375
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1716.0,
      "completions/max_terminated_length": 1716.0,
      "completions/mean_length": 575.125,
      "completions/mean_terminated_length": 575.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.24408502783626318,
      "epoch": 0.00373,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.24906522035598755,
      "kl": 0.5017283204942942,
      "learning_rate": 3.6754614051617246e-05,
      "loss": 0.0021,
      "num_tokens": 8820356.0,
      "reward": 0.6631232500076294,
      "reward_std": 0.530144214630127,
      "rewards/rollout_reward_func/mean": 0.6631232500076294,
      "rewards/rollout_reward_func/std": 0.5164305567741394,
      "sampling/importance_sampling_ratio/max": 1.1830371618270874,
      "sampling/importance_sampling_ratio/mean": 1.0130929946899414,
      "sampling/importance_sampling_ratio/min": 0.9265527725219727,
      "sampling/sampling_logp_difference/max": 0.17003631591796875,
      "sampling/sampling_logp_difference/mean": 0.008413991890847683,
      "step": 373,
      "step_time": 9.601571333998436
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 815.0,
      "completions/max_terminated_length": 815.0,
      "completions/mean_length": 328.875,
      "completions/mean_terminated_length": 328.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3136111441999674,
      "epoch": 0.00374,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.14787882566452026,
      "kl": 0.44697335362434387,
      "learning_rate": 3.668905382937045e-05,
      "loss": -0.0086,
      "num_tokens": 8839942.0,
      "reward": 0.46706533432006836,
      "reward_std": 0.6624480485916138,
      "rewards/rollout_reward_func/mean": 0.46706533432006836,
      "rewards/rollout_reward_func/std": 0.6569041013717651,
      "sampling/importance_sampling_ratio/max": 1.1144071817398071,
      "sampling/importance_sampling_ratio/mean": 1.0050318241119385,
      "sampling/importance_sampling_ratio/min": 0.9340786337852478,
      "sampling/sampling_logp_difference/max": 0.10806220769882202,
      "sampling/sampling_logp_difference/mean": 0.006155623123049736,
      "step": 374,
      "step_time": 6.879328479000833
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1773.0,
      "completions/max_terminated_length": 1773.0,
      "completions/mean_length": 539.625,
      "completions/mean_terminated_length": 539.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.28153151739388704,
      "epoch": 0.00375,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2715558111667633,
      "kl": 0.4816452022641897,
      "learning_rate": 3.662339213355806e-05,
      "loss": 0.014,
      "num_tokens": 8862743.0,
      "reward": 0.6644850373268127,
      "reward_std": 0.5269597172737122,
      "rewards/rollout_reward_func/mean": 0.6644850373268127,
      "rewards/rollout_reward_func/std": 0.5142312049865723,
      "sampling/importance_sampling_ratio/max": 1.1062045097351074,
      "sampling/importance_sampling_ratio/mean": 1.0175925493240356,
      "sampling/importance_sampling_ratio/min": 0.9979625940322876,
      "sampling/sampling_logp_difference/max": 0.06749272346496582,
      "sampling/sampling_logp_difference/mean": 0.006006450392305851,
      "step": 375,
      "step_time": 9.756757150002159
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1752.0,
      "completions/max_terminated_length": 1752.0,
      "completions/mean_length": 485.1875,
      "completions/mean_terminated_length": 485.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.25053481198847294,
      "epoch": 0.00376,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3005993664264679,
      "kl": 1.2012933809310198,
      "learning_rate": 3.6557629746759815e-05,
      "loss": 0.0293,
      "num_tokens": 8884358.0,
      "reward": 0.4948984980583191,
      "reward_std": 0.7515295743942261,
      "rewards/rollout_reward_func/mean": 0.4948984980583191,
      "rewards/rollout_reward_func/std": 0.7260562777519226,
      "sampling/importance_sampling_ratio/max": 1.0792665481567383,
      "sampling/importance_sampling_ratio/mean": 0.9926448464393616,
      "sampling/importance_sampling_ratio/min": 0.895857572555542,
      "sampling/sampling_logp_difference/max": 0.11014366149902344,
      "sampling/sampling_logp_difference/mean": 0.008771705441176891,
      "step": 376,
      "step_time": 10.098384390999854
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2727.0,
      "completions/max_terminated_length": 2727.0,
      "completions/mean_length": 821.875,
      "completions/mean_terminated_length": 821.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5207322146743536,
      "epoch": 0.00377,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.24528230726718903,
      "kl": 0.606095053255558,
      "learning_rate": 3.64917674527556e-05,
      "loss": -0.0217,
      "num_tokens": 8912355.0,
      "reward": 0.29217469692230225,
      "reward_std": 0.7399823069572449,
      "rewards/rollout_reward_func/mean": 0.29217469692230225,
      "rewards/rollout_reward_func/std": 0.7970102429389954,
      "sampling/importance_sampling_ratio/max": 1.132250428199768,
      "sampling/importance_sampling_ratio/mean": 1.0198782682418823,
      "sampling/importance_sampling_ratio/min": 0.9645786881446838,
      "sampling/sampling_logp_difference/max": 0.1094781756401062,
      "sampling/sampling_logp_difference/mean": 0.008929422125220299,
      "step": 377,
      "step_time": 12.791664909999781
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1540.0,
      "completions/max_terminated_length": 1540.0,
      "completions/mean_length": 557.125,
      "completions/mean_terminated_length": 544.4000244140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5058183632791042,
      "epoch": 0.00378,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.10296154022216797,
      "kl": 0.39771849289536476,
      "learning_rate": 3.6425806036516e-05,
      "loss": 0.0022,
      "num_tokens": 8935922.0,
      "reward": 0.7993063926696777,
      "reward_std": 0.4361022114753723,
      "rewards/rollout_reward_func/mean": 0.7993063926696777,
      "rewards/rollout_reward_func/std": 0.4315054714679718,
      "sampling/importance_sampling_ratio/max": 1.0739967823028564,
      "sampling/importance_sampling_ratio/mean": 0.9724860191345215,
      "sampling/importance_sampling_ratio/min": 0.48311272263526917,
      "sampling/sampling_logp_difference/max": 0.1417534351348877,
      "sampling/sampling_logp_difference/mean": 0.014280815608799458,
      "step": 378,
      "step_time": 9.331539392000195
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 871.0,
      "completions/max_terminated_length": 871.0,
      "completions/mean_length": 612.875,
      "completions/mean_terminated_length": 612.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.24877460021525621,
      "epoch": 0.00379,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.29654955863952637,
      "kl": 0.6071595698595047,
      "learning_rate": 3.635974628419295e-05,
      "loss": 0.0049,
      "num_tokens": 8961045.0,
      "reward": 0.5550836324691772,
      "reward_std": 0.7498250603675842,
      "rewards/rollout_reward_func/mean": 0.5550836324691772,
      "rewards/rollout_reward_func/std": 0.7272385954856873,
      "sampling/importance_sampling_ratio/max": 1.026349663734436,
      "sampling/importance_sampling_ratio/mean": 0.9961780905723572,
      "sampling/importance_sampling_ratio/min": 0.9003207087516785,
      "sampling/sampling_logp_difference/max": 0.11169910430908203,
      "sampling/sampling_logp_difference/mean": 0.005749145522713661,
      "step": 379,
      "step_time": 7.503084576998845
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 808.0,
      "completions/max_terminated_length": 808.0,
      "completions/mean_length": 491.6875,
      "completions/mean_terminated_length": 491.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2346731536090374,
      "epoch": 0.0038,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3963688313961029,
      "kl": 0.4349590390920639,
      "learning_rate": 3.629358898311044e-05,
      "loss": 0.0079,
      "num_tokens": 8983273.0,
      "reward": 0.7484447360038757,
      "reward_std": 0.5835862159729004,
      "rewards/rollout_reward_func/mean": 0.7484447360038757,
      "rewards/rollout_reward_func/std": 0.5717742443084717,
      "sampling/importance_sampling_ratio/max": 1.1115286350250244,
      "sampling/importance_sampling_ratio/mean": 0.9956996440887451,
      "sampling/importance_sampling_ratio/min": 0.8692680597305298,
      "sampling/sampling_logp_difference/max": 0.11095261573791504,
      "sampling/sampling_logp_difference/mean": 0.008151913993060589,
      "step": 380,
      "step_time": 7.094439366000188
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1791.0,
      "completions/max_terminated_length": 1791.0,
      "completions/mean_length": 814.4375,
      "completions/mean_terminated_length": 814.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4494324401021004,
      "epoch": 0.00381,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3128831386566162,
      "kl": 0.5440673157572746,
      "learning_rate": 3.622733492175504e-05,
      "loss": -0.0023,
      "num_tokens": 9011264.0,
      "reward": 0.6276761889457703,
      "reward_std": 0.7375520467758179,
      "rewards/rollout_reward_func/mean": 0.6276761889457703,
      "rewards/rollout_reward_func/std": 0.7125565409660339,
      "sampling/importance_sampling_ratio/max": 1.0460220575332642,
      "sampling/importance_sampling_ratio/mean": 0.9910693764686584,
      "sampling/importance_sampling_ratio/min": 0.9254970550537109,
      "sampling/sampling_logp_difference/max": 0.1275930404663086,
      "sampling/sampling_logp_difference/mean": 0.011108539998531342,
      "step": 381,
      "step_time": 10.363134981000258
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1824.0,
      "completions/max_terminated_length": 1824.0,
      "completions/mean_length": 429.0625,
      "completions/mean_terminated_length": 429.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7603917401283979,
      "epoch": 0.00382,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5955827832221985,
      "kl": 0.8186183869838715,
      "learning_rate": 3.616098488976657e-05,
      "loss": -0.0447,
      "num_tokens": 9032499.0,
      "reward": 0.360960990190506,
      "reward_std": 0.7992439270019531,
      "rewards/rollout_reward_func/mean": 0.360960990190506,
      "rewards/rollout_reward_func/std": 0.8108873963356018,
      "sampling/importance_sampling_ratio/max": 1.1063594818115234,
      "sampling/importance_sampling_ratio/mean": 0.9847089052200317,
      "sampling/importance_sampling_ratio/min": 0.6353104114532471,
      "sampling/sampling_logp_difference/max": 0.15542888641357422,
      "sampling/sampling_logp_difference/mean": 0.016072586178779602,
      "step": 382,
      "step_time": 9.884147056000074
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2844.0,
      "completions/max_terminated_length": 2844.0,
      "completions/mean_length": 638.125,
      "completions/mean_terminated_length": 638.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.36403103079646826,
      "epoch": 0.00383,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.17344717681407928,
      "kl": 0.6107500903308392,
      "learning_rate": 3.6094539677928645e-05,
      "loss": 0.0257,
      "num_tokens": 9056670.0,
      "reward": 0.38014310598373413,
      "reward_std": 0.8048261404037476,
      "rewards/rollout_reward_func/mean": 0.38014310598373413,
      "rewards/rollout_reward_func/std": 0.7797057628631592,
      "sampling/importance_sampling_ratio/max": 1.0490210056304932,
      "sampling/importance_sampling_ratio/mean": 1.000303864479065,
      "sampling/importance_sampling_ratio/min": 0.9264886379241943,
      "sampling/sampling_logp_difference/max": 0.08715319633483887,
      "sampling/sampling_logp_difference/mean": 0.008129633031785488,
      "step": 383,
      "step_time": 12.733928850999291
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1594.0,
      "completions/max_terminated_length": 1594.0,
      "completions/mean_length": 445.3125,
      "completions/mean_terminated_length": 445.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.24045330937951803,
      "epoch": 0.00384,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.11605197191238403,
      "kl": 0.5935199540108442,
      "learning_rate": 3.602800007815927e-05,
      "loss": 0.0117,
      "num_tokens": 9077908.0,
      "reward": 0.530332088470459,
      "reward_std": 0.5639506578445435,
      "rewards/rollout_reward_func/mean": 0.530332088470459,
      "rewards/rollout_reward_func/std": 0.5504144430160522,
      "sampling/importance_sampling_ratio/max": 1.1151446104049683,
      "sampling/importance_sampling_ratio/mean": 1.0092833042144775,
      "sampling/importance_sampling_ratio/min": 0.9607274532318115,
      "sampling/sampling_logp_difference/max": 0.10705259442329407,
      "sampling/sampling_logp_difference/mean": 0.005097415763884783,
      "step": 384,
      "step_time": 9.627501446998394
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1585.0,
      "completions/max_terminated_length": 1585.0,
      "completions/mean_length": 388.4375,
      "completions/mean_terminated_length": 388.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3608475364744663,
      "epoch": 0.00385,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.22793187201023102,
      "kl": 2.1520185247063637,
      "learning_rate": 3.5961366883501413e-05,
      "loss": -0.0008,
      "num_tokens": 9098708.0,
      "reward": 0.17300114035606384,
      "reward_std": 0.856629490852356,
      "rewards/rollout_reward_func/mean": 0.17300114035606384,
      "rewards/rollout_reward_func/std": 0.8312443494796753,
      "sampling/importance_sampling_ratio/max": 1.0560925006866455,
      "sampling/importance_sampling_ratio/mean": 1.0095410346984863,
      "sampling/importance_sampling_ratio/min": 0.9969687461853027,
      "sampling/sampling_logp_difference/max": 0.051442861557006836,
      "sampling/sampling_logp_difference/mean": 0.004184390418231487,
      "step": 385,
      "step_time": 9.123970560000998
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1675.0,
      "completions/max_terminated_length": 1675.0,
      "completions/mean_length": 709.9375,
      "completions/mean_terminated_length": 709.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3845339398831129,
      "epoch": 0.00386,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21268585324287415,
      "kl": 0.622815415263176,
      "learning_rate": 3.5894640888113526e-05,
      "loss": 0.0029,
      "num_tokens": 9125044.0,
      "reward": 0.4652027487754822,
      "reward_std": 0.5534366965293884,
      "rewards/rollout_reward_func/mean": 0.4652027487754822,
      "rewards/rollout_reward_func/std": 0.5530131459236145,
      "sampling/importance_sampling_ratio/max": 1.1458910703659058,
      "sampling/importance_sampling_ratio/mean": 1.009918212890625,
      "sampling/importance_sampling_ratio/min": 0.857044517993927,
      "sampling/sampling_logp_difference/max": 0.1110689640045166,
      "sampling/sampling_logp_difference/mean": 0.009240998886525631,
      "step": 386,
      "step_time": 9.90733753999939
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1691.0,
      "completions/max_terminated_length": 1691.0,
      "completions/mean_length": 488.5,
      "completions/mean_terminated_length": 488.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.33496819948777556,
      "epoch": 0.00387,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19780203700065613,
      "kl": 0.47386787831783295,
      "learning_rate": 3.5827822887260103e-05,
      "loss": -0.0097,
      "num_tokens": 9147751.0,
      "reward": 0.7255774736404419,
      "reward_std": 0.4778828024864197,
      "rewards/rollout_reward_func/mean": 0.7255774736404419,
      "rewards/rollout_reward_func/std": 0.4909476339817047,
      "sampling/importance_sampling_ratio/max": 1.0199335813522339,
      "sampling/importance_sampling_ratio/mean": 0.9855973124504089,
      "sampling/importance_sampling_ratio/min": 0.8849200010299683,
      "sampling/sampling_logp_difference/max": 0.12443208694458008,
      "sampling/sampling_logp_difference/mean": 0.006994025316089392,
      "step": 387,
      "step_time": 9.460597914999198
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 915.0,
      "completions/max_terminated_length": 915.0,
      "completions/mean_length": 451.625,
      "completions/mean_terminated_length": 451.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.24471826385706663,
      "epoch": 0.00388,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.195389524102211,
      "kl": 1.035070151090622,
      "learning_rate": 3.576091367730218e-05,
      "loss": 0.0082,
      "num_tokens": 9169550.0,
      "reward": 0.559204638004303,
      "reward_std": 0.721969723701477,
      "rewards/rollout_reward_func/mean": 0.559204638004303,
      "rewards/rollout_reward_func/std": 0.7177640795707703,
      "sampling/importance_sampling_ratio/max": 1.1143460273742676,
      "sampling/importance_sampling_ratio/mean": 1.0062720775604248,
      "sampling/importance_sampling_ratio/min": 0.9391267895698547,
      "sampling/sampling_logp_difference/max": 0.10830807685852051,
      "sampling/sampling_logp_difference/mean": 0.004238985478878021,
      "step": 388,
      "step_time": 7.458844507998947
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 867.0,
      "completions/max_terminated_length": 867.0,
      "completions/mean_length": 489.125,
      "completions/mean_terminated_length": 489.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3336809570901096,
      "epoch": 0.00389,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2059955596923828,
      "kl": 0.6538882032036781,
      "learning_rate": 3.569391405568788e-05,
      "loss": -0.0481,
      "num_tokens": 9192124.0,
      "reward": 0.7297975420951843,
      "reward_std": 0.47417956590652466,
      "rewards/rollout_reward_func/mean": 0.7297975420951843,
      "rewards/rollout_reward_func/std": 0.4835103452205658,
      "sampling/importance_sampling_ratio/max": 1.1061750650405884,
      "sampling/importance_sampling_ratio/mean": 1.010161280632019,
      "sampling/importance_sampling_ratio/min": 0.9256573915481567,
      "sampling/sampling_logp_difference/max": 0.10691165924072266,
      "sampling/sampling_logp_difference/mean": 0.008713415823876858,
      "step": 389,
      "step_time": 7.476394178001101
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1702.0,
      "completions/max_terminated_length": 1702.0,
      "completions/mean_length": 489.3125,
      "completions/mean_terminated_length": 489.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3547071497887373,
      "epoch": 0.0039,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.10007643699645996,
      "kl": 0.6033640000969172,
      "learning_rate": 3.562682482094284e-05,
      "loss": -0.0066,
      "num_tokens": 9214849.0,
      "reward": 0.4032995402812958,
      "reward_std": 0.6649142503738403,
      "rewards/rollout_reward_func/mean": 0.4032995402812958,
      "rewards/rollout_reward_func/std": 0.6491382718086243,
      "sampling/importance_sampling_ratio/max": 1.1360441446304321,
      "sampling/importance_sampling_ratio/mean": 1.0060228109359741,
      "sampling/importance_sampling_ratio/min": 0.9000877141952515,
      "sampling/sampling_logp_difference/max": 0.1076352596282959,
      "sampling/sampling_logp_difference/mean": 0.006674706004559994,
      "step": 390,
      "step_time": 9.9372563980005
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1770.0,
      "completions/max_terminated_length": 1770.0,
      "completions/mean_length": 525.0,
      "completions/mean_terminated_length": 525.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2901929635554552,
      "epoch": 0.00391,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2314658910036087,
      "kl": 2.168528724461794,
      "learning_rate": 3.5559646772660795e-05,
      "loss": 0.0222,
      "num_tokens": 9237678.0,
      "reward": 0.8014934062957764,
      "reward_std": 0.43354320526123047,
      "rewards/rollout_reward_func/mean": 0.8014934062957764,
      "rewards/rollout_reward_func/std": 0.4269736409187317,
      "sampling/importance_sampling_ratio/max": 1.0231837034225464,
      "sampling/importance_sampling_ratio/mean": 1.002927541732788,
      "sampling/importance_sampling_ratio/min": 0.9813400506973267,
      "sampling/sampling_logp_difference/max": 0.02277369797229767,
      "sampling/sampling_logp_difference/mean": 0.002628960646688938,
      "step": 391,
      "step_time": 9.737258985999688
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1657.0,
      "completions/max_terminated_length": 1657.0,
      "completions/mean_length": 300.625,
      "completions/mean_terminated_length": 300.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2645663362927735,
      "epoch": 0.00392,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06539900600910187,
      "kl": 0.5007775984704494,
      "learning_rate": 3.549238071149396e-05,
      "loss": 0.0049,
      "num_tokens": 9255456.0,
      "reward": 0.5953959226608276,
      "reward_std": 0.4722805619239807,
      "rewards/rollout_reward_func/mean": 0.5953959226608276,
      "rewards/rollout_reward_func/std": 0.5396926403045654,
      "sampling/importance_sampling_ratio/max": 1.0413239002227783,
      "sampling/importance_sampling_ratio/mean": 0.9946914911270142,
      "sampling/importance_sampling_ratio/min": 0.9141222238540649,
      "sampling/sampling_logp_difference/max": 0.08941364288330078,
      "sampling/sampling_logp_difference/mean": 0.004600420128554106,
      "step": 392,
      "step_time": 9.16711198199937
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1612.0,
      "completions/max_terminated_length": 1612.0,
      "completions/mean_length": 444.375,
      "completions/mean_terminated_length": 444.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.32805049046874046,
      "epoch": 0.00393,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1335204541683197,
      "kl": 0.7758802324533463,
      "learning_rate": 3.542502743914351e-05,
      "loss": -0.0109,
      "num_tokens": 9276886.0,
      "reward": 0.7384160161018372,
      "reward_std": 0.5734807848930359,
      "rewards/rollout_reward_func/mean": 0.7384160161018372,
      "rewards/rollout_reward_func/std": 0.5934047698974609,
      "sampling/importance_sampling_ratio/max": 1.02571439743042,
      "sampling/importance_sampling_ratio/mean": 0.9849710464477539,
      "sampling/importance_sampling_ratio/min": 0.8887885212898254,
      "sampling/sampling_logp_difference/max": 0.119354248046875,
      "sampling/sampling_logp_difference/mean": 0.007740316912531853,
      "step": 393,
      "step_time": 9.337079361000178
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1734.0,
      "completions/max_terminated_length": 1734.0,
      "completions/mean_length": 527.375,
      "completions/mean_terminated_length": 527.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.41468701185658574,
      "epoch": 0.00394,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.17051926255226135,
      "kl": 0.6778979506343603,
      "learning_rate": 3.535758775835005e-05,
      "loss": 0.0054,
      "num_tokens": 9299901.0,
      "reward": 0.6145593523979187,
      "reward_std": 0.6213626861572266,
      "rewards/rollout_reward_func/mean": 0.6145593523979187,
      "rewards/rollout_reward_func/std": 0.6215264201164246,
      "sampling/importance_sampling_ratio/max": 1.110304355621338,
      "sampling/importance_sampling_ratio/mean": 0.992222785949707,
      "sampling/importance_sampling_ratio/min": 0.8580025434494019,
      "sampling/sampling_logp_difference/max": 0.12202882766723633,
      "sampling/sampling_logp_difference/mean": 0.010809465311467648,
      "step": 394,
      "step_time": 9.660691787998985
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1855.0,
      "completions/max_terminated_length": 1476.0,
      "completions/mean_length": 474.25,
      "completions/mean_terminated_length": 382.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6725482456386089,
      "epoch": 0.00395,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7143154740333557,
      "kl": 0.4643423054367304,
      "learning_rate": 3.529006247288404e-05,
      "loss": -0.0199,
      "num_tokens": 9322002.0,
      "reward": 0.5976954698562622,
      "reward_std": 0.6444228887557983,
      "rewards/rollout_reward_func/mean": 0.5976954698562622,
      "rewards/rollout_reward_func/std": 0.6497763991355896,
      "sampling/importance_sampling_ratio/max": 1.0321837663650513,
      "sampling/importance_sampling_ratio/mean": 0.9606999158859253,
      "sampling/importance_sampling_ratio/min": 0.583530604839325,
      "sampling/sampling_logp_difference/max": 0.11972188949584961,
      "sampling/sampling_logp_difference/mean": 0.020661259070038795,
      "step": 395,
      "step_time": 10.482112677001169
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1763.0,
      "completions/max_terminated_length": 1763.0,
      "completions/mean_length": 733.125,
      "completions/mean_terminated_length": 733.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4408590942621231,
      "epoch": 0.00396,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.14896465837955475,
      "kl": 0.6468664556741714,
      "learning_rate": 3.522245238753618e-05,
      "loss": 0.016,
      "num_tokens": 9349274.0,
      "reward": 0.28018075227737427,
      "reward_std": 0.5999056696891785,
      "rewards/rollout_reward_func/mean": 0.28018075227737427,
      "rewards/rollout_reward_func/std": 0.6153005361557007,
      "sampling/importance_sampling_ratio/max": 1.0338584184646606,
      "sampling/importance_sampling_ratio/mean": 0.9818838834762573,
      "sampling/importance_sampling_ratio/min": 0.8601325750350952,
      "sampling/sampling_logp_difference/max": 0.09864640235900879,
      "sampling/sampling_logp_difference/mean": 0.007579031400382519,
      "step": 396,
      "step_time": 10.164932503998898
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1853.0,
      "completions/max_terminated_length": 1853.0,
      "completions/mean_length": 772.5,
      "completions/mean_terminated_length": 772.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5400008913129568,
      "epoch": 0.00397,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.20949658751487732,
      "kl": 2.536824679002166,
      "learning_rate": 3.515475830810788e-05,
      "loss": 0.0159,
      "num_tokens": 9376635.0,
      "reward": 0.23373597860336304,
      "reward_std": 0.7958845496177673,
      "rewards/rollout_reward_func/mean": 0.23373597860336304,
      "rewards/rollout_reward_func/std": 0.7724964618682861,
      "sampling/importance_sampling_ratio/max": 1.1001425981521606,
      "sampling/importance_sampling_ratio/mean": 0.9811225533485413,
      "sampling/importance_sampling_ratio/min": 0.877095639705658,
      "sampling/sampling_logp_difference/max": 0.13103437423706055,
      "sampling/sampling_logp_difference/mean": 0.0131907993927598,
      "step": 397,
      "step_time": 10.76770480399864
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2639.0,
      "completions/max_terminated_length": 2639.0,
      "completions/mean_length": 844.125,
      "completions/mean_terminated_length": 844.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.691666828468442,
      "epoch": 0.00398,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2847997844219208,
      "kl": 0.5700593739748001,
      "learning_rate": 3.50869810414016e-05,
      "loss": -0.0375,
      "num_tokens": 9405922.0,
      "reward": 0.7989120483398438,
      "reward_std": 0.43895143270492554,
      "rewards/rollout_reward_func/mean": 0.7989120483398438,
      "rewards/rollout_reward_func/std": 0.4323985278606415,
      "sampling/importance_sampling_ratio/max": 1.0305225849151611,
      "sampling/importance_sampling_ratio/mean": 0.9667282104492188,
      "sampling/importance_sampling_ratio/min": 0.7776098847389221,
      "sampling/sampling_logp_difference/max": 0.16749286651611328,
      "sampling/sampling_logp_difference/mean": 0.01985451951622963,
      "step": 398,
      "step_time": 12.941537464001158
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 907.0,
      "completions/max_terminated_length": 907.0,
      "completions/mean_length": 338.0,
      "completions/mean_terminated_length": 338.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.43615838792175055,
      "epoch": 0.00399,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.14489829540252686,
      "kl": 0.5457345116883516,
      "learning_rate": 3.501912139521125e-05,
      "loss": 0.0079,
      "num_tokens": 9425222.0,
      "reward": 0.4557572603225708,
      "reward_std": 0.5821672081947327,
      "rewards/rollout_reward_func/mean": 0.4557572603225708,
      "rewards/rollout_reward_func/std": 0.5624637603759766,
      "sampling/importance_sampling_ratio/max": 1.0770448446273804,
      "sampling/importance_sampling_ratio/mean": 0.9986352920532227,
      "sampling/importance_sampling_ratio/min": 0.8761305212974548,
      "sampling/sampling_logp_difference/max": 0.10946798324584961,
      "sampling/sampling_logp_difference/mean": 0.005334190092980862,
      "step": 399,
      "step_time": 7.153114574000028
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1754.0,
      "completions/max_terminated_length": 1754.0,
      "completions/mean_length": 654.3125,
      "completions/mean_terminated_length": 654.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4097666572779417,
      "epoch": 0.004,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12170318514108658,
      "kl": 0.6051871217787266,
      "learning_rate": 3.49511801783126e-05,
      "loss": -0.0035,
      "num_tokens": 9451191.0,
      "reward": 0.5482263565063477,
      "reward_std": 0.6195113658905029,
      "rewards/rollout_reward_func/mean": 0.5482263565063477,
      "rewards/rollout_reward_func/std": 0.6373432278633118,
      "sampling/importance_sampling_ratio/max": 1.0686990022659302,
      "sampling/importance_sampling_ratio/mean": 0.9729100465774536,
      "sampling/importance_sampling_ratio/min": 0.7990859746932983,
      "sampling/sampling_logp_difference/max": 0.23273897171020508,
      "sampling/sampling_logp_difference/mean": 0.013899194076657295,
      "step": 400,
      "step_time": 9.990303853999649
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2419.0,
      "completions/max_terminated_length": 2419.0,
      "completions/mean_length": 623.0,
      "completions/mean_terminated_length": 623.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5797332413494587,
      "epoch": 0.00401,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2365940362215042,
      "kl": 1.6211334988474846,
      "learning_rate": 3.488315820045356e-05,
      "loss": -0.0134,
      "num_tokens": 9476266.0,
      "reward": 0.47696301341056824,
      "reward_std": 0.5195108652114868,
      "rewards/rollout_reward_func/mean": 0.47696301341056824,
      "rewards/rollout_reward_func/std": 0.6501075625419617,
      "sampling/importance_sampling_ratio/max": 1.131629228591919,
      "sampling/importance_sampling_ratio/mean": 1.0077385902404785,
      "sampling/importance_sampling_ratio/min": 0.9065260291099548,
      "sampling/sampling_logp_difference/max": 0.1211388111114502,
      "sampling/sampling_logp_difference/mean": 0.010793443769216537,
      "step": 401,
      "step_time": 11.677488792000077
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1764.0,
      "completions/max_terminated_length": 1764.0,
      "completions/mean_length": 493.4375,
      "completions/mean_terminated_length": 386.2857360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8214016789570451,
      "epoch": 0.00402,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6195135116577148,
      "kl": 0.4268805608153343,
      "learning_rate": 3.481505627234463e-05,
      "loss": 0.0112,
      "num_tokens": 9498490.0,
      "reward": 0.5221271514892578,
      "reward_std": 0.5760427117347717,
      "rewards/rollout_reward_func/mean": 0.5221271514892578,
      "rewards/rollout_reward_func/std": 0.5601909756660461,
      "sampling/importance_sampling_ratio/max": 1.150269627571106,
      "sampling/importance_sampling_ratio/mean": 0.9903530478477478,
      "sampling/importance_sampling_ratio/min": 0.5255114436149597,
      "sampling/sampling_logp_difference/max": 0.2112419605255127,
      "sampling/sampling_logp_difference/mean": 0.029344674199819565,
      "step": 402,
      "step_time": 10.249449274999279
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 879.0,
      "completions/max_terminated_length": 879.0,
      "completions/mean_length": 341.0625,
      "completions/mean_terminated_length": 341.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.34962758235633373,
      "epoch": 0.00403,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1209549605846405,
      "kl": 0.6168643143028021,
      "learning_rate": 3.4746875205649146e-05,
      "loss": 0.0026,
      "num_tokens": 9517672.0,
      "reward": 0.5920791029930115,
      "reward_std": 0.5413715243339539,
      "rewards/rollout_reward_func/mean": 0.5920791029930115,
      "rewards/rollout_reward_func/std": 0.5439906716346741,
      "sampling/importance_sampling_ratio/max": 1.0623682737350464,
      "sampling/importance_sampling_ratio/mean": 1.004093050956726,
      "sampling/importance_sampling_ratio/min": 0.9691905379295349,
      "sampling/sampling_logp_difference/max": 0.060921669006347656,
      "sampling/sampling_logp_difference/mean": 0.004399741068482399,
      "step": 403,
      "step_time": 6.938582816001144
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2466.0,
      "completions/max_terminated_length": 2466.0,
      "completions/mean_length": 683.5,
      "completions/mean_terminated_length": 683.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5381776951253414,
      "epoch": 0.00404,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.25386425852775574,
      "kl": 0.3799141040071845,
      "learning_rate": 3.467861581297367e-05,
      "loss": 0.0062,
      "num_tokens": 9543505.0,
      "reward": 0.4942741096019745,
      "reward_std": 0.7897206544876099,
      "rewards/rollout_reward_func/mean": 0.4942741096019745,
      "rewards/rollout_reward_func/std": 0.8159600496292114,
      "sampling/importance_sampling_ratio/max": 1.0297006368637085,
      "sampling/importance_sampling_ratio/mean": 0.980741560459137,
      "sampling/importance_sampling_ratio/min": 0.8977947235107422,
      "sampling/sampling_logp_difference/max": 0.08828115463256836,
      "sampling/sampling_logp_difference/mean": 0.008286667056381702,
      "step": 404,
      "step_time": 12.219259049999891
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2809.0,
      "completions/max_terminated_length": 2809.0,
      "completions/mean_length": 530.0625,
      "completions/mean_terminated_length": 530.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5055952407419682,
      "epoch": 0.00405,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.10579125583171844,
      "kl": 0.3927855081856251,
      "learning_rate": 3.461027890785828e-05,
      "loss": 0.006,
      "num_tokens": 9566657.0,
      "reward": 0.4758409261703491,
      "reward_std": 0.6480588912963867,
      "rewards/rollout_reward_func/mean": 0.4758409261703491,
      "rewards/rollout_reward_func/std": 0.647657573223114,
      "sampling/importance_sampling_ratio/max": 1.1527551412582397,
      "sampling/importance_sampling_ratio/mean": 1.0163654088974,
      "sampling/importance_sampling_ratio/min": 0.8879061341285706,
      "sampling/sampling_logp_difference/max": 0.11920619010925293,
      "sampling/sampling_logp_difference/mean": 0.009752889163792133,
      "step": 405,
      "step_time": 12.655609585000093
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1610.0,
      "completions/max_terminated_length": 1610.0,
      "completions/mean_length": 578.375,
      "completions/mean_terminated_length": 612.6666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9389865528792143,
      "epoch": 0.00406,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.26622331142425537,
      "kl": 0.5132467858493328,
      "learning_rate": 3.454186530476685e-05,
      "loss": -0.0292,
      "num_tokens": 9590835.0,
      "reward": 0.5705388188362122,
      "reward_std": 0.8137123584747314,
      "rewards/rollout_reward_func/mean": 0.5705388188362122,
      "rewards/rollout_reward_func/std": 0.7915499806404114,
      "sampling/importance_sampling_ratio/max": 1.078421711921692,
      "sampling/importance_sampling_ratio/mean": 0.9347054958343506,
      "sampling/importance_sampling_ratio/min": 2.1507197064951633e-11,
      "sampling/sampling_logp_difference/max": 12.825655937194824,
      "sampling/sampling_logp_difference/mean": 0.23650743067264557,
      "step": 406,
      "step_time": 10.22529919399858
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1735.0,
      "completions/max_terminated_length": 1735.0,
      "completions/mean_length": 650.3125,
      "completions/mean_terminated_length": 650.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7654261253774166,
      "epoch": 0.00407,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.324076533317566,
      "kl": 0.4224965199828148,
      "learning_rate": 3.4473375819077397e-05,
      "loss": -0.2598,
      "num_tokens": 9615840.0,
      "reward": 0.5525083541870117,
      "reward_std": 0.7547932863235474,
      "rewards/rollout_reward_func/mean": 0.5525083541870117,
      "rewards/rollout_reward_func/std": 0.7321653962135315,
      "sampling/importance_sampling_ratio/max": 2.1255531311035156,
      "sampling/importance_sampling_ratio/mean": 1.064455270767212,
      "sampling/importance_sampling_ratio/min": 0.8944579362869263,
      "sampling/sampling_logp_difference/max": 0.1222696304321289,
      "sampling/sampling_logp_difference/mean": 0.024517850950360298,
      "step": 407,
      "step_time": 10.37980686800256
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1716.0,
      "completions/max_terminated_length": 1716.0,
      "completions/mean_length": 764.3125,
      "completions/mean_terminated_length": 754.0000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0267420560121536,
      "epoch": 0.00408,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6055708527565002,
      "kl": 0.6773878410458565,
      "learning_rate": 3.440481126707234e-05,
      "loss": 0.049,
      "num_tokens": 9643174.0,
      "reward": 0.48169052600860596,
      "reward_std": 0.6450338363647461,
      "rewards/rollout_reward_func/mean": 0.48169052600860596,
      "rewards/rollout_reward_func/std": 0.6429747343063354,
      "sampling/importance_sampling_ratio/max": 1.0349609851837158,
      "sampling/importance_sampling_ratio/mean": 0.9531961679458618,
      "sampling/importance_sampling_ratio/min": 0.7323054075241089,
      "sampling/sampling_logp_difference/max": 0.1982567310333252,
      "sampling/sampling_logp_difference/mean": 0.027478821575641632,
      "step": 408,
      "step_time": 10.312683289999768
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1711.0,
      "completions/max_terminated_length": 1711.0,
      "completions/mean_length": 670.3125,
      "completions/mean_terminated_length": 712.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9028977826237679,
      "epoch": 0.00409,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6618459224700928,
      "kl": 0.47739602625370026,
      "learning_rate": 3.4336172465928716e-05,
      "loss": 0.0781,
      "num_tokens": 9669272.0,
      "reward": 0.5530873537063599,
      "reward_std": 0.6294262409210205,
      "rewards/rollout_reward_func/mean": 0.5530873537063599,
      "rewards/rollout_reward_func/std": 0.7218759059906006,
      "sampling/importance_sampling_ratio/max": 1.108457326889038,
      "sampling/importance_sampling_ratio/mean": 0.9805165529251099,
      "sampling/importance_sampling_ratio/min": 0.5824363231658936,
      "sampling/sampling_logp_difference/max": 0.1140589714050293,
      "sampling/sampling_logp_difference/mean": 0.015781613066792488,
      "step": 409,
      "step_time": 10.267681784998786
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 853.0,
      "completions/max_terminated_length": 853.0,
      "completions/mean_length": 571.3125,
      "completions/mean_terminated_length": 571.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6798927783966064,
      "epoch": 0.0041,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7133686542510986,
      "kl": 0.5507989739999175,
      "learning_rate": 3.4267460233708554e-05,
      "loss": 0.1503,
      "num_tokens": 9693342.0,
      "reward": 0.47610414028167725,
      "reward_std": 0.6544132232666016,
      "rewards/rollout_reward_func/mean": 0.47610414028167725,
      "rewards/rollout_reward_func/std": 0.6525593996047974,
      "sampling/importance_sampling_ratio/max": 1.137495994567871,
      "sampling/importance_sampling_ratio/mean": 1.0147191286087036,
      "sampling/importance_sampling_ratio/min": 0.9399718046188354,
      "sampling/sampling_logp_difference/max": 0.1870594024658203,
      "sampling/sampling_logp_difference/mean": 0.016649583354592323,
      "step": 410,
      "step_time": 7.827035397001055
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2734.0,
      "completions/max_terminated_length": 2734.0,
      "completions/mean_length": 727.75,
      "completions/mean_terminated_length": 727.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5990832690149546,
      "epoch": 0.00411,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.18863800168037415,
      "kl": 0.4268999248743057,
      "learning_rate": 3.419867538934902e-05,
      "loss": 0.0137,
      "num_tokens": 9720496.0,
      "reward": 0.42967474460601807,
      "reward_std": 0.7024286985397339,
      "rewards/rollout_reward_func/mean": 0.42967474460601807,
      "rewards/rollout_reward_func/std": 0.7194387912750244,
      "sampling/importance_sampling_ratio/max": 1.1061264276504517,
      "sampling/importance_sampling_ratio/mean": 1.0097965002059937,
      "sampling/importance_sampling_ratio/min": 0.955376386642456,
      "sampling/sampling_logp_difference/max": 0.10082173347473145,
      "sampling/sampling_logp_difference/mean": 0.009000581689178944,
      "step": 411,
      "step_time": 13.214731916000346
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 849.0,
      "completions/max_terminated_length": 849.0,
      "completions/mean_length": 284.75,
      "completions/mean_terminated_length": 284.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7675218656659126,
      "epoch": 0.00412,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3258371353149414,
      "kl": 0.6578252403996885,
      "learning_rate": 3.412981875265269e-05,
      "loss": -0.0212,
      "num_tokens": 9739289.0,
      "reward": 0.42009812593460083,
      "reward_std": 0.820585310459137,
      "rewards/rollout_reward_func/mean": 0.42009812593460083,
      "rewards/rollout_reward_func/std": 0.8196895122528076,
      "sampling/importance_sampling_ratio/max": 1.1545556783676147,
      "sampling/importance_sampling_ratio/mean": 0.9954267144203186,
      "sampling/importance_sampling_ratio/min": 0.6045799255371094,
      "sampling/sampling_logp_difference/max": 0.14378738403320312,
      "sampling/sampling_logp_difference/mean": 0.021919023245573044,
      "step": 412,
      "step_time": 7.07668304300023
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1762.0,
      "completions/max_terminated_length": 1762.0,
      "completions/mean_length": 595.8125,
      "completions/mean_terminated_length": 595.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7114283107221127,
      "epoch": 0.00413,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1645776927471161,
      "kl": 0.42052726447582245,
      "learning_rate": 3.4060891144277846e-05,
      "loss": -0.0008,
      "num_tokens": 9763547.0,
      "reward": 0.7451732754707336,
      "reward_std": 0.5538823008537292,
      "rewards/rollout_reward_func/mean": 0.7451732754707336,
      "rewards/rollout_reward_func/std": 0.5825620889663696,
      "sampling/importance_sampling_ratio/max": 1.1363765001296997,
      "sampling/importance_sampling_ratio/mean": 1.0192832946777344,
      "sampling/importance_sampling_ratio/min": 0.9906336665153503,
      "sampling/sampling_logp_difference/max": 0.10859131813049316,
      "sampling/sampling_logp_difference/mean": 0.008361928164958954,
      "step": 413,
      "step_time": 9.952231179000592
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1748.0,
      "completions/max_terminated_length": 1748.0,
      "completions/mean_length": 503.0625,
      "completions/mean_terminated_length": 503.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6981067964807153,
      "epoch": 0.00414,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7110888361930847,
      "kl": 0.5346888825297356,
      "learning_rate": 3.399189338572856e-05,
      "loss": 0.0391,
      "num_tokens": 9785119.0,
      "reward": 0.7343393564224243,
      "reward_std": 0.49204468727111816,
      "rewards/rollout_reward_func/mean": 0.7343393564224243,
      "rewards/rollout_reward_func/std": 0.47575080394744873,
      "sampling/importance_sampling_ratio/max": 1.2414218187332153,
      "sampling/importance_sampling_ratio/mean": 1.0260465145111084,
      "sampling/importance_sampling_ratio/min": 0.9743067026138306,
      "sampling/sampling_logp_difference/max": 0.186029314994812,
      "sampling/sampling_logp_difference/mean": 0.013075749389827251,
      "step": 414,
      "step_time": 9.691852687999926
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 924.0,
      "completions/max_terminated_length": 924.0,
      "completions/mean_length": 505.5,
      "completions/mean_terminated_length": 505.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7257580114528537,
      "epoch": 0.00415,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2912153899669647,
      "kl": 0.6316624814644456,
      "learning_rate": 3.3922826299345026e-05,
      "loss": 0.0012,
      "num_tokens": 9807496.0,
      "reward": 0.5622926950454712,
      "reward_std": 0.6874678730964661,
      "rewards/rollout_reward_func/mean": 0.5622926950454712,
      "rewards/rollout_reward_func/std": 0.7100165486335754,
      "sampling/importance_sampling_ratio/max": 1.055029034614563,
      "sampling/importance_sampling_ratio/mean": 0.9678635597229004,
      "sampling/importance_sampling_ratio/min": 0.6565594673156738,
      "sampling/sampling_logp_difference/max": 0.11985588073730469,
      "sampling/sampling_logp_difference/mean": 0.014965438283979893,
      "step": 415,
      "step_time": 7.690247540999735
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1817.0,
      "completions/max_terminated_length": 1817.0,
      "completions/mean_length": 694.5625,
      "completions/mean_terminated_length": 738.7333984375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.988004881888628,
      "epoch": 0.00416,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9535210132598877,
      "kl": 1.0936174169182777,
      "learning_rate": 3.3853690708293704e-05,
      "loss": 0.1092,
      "num_tokens": 9834281.0,
      "reward": 0.42284542322158813,
      "reward_std": 0.7488222122192383,
      "rewards/rollout_reward_func/mean": 0.42284542322158813,
      "rewards/rollout_reward_func/std": 0.7267349362373352,
      "sampling/importance_sampling_ratio/max": 1.2765588760375977,
      "sampling/importance_sampling_ratio/mean": 0.9691144227981567,
      "sampling/importance_sampling_ratio/min": 2.0612029061339854e-07,
      "sampling/sampling_logp_difference/max": 11.663540840148926,
      "sampling/sampling_logp_difference/mean": 0.10559066385030746,
      "step": 416,
      "step_time": 10.846577319000062
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1488.0,
      "completions/max_terminated_length": 1488.0,
      "completions/mean_length": 618.0625,
      "completions/mean_terminated_length": 618.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6391468457877636,
      "epoch": 0.00417,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.47261351346969604,
      "kl": 1.8777605844661593,
      "learning_rate": 3.3784487436557514e-05,
      "loss": 0.0696,
      "num_tokens": 9858895.0,
      "reward": 0.6613080501556396,
      "reward_std": 0.5310009121894836,
      "rewards/rollout_reward_func/mean": 0.6613080501556396,
      "rewards/rollout_reward_func/std": 0.5189067125320435,
      "sampling/importance_sampling_ratio/max": 1.0436620712280273,
      "sampling/importance_sampling_ratio/mean": 0.9395931959152222,
      "sampling/importance_sampling_ratio/min": 0.5100922584533691,
      "sampling/sampling_logp_difference/max": 0.2459402084350586,
      "sampling/sampling_logp_difference/mean": 0.022484328597784042,
      "step": 417,
      "step_time": 10.15138819799904
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 862.0,
      "completions/max_terminated_length": 862.0,
      "completions/mean_length": 320.625,
      "completions/mean_terminated_length": 320.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9175955969840288,
      "epoch": 0.00418,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3972008228302002,
      "kl": 0.7033107969909906,
      "learning_rate": 3.3715217308926e-05,
      "loss": -0.04,
      "num_tokens": 9877773.0,
      "reward": 0.5474278926849365,
      "reward_std": 0.7572165727615356,
      "rewards/rollout_reward_func/mean": 0.5474278926849365,
      "rewards/rollout_reward_func/std": 0.7351523637771606,
      "sampling/importance_sampling_ratio/max": 1.0720980167388916,
      "sampling/importance_sampling_ratio/mean": 0.9776468276977539,
      "sampling/importance_sampling_ratio/min": 0.7163570523262024,
      "sampling/sampling_logp_difference/max": 0.1531515121459961,
      "sampling/sampling_logp_difference/mean": 0.01922701671719551,
      "step": 418,
      "step_time": 7.158642653000243
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1845.0,
      "completions/max_terminated_length": 1845.0,
      "completions/mean_length": 488.5,
      "completions/mean_terminated_length": 488.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5481152581050992,
      "epoch": 0.00419,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1733013391494751,
      "kl": 0.6051683835685253,
      "learning_rate": 3.3645881150985565e-05,
      "loss": 0.0083,
      "num_tokens": 9900346.0,
      "reward": 0.5385076999664307,
      "reward_std": 0.6246837377548218,
      "rewards/rollout_reward_func/mean": 0.5385076999664307,
      "rewards/rollout_reward_func/std": 0.652523934841156,
      "sampling/importance_sampling_ratio/max": 1.080888032913208,
      "sampling/importance_sampling_ratio/mean": 0.9963943958282471,
      "sampling/importance_sampling_ratio/min": 0.9355956315994263,
      "sampling/sampling_logp_difference/max": 0.07719635963439941,
      "sampling/sampling_logp_difference/mean": 0.006856988649815321,
      "step": 419,
      "step_time": 9.828812708000441
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1566.0,
      "completions/max_terminated_length": 1566.0,
      "completions/mean_length": 575.5625,
      "completions/mean_terminated_length": 575.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5870987214148045,
      "epoch": 0.0042,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5528720021247864,
      "kl": 0.5378256514668465,
      "learning_rate": 3.357647978910954e-05,
      "loss": 0.0425,
      "num_tokens": 9924366.0,
      "reward": 0.728959321975708,
      "reward_std": 0.5019761323928833,
      "rewards/rollout_reward_func/mean": 0.728959321975708,
      "rewards/rollout_reward_func/std": 0.4849759638309479,
      "sampling/importance_sampling_ratio/max": 1.0942471027374268,
      "sampling/importance_sampling_ratio/mean": 0.9789748191833496,
      "sampling/importance_sampling_ratio/min": 0.8678211569786072,
      "sampling/sampling_logp_difference/max": 0.1537494659423828,
      "sampling/sampling_logp_difference/mean": 0.018790388479828835,
      "step": 420,
      "step_time": 9.502542613000514
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1747.0,
      "completions/max_terminated_length": 1747.0,
      "completions/mean_length": 701.625,
      "completions/mean_terminated_length": 701.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8220043387264013,
      "epoch": 0.00421,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.42290446162223816,
      "kl": 0.5747890081256628,
      "learning_rate": 3.3507014050448416e-05,
      "loss": -0.0671,
      "num_tokens": 9950286.0,
      "reward": 0.5401800870895386,
      "reward_std": 0.6266507506370544,
      "rewards/rollout_reward_func/mean": 0.5401800870895386,
      "rewards/rollout_reward_func/std": 0.6482610702514648,
      "sampling/importance_sampling_ratio/max": 1.0366859436035156,
      "sampling/importance_sampling_ratio/mean": 1.0017718076705933,
      "sampling/importance_sampling_ratio/min": 0.9443310499191284,
      "sampling/sampling_logp_difference/max": 0.12004375457763672,
      "sampling/sampling_logp_difference/mean": 0.01438192930072546,
      "step": 421,
      "step_time": 10.239487848999488
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 942.0,
      "completions/max_terminated_length": 942.0,
      "completions/mean_length": 535.125,
      "completions/mean_terminated_length": 535.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5072913244366646,
      "epoch": 0.00422,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.33991703391075134,
      "kl": 0.4900359380990267,
      "learning_rate": 3.343748476291991e-05,
      "loss": 0.0121,
      "num_tokens": 9973294.0,
      "reward": 0.6781119108200073,
      "reward_std": 0.6147701144218445,
      "rewards/rollout_reward_func/mean": 0.6781119108200073,
      "rewards/rollout_reward_func/std": 0.6087037324905396,
      "sampling/importance_sampling_ratio/max": 1.1043634414672852,
      "sampling/importance_sampling_ratio/mean": 0.9978954792022705,
      "sampling/importance_sampling_ratio/min": 0.9173483848571777,
      "sampling/sampling_logp_difference/max": 0.09875679016113281,
      "sampling/sampling_logp_difference/mean": 0.008721997030079365,
      "step": 422,
      "step_time": 7.681494927997846
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1770.0,
      "completions/max_terminated_length": 1770.0,
      "completions/mean_length": 537.375,
      "completions/mean_terminated_length": 537.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5110290516167879,
      "epoch": 0.00423,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.14359425008296967,
      "kl": 1.5958131328225136,
      "learning_rate": 3.336789275519917e-05,
      "loss": -0.0043,
      "num_tokens": 9996265.0,
      "reward": 0.6053416728973389,
      "reward_std": 0.6307316422462463,
      "rewards/rollout_reward_func/mean": 0.6053416728973389,
      "rewards/rollout_reward_func/std": 0.6412364840507507,
      "sampling/importance_sampling_ratio/max": 1.131142020225525,
      "sampling/importance_sampling_ratio/mean": 1.0116039514541626,
      "sampling/importance_sampling_ratio/min": 0.9184310436248779,
      "sampling/sampling_logp_difference/max": 0.12317180633544922,
      "sampling/sampling_logp_difference/mean": 0.010907256975769997,
      "step": 423,
      "step_time": 9.89791882299869
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1619.0,
      "completions/max_terminated_length": 1619.0,
      "completions/mean_length": 409.875,
      "completions/mean_terminated_length": 409.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5237524639815092,
      "epoch": 0.00424,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.16794754564762115,
      "kl": 0.4495466612279415,
      "learning_rate": 3.3298238856708845e-05,
      "loss": -0.0055,
      "num_tokens": 10017376.0,
      "reward": 0.4683135151863098,
      "reward_std": 0.6614490151405334,
      "rewards/rollout_reward_func/mean": 0.4683135151863098,
      "rewards/rollout_reward_func/std": 0.6563630700111389,
      "sampling/importance_sampling_ratio/max": 1.0611085891723633,
      "sampling/importance_sampling_ratio/mean": 1.0015487670898438,
      "sampling/importance_sampling_ratio/min": 0.9139368534088135,
      "sampling/sampling_logp_difference/max": 0.08792269229888916,
      "sampling/sampling_logp_difference/mean": 0.009999245405197144,
      "step": 424,
      "step_time": 9.695940874999906
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1540.0,
      "completions/max_terminated_length": 1540.0,
      "completions/mean_length": 486.6875,
      "completions/mean_terminated_length": 486.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4938153102993965,
      "epoch": 0.00425,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3936285972595215,
      "kl": 0.33048933930695057,
      "learning_rate": 3.3228523897609235e-05,
      "loss": 0.0235,
      "num_tokens": 10039541.0,
      "reward": 0.4996128976345062,
      "reward_std": 0.8294677734375,
      "rewards/rollout_reward_func/mean": 0.4996128976345062,
      "rewards/rollout_reward_func/std": 0.8055407404899597,
      "sampling/importance_sampling_ratio/max": 1.0872386693954468,
      "sampling/importance_sampling_ratio/mean": 0.9856383800506592,
      "sampling/importance_sampling_ratio/min": 0.9264695048332214,
      "sampling/sampling_logp_difference/max": 0.08360958099365234,
      "sampling/sampling_logp_difference/mean": 0.011944936588406563,
      "step": 425,
      "step_time": 9.033099533000495
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1736.0,
      "completions/max_terminated_length": 1736.0,
      "completions/mean_length": 567.8125,
      "completions/mean_terminated_length": 567.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4998687980696559,
      "epoch": 0.00426,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21246832609176636,
      "kl": 0.6474628504365683,
      "learning_rate": 3.3158748708788384e-05,
      "loss": 0.0091,
      "num_tokens": 10062740.0,
      "reward": 0.8648368120193481,
      "reward_std": 0.38229936361312866,
      "rewards/rollout_reward_func/mean": 0.8648368120193481,
      "rewards/rollout_reward_func/std": 0.3695077896118164,
      "sampling/importance_sampling_ratio/max": 1.1165707111358643,
      "sampling/importance_sampling_ratio/mean": 1.0159037113189697,
      "sampling/importance_sampling_ratio/min": 0.9827269315719604,
      "sampling/sampling_logp_difference/max": 0.10900115966796875,
      "sampling/sampling_logp_difference/mean": 0.006623582448810339,
      "step": 426,
      "step_time": 9.746105150998119
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1610.0,
      "completions/max_terminated_length": 1610.0,
      "completions/mean_length": 414.75,
      "completions/mean_terminated_length": 414.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.40827152878046036,
      "epoch": 0.00427,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09992268681526184,
      "kl": 0.4097367478534579,
      "learning_rate": 3.308891412185217e-05,
      "loss": -0.0004,
      "num_tokens": 10083241.0,
      "reward": 0.7926955223083496,
      "reward_std": 0.4496910870075226,
      "rewards/rollout_reward_func/mean": 0.7926955223083496,
      "rewards/rollout_reward_func/std": 0.4457743167877197,
      "sampling/importance_sampling_ratio/max": 1.0989665985107422,
      "sampling/importance_sampling_ratio/mean": 1.001367449760437,
      "sampling/importance_sampling_ratio/min": 0.9301401376724243,
      "sampling/sampling_logp_difference/max": 0.08368039131164551,
      "sampling/sampling_logp_difference/mean": 0.007650079671293497,
      "step": 427,
      "step_time": 9.309216256000582
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1718.0,
      "completions/max_terminated_length": 1718.0,
      "completions/mean_length": 570.25,
      "completions/mean_terminated_length": 570.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49296560883522034,
      "epoch": 0.00428,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1333617866039276,
      "kl": 0.4495165180414915,
      "learning_rate": 3.301902096911438e-05,
      "loss": 0.0137,
      "num_tokens": 10107096.0,
      "reward": 0.7323318719863892,
      "reward_std": 0.4601137638092041,
      "rewards/rollout_reward_func/mean": 0.7323318719863892,
      "rewards/rollout_reward_func/std": 0.4793003797531128,
      "sampling/importance_sampling_ratio/max": 1.1221542358398438,
      "sampling/importance_sampling_ratio/mean": 1.0003001689910889,
      "sampling/importance_sampling_ratio/min": 0.8928087949752808,
      "sampling/sampling_logp_difference/max": 0.11456871032714844,
      "sampling/sampling_logp_difference/mean": 0.009420101530849934,
      "step": 428,
      "step_time": 9.645055352000782
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2628.0,
      "completions/max_terminated_length": 2628.0,
      "completions/mean_length": 676.6875,
      "completions/mean_terminated_length": 608.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0524565167725086,
      "epoch": 0.00429,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5381286144256592,
      "kl": 0.4900663122534752,
      "learning_rate": 3.294907008358685e-05,
      "loss": -0.0993,
      "num_tokens": 10132605.0,
      "reward": 0.6185888051986694,
      "reward_std": 0.7278683185577393,
      "rewards/rollout_reward_func/mean": 0.6185888051986694,
      "rewards/rollout_reward_func/std": 0.7180782556533813,
      "sampling/importance_sampling_ratio/max": 1.0603554248809814,
      "sampling/importance_sampling_ratio/mean": 1.01093590259552,
      "sampling/importance_sampling_ratio/min": 0.9343549609184265,
      "sampling/sampling_logp_difference/max": 0.17896604537963867,
      "sampling/sampling_logp_difference/mean": 0.02305012196302414,
      "step": 429,
      "step_time": 12.679399759998887
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1700.0,
      "completions/max_terminated_length": 1700.0,
      "completions/mean_length": 579.5625,
      "completions/mean_terminated_length": 579.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7792944796383381,
      "epoch": 0.0043,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.16407586634159088,
      "kl": 0.5804638359695673,
      "learning_rate": 3.287906229896946e-05,
      "loss": 0.0018,
      "num_tokens": 10156577.0,
      "reward": 0.6943863034248352,
      "reward_std": 0.44125598669052124,
      "rewards/rollout_reward_func/mean": 0.6943863034248352,
      "rewards/rollout_reward_func/std": 0.680499255657196,
      "sampling/importance_sampling_ratio/max": 1.0579588413238525,
      "sampling/importance_sampling_ratio/mean": 1.0028170347213745,
      "sampling/importance_sampling_ratio/min": 0.9457554221153259,
      "sampling/sampling_logp_difference/max": 0.0625143051147461,
      "sampling/sampling_logp_difference/mean": 0.008360964246094227,
      "step": 430,
      "step_time": 9.75893636300134
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1608.0,
      "completions/max_terminated_length": 1608.0,
      "completions/mean_length": 651.375,
      "completions/mean_terminated_length": 651.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7346131280064583,
      "epoch": 0.00431,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.22133062779903412,
      "kl": 0.620928768068552,
      "learning_rate": 3.280899844964024e-05,
      "loss": -0.0136,
      "num_tokens": 10182743.0,
      "reward": 0.38345521688461304,
      "reward_std": 0.8985757827758789,
      "rewards/rollout_reward_func/mean": 0.38345521688461304,
      "rewards/rollout_reward_func/std": 0.8681080937385559,
      "sampling/importance_sampling_ratio/max": 1.0823917388916016,
      "sampling/importance_sampling_ratio/mean": 1.011925220489502,
      "sampling/importance_sampling_ratio/min": 0.9635922312736511,
      "sampling/sampling_logp_difference/max": 0.07752132415771484,
      "sampling/sampling_logp_difference/mean": 0.009758299216628075,
      "step": 431,
      "step_time": 10.159818301999621
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1745.0,
      "completions/max_terminated_length": 1745.0,
      "completions/mean_length": 555.25,
      "completions/mean_terminated_length": 555.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6738170124590397,
      "epoch": 0.00432,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3007461130619049,
      "kl": 0.44247581623494625,
      "learning_rate": 3.273887937064545e-05,
      "loss": 0.0063,
      "num_tokens": 10205843.0,
      "reward": 0.6259163618087769,
      "reward_std": 0.733720064163208,
      "rewards/rollout_reward_func/mean": 0.6259163618087769,
      "rewards/rollout_reward_func/std": 0.7089503407478333,
      "sampling/importance_sampling_ratio/max": 1.1096810102462769,
      "sampling/importance_sampling_ratio/mean": 0.9956022500991821,
      "sampling/importance_sampling_ratio/min": 0.9119305610656738,
      "sampling/sampling_logp_difference/max": 0.1732950210571289,
      "sampling/sampling_logp_difference/mean": 0.014945861883461475,
      "step": 432,
      "step_time": 9.744770033002169
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1727.0,
      "completions/max_terminated_length": 1727.0,
      "completions/mean_length": 527.1875,
      "completions/mean_terminated_length": 527.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7240229323506355,
      "epoch": 0.00433,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.29576048254966736,
      "kl": 0.6050397660583258,
      "learning_rate": 3.266870589768956e-05,
      "loss": 0.0006,
      "num_tokens": 10229422.0,
      "reward": 0.30640149116516113,
      "reward_std": 0.8048019409179688,
      "rewards/rollout_reward_func/mean": 0.30640149116516113,
      "rewards/rollout_reward_func/std": 0.8652485609054565,
      "sampling/importance_sampling_ratio/max": 1.101239562034607,
      "sampling/importance_sampling_ratio/mean": 0.9981024265289307,
      "sampling/importance_sampling_ratio/min": 0.8888553977012634,
      "sampling/sampling_logp_difference/max": 0.11624360084533691,
      "sampling/sampling_logp_difference/mean": 0.010926023125648499,
      "step": 433,
      "step_time": 9.994251951000479
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 844.0,
      "completions/max_terminated_length": 844.0,
      "completions/mean_length": 459.1875,
      "completions/mean_terminated_length": 459.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.49993154779076576,
      "epoch": 0.00434,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.17490893602371216,
      "kl": 0.3291987795382738,
      "learning_rate": 3.259847886712534e-05,
      "loss": 0.0009,
      "num_tokens": 10251544.0,
      "reward": 0.36758723855018616,
      "reward_std": 0.7776446342468262,
      "rewards/rollout_reward_func/mean": 0.36758723855018616,
      "rewards/rollout_reward_func/std": 0.798039436340332,
      "sampling/importance_sampling_ratio/max": 1.0691977739334106,
      "sampling/importance_sampling_ratio/mean": 1.0037602186203003,
      "sampling/importance_sampling_ratio/min": 0.9257723689079285,
      "sampling/sampling_logp_difference/max": 0.07649588584899902,
      "sampling/sampling_logp_difference/mean": 0.008522538468241692,
      "step": 434,
      "step_time": 7.131112055999438
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2644.0,
      "completions/max_terminated_length": 2644.0,
      "completions/mean_length": 874.625,
      "completions/mean_terminated_length": 874.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5989832356572151,
      "epoch": 0.00435,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.20147714018821716,
      "kl": 0.3917876952327788,
      "learning_rate": 3.252819911594388e-05,
      "loss": -0.0057,
      "num_tokens": 10280124.0,
      "reward": 0.7522997260093689,
      "reward_std": 0.3675282597541809,
      "rewards/rollout_reward_func/mean": 0.7522997260093689,
      "rewards/rollout_reward_func/std": 0.5635508894920349,
      "sampling/importance_sampling_ratio/max": 1.1159559488296509,
      "sampling/importance_sampling_ratio/mean": 1.004692792892456,
      "sampling/importance_sampling_ratio/min": 0.7795676589012146,
      "sampling/sampling_logp_difference/max": 0.20969843864440918,
      "sampling/sampling_logp_difference/mean": 0.013496816158294678,
      "step": 435,
      "step_time": 12.565505141001267
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 880.0,
      "completions/max_terminated_length": 880.0,
      "completions/mean_length": 291.75,
      "completions/mean_terminated_length": 291.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4846190009266138,
      "epoch": 0.00436,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19056151807308197,
      "kl": 0.5709140384569764,
      "learning_rate": 3.245786748176462e-05,
      "loss": 0.0041,
      "num_tokens": 10297737.0,
      "reward": 0.5003809928894043,
      "reward_std": 0.827323317527771,
      "rewards/rollout_reward_func/mean": 0.5003809928894043,
      "rewards/rollout_reward_func/std": 0.8034564256668091,
      "sampling/importance_sampling_ratio/max": 1.044302225112915,
      "sampling/importance_sampling_ratio/mean": 0.9884238243103027,
      "sampling/importance_sampling_ratio/min": 0.9148637056350708,
      "sampling/sampling_logp_difference/max": 0.08899736404418945,
      "sampling/sampling_logp_difference/mean": 0.009572825394570827,
      "step": 436,
      "step_time": 6.931109358000867
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1695.0,
      "completions/max_terminated_length": 1695.0,
      "completions/mean_length": 619.25,
      "completions/mean_terminated_length": 619.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3995943139307201,
      "epoch": 0.00437,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.18795931339263916,
      "kl": 0.3326401626691222,
      "learning_rate": 3.238748480282535e-05,
      "loss": -0.0024,
      "num_tokens": 10322290.0,
      "reward": 0.48968660831451416,
      "reward_std": 0.6016489267349243,
      "rewards/rollout_reward_func/mean": 0.48968660831451416,
      "rewards/rollout_reward_func/std": 0.7248833775520325,
      "sampling/importance_sampling_ratio/max": 1.0588446855545044,
      "sampling/importance_sampling_ratio/mean": 0.9969238042831421,
      "sampling/importance_sampling_ratio/min": 0.8997687101364136,
      "sampling/sampling_logp_difference/max": 0.1045675277709961,
      "sampling/sampling_logp_difference/mean": 0.0064063421450555325,
      "step": 437,
      "step_time": 9.720405116998336
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1725.0,
      "completions/max_terminated_length": 1725.0,
      "completions/mean_length": 791.0,
      "completions/mean_terminated_length": 791.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4634701292961836,
      "epoch": 0.00438,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19379381835460663,
      "kl": 0.4684780677780509,
      "learning_rate": 3.2317051917972236e-05,
      "loss": 0.0062,
      "num_tokens": 10350614.0,
      "reward": 0.6008954644203186,
      "reward_std": 0.5296896696090698,
      "rewards/rollout_reward_func/mean": 0.6008954644203186,
      "rewards/rollout_reward_func/std": 0.5322341918945312,
      "sampling/importance_sampling_ratio/max": 1.125968337059021,
      "sampling/importance_sampling_ratio/mean": 0.9945844411849976,
      "sampling/importance_sampling_ratio/min": 0.8976160287857056,
      "sampling/sampling_logp_difference/max": 0.11753511428833008,
      "sampling/sampling_logp_difference/mean": 0.012424886226654053,
      "step": 438,
      "step_time": 10.683527528999548
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2478.0,
      "completions/max_terminated_length": 2478.0,
      "completions/mean_length": 968.9375,
      "completions/mean_terminated_length": 968.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5428985860198736,
      "epoch": 0.00439,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.41139212250709534,
      "kl": 0.38896804116666317,
      "learning_rate": 3.224656966664981e-05,
      "loss": 0.017,
      "num_tokens": 10381152.0,
      "reward": 0.7565404176712036,
      "reward_std": 0.5371659994125366,
      "rewards/rollout_reward_func/mean": 0.7565404176712036,
      "rewards/rollout_reward_func/std": 0.5633623600006104,
      "sampling/importance_sampling_ratio/max": 1.1566650867462158,
      "sampling/importance_sampling_ratio/mean": 1.005142092704773,
      "sampling/importance_sampling_ratio/min": 0.8682038187980652,
      "sampling/sampling_logp_difference/max": 0.22256040573120117,
      "sampling/sampling_logp_difference/mean": 0.014667102135717869,
      "step": 439,
      "step_time": 12.476427356000386
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 879.0,
      "completions/max_terminated_length": 879.0,
      "completions/mean_length": 321.1875,
      "completions/mean_terminated_length": 321.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.30939739290624857,
      "epoch": 0.0044,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.10973487794399261,
      "kl": 0.41412969306111336,
      "learning_rate": 3.217603888889097e-05,
      "loss": 0.0034,
      "num_tokens": 10400035.0,
      "reward": 0.6041682958602905,
      "reward_std": 0.637215256690979,
      "rewards/rollout_reward_func/mean": 0.6041682958602905,
      "rewards/rollout_reward_func/std": 0.640823245048523,
      "sampling/importance_sampling_ratio/max": 1.1717854738235474,
      "sampling/importance_sampling_ratio/mean": 1.010106086730957,
      "sampling/importance_sampling_ratio/min": 0.915781557559967,
      "sampling/sampling_logp_difference/max": 0.0951378345489502,
      "sampling/sampling_logp_difference/mean": 0.007519927341490984,
      "step": 440,
      "step_time": 6.973591551000027
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1765.0,
      "completions/max_terminated_length": 1765.0,
      "completions/mean_length": 539.125,
      "completions/mean_terminated_length": 539.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4319627471268177,
      "epoch": 0.00441,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.15723076462745667,
      "kl": 0.4845624007284641,
      "learning_rate": 3.210546042530699e-05,
      "loss": -0.0009,
      "num_tokens": 10423211.0,
      "reward": 0.5017069578170776,
      "reward_std": 0.8204874992370605,
      "rewards/rollout_reward_func/mean": 0.5017069578170776,
      "rewards/rollout_reward_func/std": 0.8048226237297058,
      "sampling/importance_sampling_ratio/max": 1.1294875144958496,
      "sampling/importance_sampling_ratio/mean": 1.0048139095306396,
      "sampling/importance_sampling_ratio/min": 0.9263682961463928,
      "sampling/sampling_logp_difference/max": 0.10231828689575195,
      "sampling/sampling_logp_difference/mean": 0.00742411520332098,
      "step": 441,
      "step_time": 9.758478265000122
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2557.0,
      "completions/max_terminated_length": 2557.0,
      "completions/mean_length": 704.625,
      "completions/mean_terminated_length": 704.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3960277121514082,
      "epoch": 0.00442,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19722133874893188,
      "kl": 0.5285364650189877,
      "learning_rate": 3.2034835117077446e-05,
      "loss": -0.001,
      "num_tokens": 10449053.0,
      "reward": 0.611964762210846,
      "reward_std": 0.6286856532096863,
      "rewards/rollout_reward_func/mean": 0.611964762210846,
      "rewards/rollout_reward_func/std": 0.6336271166801453,
      "sampling/importance_sampling_ratio/max": 1.0159248113632202,
      "sampling/importance_sampling_ratio/mean": 0.984372079372406,
      "sampling/importance_sampling_ratio/min": 0.896038830280304,
      "sampling/sampling_logp_difference/max": 0.1097559928894043,
      "sampling/sampling_logp_difference/mean": 0.008366463705897331,
      "step": 442,
      "step_time": 12.120016617000147
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1871.0,
      "completions/max_terminated_length": 1871.0,
      "completions/mean_length": 503.5625,
      "completions/mean_terminated_length": 503.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.35645896196365356,
      "epoch": 0.00443,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09676048159599304,
      "kl": 0.5758144930005074,
      "learning_rate": 3.196416380594026e-05,
      "loss": -0.0044,
      "num_tokens": 10471696.0,
      "reward": 0.48678967356681824,
      "reward_std": 0.7430130839347839,
      "rewards/rollout_reward_func/mean": 0.48678967356681824,
      "rewards/rollout_reward_func/std": 0.7377403378486633,
      "sampling/importance_sampling_ratio/max": 1.061285138130188,
      "sampling/importance_sampling_ratio/mean": 1.0034221410751343,
      "sampling/importance_sampling_ratio/min": 0.9175456166267395,
      "sampling/sampling_logp_difference/max": 0.08666086196899414,
      "sampling/sampling_logp_difference/mean": 0.00644169794395566,
      "step": 443,
      "step_time": 10.094090317999871
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1594.0,
      "completions/max_terminated_length": 1594.0,
      "completions/mean_length": 422.25,
      "completions/mean_terminated_length": 422.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3728369418531656,
      "epoch": 0.00444,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.24413183331489563,
      "kl": 1.0407012477517128,
      "learning_rate": 3.189344733418161e-05,
      "loss": -0.0103,
      "num_tokens": 10492772.0,
      "reward": 0.37449973821640015,
      "reward_std": 0.8872742652893066,
      "rewards/rollout_reward_func/mean": 0.37449973821640015,
      "rewards/rollout_reward_func/std": 0.8751745820045471,
      "sampling/importance_sampling_ratio/max": 1.122226357460022,
      "sampling/importance_sampling_ratio/mean": 0.993268609046936,
      "sampling/importance_sampling_ratio/min": 0.8955214619636536,
      "sampling/sampling_logp_difference/max": 0.12016487121582031,
      "sampling/sampling_logp_difference/mean": 0.011948981322348118,
      "step": 444,
      "step_time": 9.728127580000546
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1594.0,
      "completions/max_terminated_length": 1594.0,
      "completions/mean_length": 326.625,
      "completions/mean_terminated_length": 326.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.24982158280909061,
      "epoch": 0.00445,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.17612352967262268,
      "kl": 0.33545841090381145,
      "learning_rate": 3.182268654462591e-05,
      "loss": -0.0096,
      "num_tokens": 10511286.0,
      "reward": 0.8054916262626648,
      "reward_std": 0.5501527786254883,
      "rewards/rollout_reward_func/mean": 0.8054916262626648,
      "rewards/rollout_reward_func/std": 0.5556681752204895,
      "sampling/importance_sampling_ratio/max": 1.12662672996521,
      "sampling/importance_sampling_ratio/mean": 1.0125939846038818,
      "sampling/importance_sampling_ratio/min": 0.9944058060646057,
      "sampling/sampling_logp_difference/max": 0.11894083023071289,
      "sampling/sampling_logp_difference/mean": 0.004844082985073328,
      "step": 445,
      "step_time": 9.013482557000316
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1672.0,
      "completions/max_terminated_length": 1672.0,
      "completions/mean_length": 458.4375,
      "completions/mean_terminated_length": 458.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.25894633028656244,
      "epoch": 0.00446,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3783493638038635,
      "kl": 0.9608308132737875,
      "learning_rate": 3.1751882280625824e-05,
      "loss": 0.0227,
      "num_tokens": 10532995.0,
      "reward": 0.681960940361023,
      "reward_std": 0.5536350011825562,
      "rewards/rollout_reward_func/mean": 0.681960940361023,
      "rewards/rollout_reward_func/std": 0.5983775854110718,
      "sampling/importance_sampling_ratio/max": 1.1449363231658936,
      "sampling/importance_sampling_ratio/mean": 1.0112711191177368,
      "sampling/importance_sampling_ratio/min": 0.9712468981742859,
      "sampling/sampling_logp_difference/max": 0.11303138732910156,
      "sampling/sampling_logp_difference/mean": 0.006588522810488939,
      "step": 446,
      "step_time": 9.45524376999856
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1796.0,
      "completions/max_terminated_length": 1796.0,
      "completions/mean_length": 718.375,
      "completions/mean_terminated_length": 718.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.39343155547976494,
      "epoch": 0.00447,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3993067443370819,
      "kl": 0.47657502070069313,
      "learning_rate": 3.1681035386052064e-05,
      "loss": 0.0015,
      "num_tokens": 10559602.0,
      "reward": 0.6337699294090271,
      "reward_std": 0.7977093458175659,
      "rewards/rollout_reward_func/mean": 0.6337699294090271,
      "rewards/rollout_reward_func/std": 0.7873764038085938,
      "sampling/importance_sampling_ratio/max": 1.0731083154678345,
      "sampling/importance_sampling_ratio/mean": 0.9930096864700317,
      "sampling/importance_sampling_ratio/min": 0.9139974117279053,
      "sampling/sampling_logp_difference/max": 0.10199332237243652,
      "sampling/sampling_logp_difference/mean": 0.008155551739037037,
      "step": 447,
      "step_time": 10.127079377999507
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2540.0,
      "completions/max_terminated_length": 2540.0,
      "completions/mean_length": 723.9375,
      "completions/mean_terminated_length": 723.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3699399419128895,
      "epoch": 0.00448,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13934378325939178,
      "kl": 0.4552036300301552,
      "learning_rate": 3.161014670528353e-05,
      "loss": -0.0011,
      "num_tokens": 10585960.0,
      "reward": 0.6801760196685791,
      "reward_std": 0.5648406744003296,
      "rewards/rollout_reward_func/mean": 0.6801760196685791,
      "rewards/rollout_reward_func/std": 0.6097784042358398,
      "sampling/importance_sampling_ratio/max": 1.0322799682617188,
      "sampling/importance_sampling_ratio/mean": 0.989983081817627,
      "sampling/importance_sampling_ratio/min": 0.9008469581604004,
      "sampling/sampling_logp_difference/max": 0.12268590927124023,
      "sampling/sampling_logp_difference/mean": 0.007343275472521782,
      "step": 448,
      "step_time": 12.259400646999893
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1595.0,
      "completions/max_terminated_length": 1595.0,
      "completions/mean_length": 595.1875,
      "completions/mean_terminated_length": 595.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3214338351972401,
      "epoch": 0.00449,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21718402206897736,
      "kl": 0.6500730104744434,
      "learning_rate": 3.153921708319707e-05,
      "loss": -0.0033,
      "num_tokens": 10610510.0,
      "reward": 0.5954734683036804,
      "reward_std": 0.5381704568862915,
      "rewards/rollout_reward_func/mean": 0.5954734683036804,
      "rewards/rollout_reward_func/std": 0.5394914746284485,
      "sampling/importance_sampling_ratio/max": 1.04869544506073,
      "sampling/importance_sampling_ratio/mean": 0.9938335418701172,
      "sampling/importance_sampling_ratio/min": 0.9132229089736938,
      "sampling/sampling_logp_difference/max": 0.0908811092376709,
      "sampling/sampling_logp_difference/mean": 0.005706118419766426,
      "step": 449,
      "step_time": 9.498276508002164
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 888.0,
      "completions/max_terminated_length": 888.0,
      "completions/mean_length": 521.125,
      "completions/mean_terminated_length": 521.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.29106704145669937,
      "epoch": 0.0045,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07752793282270432,
      "kl": 0.6062231417745352,
      "learning_rate": 3.1468247365157537e-05,
      "loss": 0.0113,
      "num_tokens": 10633366.0,
      "reward": 0.734040379524231,
      "reward_std": 0.46200108528137207,
      "rewards/rollout_reward_func/mean": 0.734040379524231,
      "rewards/rollout_reward_func/std": 0.47582289576530457,
      "sampling/importance_sampling_ratio/max": 1.053429365158081,
      "sampling/importance_sampling_ratio/mean": 1.0073636770248413,
      "sampling/importance_sampling_ratio/min": 0.9625693559646606,
      "sampling/sampling_logp_difference/max": 0.052910685539245605,
      "sampling/sampling_logp_difference/mean": 0.0039354413747787476,
      "step": 450,
      "step_time": 7.367297488999611
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2585.0,
      "completions/max_terminated_length": 2585.0,
      "completions/mean_length": 589.9375,
      "completions/mean_terminated_length": 589.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.27843848802149296,
      "epoch": 0.00451,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.10579805076122284,
      "kl": 0.591363525018096,
      "learning_rate": 3.13972383970076e-05,
      "loss": -0.0051,
      "num_tokens": 10657553.0,
      "reward": 0.5973840355873108,
      "reward_std": 0.5396814346313477,
      "rewards/rollout_reward_func/mean": 0.5973840355873108,
      "rewards/rollout_reward_func/std": 0.5375606417655945,
      "sampling/importance_sampling_ratio/max": 1.1048179864883423,
      "sampling/importance_sampling_ratio/mean": 1.01421058177948,
      "sampling/importance_sampling_ratio/min": 0.9789706468582153,
      "sampling/sampling_logp_difference/max": 0.09961462020874023,
      "sampling/sampling_logp_difference/mean": 0.006096962373703718,
      "step": 451,
      "step_time": 12.548346794998906
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 924.0,
      "completions/max_terminated_length": 924.0,
      "completions/mean_length": 455.0625,
      "completions/mean_terminated_length": 455.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2486275399569422,
      "epoch": 0.00452,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.16621743142604828,
      "kl": 0.38327374309301376,
      "learning_rate": 3.1326191025057796e-05,
      "loss": -0.0012,
      "num_tokens": 10679628.0,
      "reward": 0.48787960410118103,
      "reward_std": 0.7393980026245117,
      "rewards/rollout_reward_func/mean": 0.48787960410118103,
      "rewards/rollout_reward_func/std": 0.7281366586685181,
      "sampling/importance_sampling_ratio/max": 1.0333303213119507,
      "sampling/importance_sampling_ratio/mean": 0.9994915723800659,
      "sampling/importance_sampling_ratio/min": 0.9656228423118591,
      "sampling/sampling_logp_difference/max": 0.03528857231140137,
      "sampling/sampling_logp_difference/mean": 0.0021643631625920534,
      "step": 452,
      "step_time": 7.397417303000111
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1862.0,
      "completions/max_terminated_length": 1862.0,
      "completions/mean_length": 635.0625,
      "completions/mean_terminated_length": 635.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.30012185184750706,
      "epoch": 0.00453,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12199398875236511,
      "kl": 0.4916687663644552,
      "learning_rate": 3.1255106096076326e-05,
      "loss": 0.0196,
      "num_tokens": 10704613.0,
      "reward": 0.6028099060058594,
      "reward_std": 0.5483492612838745,
      "rewards/rollout_reward_func/mean": 0.6028099060058594,
      "rewards/rollout_reward_func/std": 0.5302600860595703,
      "sampling/importance_sampling_ratio/max": 1.0601626634597778,
      "sampling/importance_sampling_ratio/mean": 0.9911554455757141,
      "sampling/importance_sampling_ratio/min": 0.910920262336731,
      "sampling/sampling_logp_difference/max": 0.09370625019073486,
      "sampling/sampling_logp_difference/mean": 0.00646995659917593,
      "step": 453,
      "step_time": 10.415251684002214
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2666.0,
      "completions/max_terminated_length": 2666.0,
      "completions/mean_length": 536.625,
      "completions/mean_terminated_length": 536.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2931267889216542,
      "epoch": 0.00454,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09525609761476517,
      "kl": 0.40606039576232433,
      "learning_rate": 3.1183984457279025e-05,
      "loss": -0.0023,
      "num_tokens": 10728140.0,
      "reward": 0.6139886975288391,
      "reward_std": 0.6463134288787842,
      "rewards/rollout_reward_func/mean": 0.6139886975288391,
      "rewards/rollout_reward_func/std": 0.6295685172080994,
      "sampling/importance_sampling_ratio/max": 1.1209286451339722,
      "sampling/importance_sampling_ratio/mean": 1.0096750259399414,
      "sampling/importance_sampling_ratio/min": 0.9127695560455322,
      "sampling/sampling_logp_difference/max": 0.1138010025024414,
      "sampling/sampling_logp_difference/mean": 0.00917104259133339,
      "step": 454,
      "step_time": 12.122162454001227
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1781.0,
      "completions/max_terminated_length": 1781.0,
      "completions/mean_length": 622.125,
      "completions/mean_terminated_length": 622.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2708528582006693,
      "epoch": 0.00455,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08852022886276245,
      "kl": 0.5084255672991276,
      "learning_rate": 3.1112826956319255e-05,
      "loss": 0.0005,
      "num_tokens": 10753153.0,
      "reward": 0.6635453104972839,
      "reward_std": 0.5279179811477661,
      "rewards/rollout_reward_func/mean": 0.6635453104972839,
      "rewards/rollout_reward_func/std": 0.5156021118164062,
      "sampling/importance_sampling_ratio/max": 1.0883269309997559,
      "sampling/importance_sampling_ratio/mean": 0.9852510094642639,
      "sampling/importance_sampling_ratio/min": 0.9120153188705444,
      "sampling/sampling_logp_difference/max": 0.09211289882659912,
      "sampling/sampling_logp_difference/mean": 0.007978013716638088,
      "step": 455,
      "step_time": 10.01601370300068
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 889.0,
      "completions/max_terminated_length": 889.0,
      "completions/mean_length": 505.3125,
      "completions/mean_terminated_length": 505.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2191436756402254,
      "epoch": 0.00456,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.04365120083093643,
      "kl": 0.6040051579475403,
      "learning_rate": 3.1041634441277776e-05,
      "loss": 0.0033,
      "num_tokens": 10776086.0,
      "reward": 0.7299801111221313,
      "reward_std": 0.5000938177108765,
      "rewards/rollout_reward_func/mean": 0.7299801111221313,
      "rewards/rollout_reward_func/std": 0.48314139246940613,
      "sampling/importance_sampling_ratio/max": 1.0356804132461548,
      "sampling/importance_sampling_ratio/mean": 1.009040117263794,
      "sampling/importance_sampling_ratio/min": 0.9937456846237183,
      "sampling/sampling_logp_difference/max": 0.03521168231964111,
      "sampling/sampling_logp_difference/mean": 0.0031130495481193066,
      "step": 456,
      "step_time": 7.469605889000377
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 921.0,
      "completions/max_terminated_length": 921.0,
      "completions/mean_length": 472.8125,
      "completions/mean_terminated_length": 472.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.37743410281836987,
      "epoch": 0.00457,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21087169647216797,
      "kl": 0.5517049618065357,
      "learning_rate": 3.097040776065267e-05,
      "loss": 0.002,
      "num_tokens": 10798500.0,
      "reward": 0.5982781648635864,
      "reward_std": 0.5360516905784607,
      "rewards/rollout_reward_func/mean": 0.5982781648635864,
      "rewards/rollout_reward_func/std": 0.5360774993896484,
      "sampling/importance_sampling_ratio/max": 1.0353015661239624,
      "sampling/importance_sampling_ratio/mean": 0.9965384602546692,
      "sampling/importance_sampling_ratio/min": 0.9457602500915527,
      "sampling/sampling_logp_difference/max": 0.05585587024688721,
      "sampling/sampling_logp_difference/mean": 0.0035816386807709932,
      "step": 457,
      "step_time": 8.171203146000153
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 835.0,
      "completions/max_terminated_length": 835.0,
      "completions/mean_length": 359.75,
      "completions/mean_terminated_length": 359.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.19167738407850266,
      "epoch": 0.00458,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.0403437539935112,
      "kl": 0.5838537439703941,
      "learning_rate": 3.089914776334923e-05,
      "loss": 0.0002,
      "num_tokens": 10818288.0,
      "reward": 0.7935810089111328,
      "reward_std": 0.28493279218673706,
      "rewards/rollout_reward_func/mean": 0.7935810089111328,
      "rewards/rollout_reward_func/std": 0.44384443759918213,
      "sampling/importance_sampling_ratio/max": 1.0056747198104858,
      "sampling/importance_sampling_ratio/mean": 0.997326135635376,
      "sampling/importance_sampling_ratio/min": 0.97308748960495,
      "sampling/sampling_logp_difference/max": 0.02726045250892639,
      "sampling/sampling_logp_difference/mean": 0.0013909258414059877,
      "step": 458,
      "step_time": 6.921414386999459
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1834.0,
      "completions/max_terminated_length": 1834.0,
      "completions/mean_length": 607.1875,
      "completions/mean_terminated_length": 607.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.27309711277484894,
      "epoch": 0.00459,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2018127292394638,
      "kl": 1.2838031463325024,
      "learning_rate": 3.08278552986698e-05,
      "loss": 0.011,
      "num_tokens": 10843062.0,
      "reward": 0.46137696504592896,
      "reward_std": 0.5763552188873291,
      "rewards/rollout_reward_func/mean": 0.46137696504592896,
      "rewards/rollout_reward_func/std": 0.5570283532142639,
      "sampling/importance_sampling_ratio/max": 1.126406192779541,
      "sampling/importance_sampling_ratio/mean": 0.9889816045761108,
      "sampling/importance_sampling_ratio/min": 0.9063283801078796,
      "sampling/sampling_logp_difference/max": 0.12940597534179688,
      "sampling/sampling_logp_difference/mean": 0.009202023968100548,
      "step": 459,
      "step_time": 10.084629196998321
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1754.0,
      "completions/max_terminated_length": 1754.0,
      "completions/mean_length": 351.0,
      "completions/mean_terminated_length": 351.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2282143523916602,
      "epoch": 0.0046,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.033584363758563995,
      "kl": 0.46837919391691685,
      "learning_rate": 3.07565312163037e-05,
      "loss": -0.0006,
      "num_tokens": 10862619.0,
      "reward": 0.6565725803375244,
      "reward_std": 0.5384612083435059,
      "rewards/rollout_reward_func/mean": 0.6565725803375244,
      "rewards/rollout_reward_func/std": 0.5262130498886108,
      "sampling/importance_sampling_ratio/max": 1.0350985527038574,
      "sampling/importance_sampling_ratio/mean": 1.001993179321289,
      "sampling/importance_sampling_ratio/min": 0.9724178910255432,
      "sampling/sampling_logp_difference/max": 0.03391626477241516,
      "sampling/sampling_logp_difference/mean": 0.001995101338252425,
      "step": 460,
      "step_time": 9.503591574999518
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2781.0,
      "completions/max_terminated_length": 2781.0,
      "completions/mean_length": 566.375,
      "completions/mean_terminated_length": 566.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.36179034784436226,
      "epoch": 0.00461,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4191066324710846,
      "kl": 0.5080958530306816,
      "learning_rate": 3.0685176366317094e-05,
      "loss": 0.0044,
      "num_tokens": 10885766.0,
      "reward": 0.6827026009559631,
      "reward_std": 0.6044751405715942,
      "rewards/rollout_reward_func/mean": 0.6827026009559631,
      "rewards/rollout_reward_func/std": 0.6022382974624634,
      "sampling/importance_sampling_ratio/max": 1.4468339681625366,
      "sampling/importance_sampling_ratio/mean": 1.0204672813415527,
      "sampling/importance_sampling_ratio/min": 0.7811205387115479,
      "sampling/sampling_logp_difference/max": 0.30565547943115234,
      "sampling/sampling_logp_difference/mean": 0.014012224040925503,
      "step": 461,
      "step_time": 12.579027792000488
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1716.0,
      "completions/max_terminated_length": 1716.0,
      "completions/mean_length": 638.8125,
      "completions/mean_terminated_length": 638.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.27388443797826767,
      "epoch": 0.00462,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2050049901008606,
      "kl": 0.6383340694010258,
      "learning_rate": 3.061379159914282e-05,
      "loss": -0.0026,
      "num_tokens": 10911178.0,
      "reward": 0.6780573129653931,
      "reward_std": 0.5719543695449829,
      "rewards/rollout_reward_func/mean": 0.6780573129653931,
      "rewards/rollout_reward_func/std": 0.6136854290962219,
      "sampling/importance_sampling_ratio/max": 1.1009705066680908,
      "sampling/importance_sampling_ratio/mean": 0.9731874465942383,
      "sampling/importance_sampling_ratio/min": 0.7635341286659241,
      "sampling/sampling_logp_difference/max": 0.24970054626464844,
      "sampling/sampling_logp_difference/mean": 0.012952400371432304,
      "step": 462,
      "step_time": 9.762738440999783
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1853.0,
      "completions/max_terminated_length": 1853.0,
      "completions/mean_length": 660.8125,
      "completions/mean_terminated_length": 660.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3617057390511036,
      "epoch": 0.00463,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19244523346424103,
      "kl": 0.6350223366171122,
      "learning_rate": 3.054237776557029e-05,
      "loss": 0.0129,
      "num_tokens": 10935959.0,
      "reward": 0.6026557683944702,
      "reward_std": 0.5301851034164429,
      "rewards/rollout_reward_func/mean": 0.6026557683944702,
      "rewards/rollout_reward_func/std": 0.5303432941436768,
      "sampling/importance_sampling_ratio/max": 1.1785143613815308,
      "sampling/importance_sampling_ratio/mean": 1.0003212690353394,
      "sampling/importance_sampling_ratio/min": 0.9001795053482056,
      "sampling/sampling_logp_difference/max": 0.16413408517837524,
      "sampling/sampling_logp_difference/mean": 0.009435310028493404,
      "step": 463,
      "step_time": 10.24572519099911
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1727.0,
      "completions/max_terminated_length": 1727.0,
      "completions/mean_length": 465.25,
      "completions/mean_terminated_length": 465.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.23932817508466542,
      "epoch": 0.00464,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1984550654888153,
      "kl": 0.5065030232071877,
      "learning_rate": 3.0470935716735352e-05,
      "loss": 0.0101,
      "num_tokens": 10957507.0,
      "reward": 0.6661905646324158,
      "reward_std": 0.5226671695709229,
      "rewards/rollout_reward_func/mean": 0.6661905646324158,
      "rewards/rollout_reward_func/std": 0.5115065574645996,
      "sampling/importance_sampling_ratio/max": 1.0952460765838623,
      "sampling/importance_sampling_ratio/mean": 1.0090937614440918,
      "sampling/importance_sampling_ratio/min": 0.975307047367096,
      "sampling/sampling_logp_difference/max": 0.09193277359008789,
      "sampling/sampling_logp_difference/mean": 0.004220840986818075,
      "step": 464,
      "step_time": 9.943945717000133
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 871.0,
      "completions/max_terminated_length": 871.0,
      "completions/mean_length": 433.3125,
      "completions/mean_terminated_length": 433.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.35958707612007856,
      "epoch": 0.00465,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.23734702169895172,
      "kl": 0.5193411931395531,
      "learning_rate": 3.0399466304110103e-05,
      "loss": 0.0102,
      "num_tokens": 10979337.0,
      "reward": 0.5939549207687378,
      "reward_std": 0.4753900170326233,
      "rewards/rollout_reward_func/mean": 0.5939549207687378,
      "rewards/rollout_reward_func/std": 0.541564404964447,
      "sampling/importance_sampling_ratio/max": 1.0300029516220093,
      "sampling/importance_sampling_ratio/mean": 0.9962567090988159,
      "sampling/importance_sampling_ratio/min": 0.93528151512146,
      "sampling/sampling_logp_difference/max": 0.11617636680603027,
      "sampling/sampling_logp_difference/mean": 0.008209389634430408,
      "step": 465,
      "step_time": 7.227615787000104
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1718.0,
      "completions/max_terminated_length": 1718.0,
      "completions/mean_length": 297.375,
      "completions/mean_terminated_length": 297.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.1956328279338777,
      "epoch": 0.00466,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06402324885129929,
      "kl": 0.379206663928926,
      "learning_rate": 3.03279703794928e-05,
      "loss": -0.0029,
      "num_tokens": 10997419.0,
      "reward": 0.7264612317085266,
      "reward_std": 0.5066505670547485,
      "rewards/rollout_reward_func/mean": 0.7264612317085266,
      "rewards/rollout_reward_func/std": 0.4895162284374237,
      "sampling/importance_sampling_ratio/max": 1.0297125577926636,
      "sampling/importance_sampling_ratio/mean": 0.9965736269950867,
      "sampling/importance_sampling_ratio/min": 0.9092161059379578,
      "sampling/sampling_logp_difference/max": 0.09550964832305908,
      "sampling/sampling_logp_difference/mean": 0.004427665378898382,
      "step": 466,
      "step_time": 9.259269373998904
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 897.0,
      "completions/max_terminated_length": 897.0,
      "completions/mean_length": 381.875,
      "completions/mean_terminated_length": 381.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.32912438176572323,
      "epoch": 0.00467,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12597137689590454,
      "kl": 0.5288605503737926,
      "learning_rate": 3.0256448794997676e-05,
      "loss": -0.0044,
      "num_tokens": 11018051.0,
      "reward": 0.4857179522514343,
      "reward_std": 0.692263662815094,
      "rewards/rollout_reward_func/mean": 0.4857179522514343,
      "rewards/rollout_reward_func/std": 0.7315097451210022,
      "sampling/importance_sampling_ratio/max": 1.1183124780654907,
      "sampling/importance_sampling_ratio/mean": 1.0065395832061768,
      "sampling/importance_sampling_ratio/min": 0.9685872197151184,
      "sampling/sampling_logp_difference/max": 0.11210870742797852,
      "sampling/sampling_logp_difference/mean": 0.004943110514432192,
      "step": 467,
      "step_time": 7.22201491100077
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 888.0,
      "completions/max_terminated_length": 888.0,
      "completions/mean_length": 433.875,
      "completions/mean_terminated_length": 433.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.20487300539389253,
      "epoch": 0.00468,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0803055390715599,
      "kl": 0.8999164327979088,
      "learning_rate": 3.0184902403044773e-05,
      "loss": 0.0118,
      "num_tokens": 11039444.0,
      "reward": 0.46068882942199707,
      "reward_std": 0.5599832534790039,
      "rewards/rollout_reward_func/mean": 0.46068882942199707,
      "rewards/rollout_reward_func/std": 0.557233989238739,
      "sampling/importance_sampling_ratio/max": 1.0275917053222656,
      "sampling/importance_sampling_ratio/mean": 0.9994294047355652,
      "sampling/importance_sampling_ratio/min": 0.9788061380386353,
      "sampling/sampling_logp_difference/max": 0.027462005615234375,
      "sampling/sampling_logp_difference/mean": 0.0018193733412772417,
      "step": 468,
      "step_time": 7.121563516999231
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1808.0,
      "completions/max_terminated_length": 1808.0,
      "completions/mean_length": 536.9375,
      "completions/mean_terminated_length": 536.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.24697526078671217,
      "epoch": 0.00469,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09931109100580215,
      "kl": 0.5640309322625399,
      "learning_rate": 3.0113332056349805e-05,
      "loss": -0.0055,
      "num_tokens": 11063050.0,
      "reward": 0.6063300371170044,
      "reward_std": 0.652036190032959,
      "rewards/rollout_reward_func/mean": 0.6063300371170044,
      "rewards/rollout_reward_func/std": 0.6371240019798279,
      "sampling/importance_sampling_ratio/max": 1.0068272352218628,
      "sampling/importance_sampling_ratio/mean": 0.9929063320159912,
      "sampling/importance_sampling_ratio/min": 0.876896858215332,
      "sampling/sampling_logp_difference/max": 0.10145020484924316,
      "sampling/sampling_logp_difference/mean": 0.0032098719384521246,
      "step": 469,
      "step_time": 9.849152699000115
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1700.0,
      "completions/max_terminated_length": 1700.0,
      "completions/mean_length": 525.625,
      "completions/mean_terminated_length": 525.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2653265818953514,
      "epoch": 0.0047,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.17212632298469543,
      "kl": 0.8414570651948452,
      "learning_rate": 3.0041738607913996e-05,
      "loss": -0.0001,
      "num_tokens": 11085798.0,
      "reward": 0.7957499027252197,
      "reward_std": 0.446486234664917,
      "rewards/rollout_reward_func/mean": 0.7957499027252197,
      "rewards/rollout_reward_func/std": 0.43923041224479675,
      "sampling/importance_sampling_ratio/max": 1.0255687236785889,
      "sampling/importance_sampling_ratio/mean": 0.9937736392021179,
      "sampling/importance_sampling_ratio/min": 0.9037137627601624,
      "sampling/sampling_logp_difference/max": 0.1011962890625,
      "sampling/sampling_logp_difference/mean": 0.0039848629385232925,
      "step": 470,
      "step_time": 9.56664208499933
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1628.0,
      "completions/max_terminated_length": 1628.0,
      "completions/mean_length": 362.8125,
      "completions/mean_terminated_length": 362.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.19789692875929177,
      "epoch": 0.00471,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.08625102788209915,
      "kl": 0.4052835311740637,
      "learning_rate": 2.9970122911013892e-05,
      "loss": -0.0121,
      "num_tokens": 11105675.0,
      "reward": 0.6550778150558472,
      "reward_std": 0.5418446660041809,
      "rewards/rollout_reward_func/mean": 0.6550778150558472,
      "rewards/rollout_reward_func/std": 0.5284820199012756,
      "sampling/importance_sampling_ratio/max": 1.0604616403579712,
      "sampling/importance_sampling_ratio/mean": 0.9978777170181274,
      "sampling/importance_sampling_ratio/min": 0.9036425352096558,
      "sampling/sampling_logp_difference/max": 0.10098659992218018,
      "sampling/sampling_logp_difference/mean": 0.005723956972360611,
      "step": 471,
      "step_time": 9.654548798999713
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 883.0,
      "completions/max_terminated_length": 883.0,
      "completions/mean_length": 393.375,
      "completions/mean_terminated_length": 393.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2638454120606184,
      "epoch": 0.00472,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19675129652023315,
      "kl": 0.8003968987613916,
      "learning_rate": 2.9898485819191216e-05,
      "loss": 0.0026,
      "num_tokens": 11126253.0,
      "reward": 0.793187141418457,
      "reward_std": 0.45129817724227905,
      "rewards/rollout_reward_func/mean": 0.793187141418457,
      "rewards/rollout_reward_func/std": 0.44474148750305176,
      "sampling/importance_sampling_ratio/max": 1.1112630367279053,
      "sampling/importance_sampling_ratio/mean": 1.0125417709350586,
      "sampling/importance_sampling_ratio/min": 0.9659556150436401,
      "sampling/sampling_logp_difference/max": 0.1054086685180664,
      "sampling/sampling_logp_difference/mean": 0.005088849924504757,
      "step": 472,
      "step_time": 7.1669169350007
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1622.0,
      "completions/max_terminated_length": 1622.0,
      "completions/mean_length": 432.6875,
      "completions/mean_terminated_length": 432.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.25456054392270744,
      "epoch": 0.00473,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21776866912841797,
      "kl": 0.7343901842832565,
      "learning_rate": 2.9826828186242664e-05,
      "loss": -0.0146,
      "num_tokens": 11147725.0,
      "reward": 0.5508513450622559,
      "reward_std": 0.7491663694381714,
      "rewards/rollout_reward_func/mean": 0.5508513450622559,
      "rewards/rollout_reward_func/std": 0.7270815372467041,
      "sampling/importance_sampling_ratio/max": 1.0230258703231812,
      "sampling/importance_sampling_ratio/mean": 0.9990797638893127,
      "sampling/importance_sampling_ratio/min": 0.9478112459182739,
      "sampling/sampling_logp_difference/max": 0.053668081760406494,
      "sampling/sampling_logp_difference/mean": 0.0020584899466484785,
      "step": 473,
      "step_time": 9.282134917000803
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 876.0,
      "completions/max_terminated_length": 876.0,
      "completions/mean_length": 427.0,
      "completions/mean_terminated_length": 427.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2208827887661755,
      "epoch": 0.00474,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1281779259443283,
      "kl": 0.680979561060667,
      "learning_rate": 2.9755150866209776e-05,
      "loss": -0.0029,
      "num_tokens": 11168895.0,
      "reward": 0.48487257957458496,
      "reward_std": 0.7589366436004639,
      "rewards/rollout_reward_func/mean": 0.48487257957458496,
      "rewards/rollout_reward_func/std": 0.7332107424736023,
      "sampling/importance_sampling_ratio/max": 1.0502159595489502,
      "sampling/importance_sampling_ratio/mean": 0.9954082369804382,
      "sampling/importance_sampling_ratio/min": 0.8893297910690308,
      "sampling/sampling_logp_difference/max": 0.11800646781921387,
      "sampling/sampling_logp_difference/mean": 0.004046901594847441,
      "step": 474,
      "step_time": 7.245219137999811
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1513.0,
      "completions/max_terminated_length": 1513.0,
      "completions/mean_length": 278.875,
      "completions/mean_terminated_length": 278.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.29805281991139054,
      "epoch": 0.00475,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07801715284585953,
      "kl": 0.3500353042036295,
      "learning_rate": 2.96834547133687e-05,
      "loss": -0.0049,
      "num_tokens": 11187333.0,
      "reward": 0.533166229724884,
      "reward_std": 0.5584126114845276,
      "rewards/rollout_reward_func/mean": 0.533166229724884,
      "rewards/rollout_reward_func/std": 0.6566128730773926,
      "sampling/importance_sampling_ratio/max": 1.0266343355178833,
      "sampling/importance_sampling_ratio/mean": 0.9914673566818237,
      "sampling/importance_sampling_ratio/min": 0.9343881607055664,
      "sampling/sampling_logp_difference/max": 0.06757158041000366,
      "sampling/sampling_logp_difference/mean": 0.0055501121096313,
      "step": 475,
      "step_time": 8.706869254000594
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1827.0,
      "completions/max_terminated_length": 1827.0,
      "completions/mean_length": 726.125,
      "completions/mean_terminated_length": 726.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3740527951158583,
      "epoch": 0.00476,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2065046727657318,
      "kl": 0.8832170814275742,
      "learning_rate": 2.9611740582220056e-05,
      "loss": 0.0203,
      "num_tokens": 11213321.0,
      "reward": 0.5489984154701233,
      "reward_std": 0.5376152396202087,
      "rewards/rollout_reward_func/mean": 0.5489984154701233,
      "rewards/rollout_reward_func/std": 0.6325501203536987,
      "sampling/importance_sampling_ratio/max": 1.017224907875061,
      "sampling/importance_sampling_ratio/mean": 0.9906054139137268,
      "sampling/importance_sampling_ratio/min": 0.9042195081710815,
      "sampling/sampling_logp_difference/max": 0.1395587921142578,
      "sampling/sampling_logp_difference/mean": 0.011641117744147778,
      "step": 476,
      "step_time": 10.249249483001222
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1852.0,
      "completions/max_terminated_length": 1852.0,
      "completions/mean_length": 681.8125,
      "completions/mean_terminated_length": 681.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3648262545466423,
      "epoch": 0.00477,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1596682369709015,
      "kl": 0.7570227608084679,
      "learning_rate": 2.9540009327478756e-05,
      "loss": 0.0085,
      "num_tokens": 11239280.0,
      "reward": 0.6839233040809631,
      "reward_std": 0.5476219058036804,
      "rewards/rollout_reward_func/mean": 0.6839233040809631,
      "rewards/rollout_reward_func/std": 0.6016260385513306,
      "sampling/importance_sampling_ratio/max": 1.0113906860351562,
      "sampling/importance_sampling_ratio/mean": 0.9809480905532837,
      "sampling/importance_sampling_ratio/min": 0.8950875401496887,
      "sampling/sampling_logp_difference/max": 0.1863698959350586,
      "sampling/sampling_logp_difference/mean": 0.00958539079874754,
      "step": 477,
      "step_time": 10.35044934599864
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1774.0,
      "completions/max_terminated_length": 1774.0,
      "completions/mean_length": 773.125,
      "completions/mean_terminated_length": 773.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.25784018728882074,
      "epoch": 0.00478,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.18981525301933289,
      "kl": 1.171007014811039,
      "learning_rate": 2.9468261804063762e-05,
      "loss": 0.0018,
      "num_tokens": 11266868.0,
      "reward": 0.5971699953079224,
      "reward_std": 0.5561882257461548,
      "rewards/rollout_reward_func/mean": 0.5971699953079224,
      "rewards/rollout_reward_func/std": 0.5374325513839722,
      "sampling/importance_sampling_ratio/max": 1.07571542263031,
      "sampling/importance_sampling_ratio/mean": 1.0071942806243896,
      "sampling/importance_sampling_ratio/min": 0.9657902717590332,
      "sampling/sampling_logp_difference/max": 0.07151913642883301,
      "sampling/sampling_logp_difference/mean": 0.004449327476322651,
      "step": 478,
      "step_time": 10.678410506999171
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1638.0,
      "completions/max_terminated_length": 1638.0,
      "completions/mean_length": 559.5625,
      "completions/mean_terminated_length": 559.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.36310521978884935,
      "epoch": 0.00479,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2382454127073288,
      "kl": 0.5627627987414598,
      "learning_rate": 2.9396498867087936e-05,
      "loss": 0.0033,
      "num_tokens": 11290426.0,
      "reward": 0.731672465801239,
      "reward_std": 0.49687087535858154,
      "rewards/rollout_reward_func/mean": 0.731672465801239,
      "rewards/rollout_reward_func/std": 0.48004651069641113,
      "sampling/importance_sampling_ratio/max": 1.0144338607788086,
      "sampling/importance_sampling_ratio/mean": 0.9841575622558594,
      "sampling/importance_sampling_ratio/min": 0.8955414891242981,
      "sampling/sampling_logp_difference/max": 0.1147618293762207,
      "sampling/sampling_logp_difference/mean": 0.010175847448408604,
      "step": 479,
      "step_time": 9.543352094000511
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 897.0,
      "completions/max_terminated_length": 897.0,
      "completions/mean_length": 380.375,
      "completions/mean_terminated_length": 380.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2825573361478746,
      "epoch": 0.0048,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12068559974431992,
      "kl": 0.37306166999042034,
      "learning_rate": 2.932472137184788e-05,
      "loss": 0.0113,
      "num_tokens": 11310515.0,
      "reward": 0.6658293008804321,
      "reward_std": 0.5239212512969971,
      "rewards/rollout_reward_func/mean": 0.6658293008804321,
      "rewards/rollout_reward_func/std": 0.5121561288833618,
      "sampling/importance_sampling_ratio/max": 1.0368759632110596,
      "sampling/importance_sampling_ratio/mean": 0.9973917007446289,
      "sampling/importance_sampling_ratio/min": 0.934420108795166,
      "sampling/sampling_logp_difference/max": 0.06790757179260254,
      "sampling/sampling_logp_difference/mean": 0.0039704847149550915,
      "step": 480,
      "step_time": 7.218032012002368
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1684.0,
      "completions/max_terminated_length": 1684.0,
      "completions/mean_length": 293.375,
      "completions/mean_terminated_length": 293.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2952017346397042,
      "epoch": 0.00481,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.10624662786722183,
      "kl": 0.7442349530756474,
      "learning_rate": 2.9252930173813656e-05,
      "loss": 0.0037,
      "num_tokens": 11328762.0,
      "reward": 0.6721348166465759,
      "reward_std": 0.3756864666938782,
      "rewards/rollout_reward_func/mean": 0.6721348166465759,
      "rewards/rollout_reward_func/std": 0.6149177551269531,
      "sampling/importance_sampling_ratio/max": 1.060102939605713,
      "sampling/importance_sampling_ratio/mean": 1.0022683143615723,
      "sampling/importance_sampling_ratio/min": 0.952560305595398,
      "sampling/sampling_logp_difference/max": 0.05830943584442139,
      "sampling/sampling_logp_difference/mean": 0.0037803347222507,
      "step": 481,
      "step_time": 9.243053170998792
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1682.0,
      "completions/max_terminated_length": 1682.0,
      "completions/mean_length": 588.125,
      "completions/mean_terminated_length": 588.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.26572230784222484,
      "epoch": 0.00482,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.14995980262756348,
      "kl": 0.6730191111564636,
      "learning_rate": 2.918112612861868e-05,
      "loss": -0.0015,
      "num_tokens": 11352830.0,
      "reward": 0.6587452292442322,
      "reward_std": 0.5348190069198608,
      "rewards/rollout_reward_func/mean": 0.6587452292442322,
      "rewards/rollout_reward_func/std": 0.5229492783546448,
      "sampling/importance_sampling_ratio/max": 1.0119738578796387,
      "sampling/importance_sampling_ratio/mean": 0.9845262169837952,
      "sampling/importance_sampling_ratio/min": 0.8844319581985474,
      "sampling/sampling_logp_difference/max": 0.12325763702392578,
      "sampling/sampling_logp_difference/mean": 0.006315275095403194,
      "step": 482,
      "step_time": 9.618760705001478
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1853.0,
      "completions/max_terminated_length": 1853.0,
      "completions/mean_length": 873.4375,
      "completions/mean_terminated_length": 873.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4152180226519704,
      "epoch": 0.00483,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12088362872600555,
      "kl": 0.5869662202894688,
      "learning_rate": 2.910931009204947e-05,
      "loss": 0.0103,
      "num_tokens": 11381827.0,
      "reward": 0.6737349033355713,
      "reward_std": 0.5131717920303345,
      "rewards/rollout_reward_func/mean": 0.6737349033355713,
      "rewards/rollout_reward_func/std": 0.5000855326652527,
      "sampling/importance_sampling_ratio/max": 1.0684196949005127,
      "sampling/importance_sampling_ratio/mean": 1.001600980758667,
      "sampling/importance_sampling_ratio/min": 0.9336187243461609,
      "sampling/sampling_logp_difference/max": 0.07038640975952148,
      "sampling/sampling_logp_difference/mean": 0.0058450098149478436,
      "step": 483,
      "step_time": 10.556272729998454
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1823.0,
      "completions/max_terminated_length": 1823.0,
      "completions/mean_length": 551.0,
      "completions/mean_terminated_length": 551.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.34682678151875734,
      "epoch": 0.00484,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1133890300989151,
      "kl": 0.6447533294558525,
      "learning_rate": 2.9037482920035463e-05,
      "loss": 0.0005,
      "num_tokens": 11405283.0,
      "reward": 0.3731769323348999,
      "reward_std": 0.8178714513778687,
      "rewards/rollout_reward_func/mean": 0.3731769323348999,
      "rewards/rollout_reward_func/std": 0.7929372787475586,
      "sampling/importance_sampling_ratio/max": 1.0660390853881836,
      "sampling/importance_sampling_ratio/mean": 1.0018091201782227,
      "sampling/importance_sampling_ratio/min": 0.9166467189788818,
      "sampling/sampling_logp_difference/max": 0.11343050003051758,
      "sampling/sampling_logp_difference/mean": 0.004962247330695391,
      "step": 484,
      "step_time": 10.420857978998356
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1817.0,
      "completions/max_terminated_length": 1817.0,
      "completions/mean_length": 755.3125,
      "completions/mean_terminated_length": 755.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3354632928967476,
      "epoch": 0.00485,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13846859335899353,
      "kl": 0.6153592951595783,
      "learning_rate": 2.8965645468638805e-05,
      "loss": -0.0037,
      "num_tokens": 11432729.0,
      "reward": 0.6134267449378967,
      "reward_std": 0.567315399646759,
      "rewards/rollout_reward_func/mean": 0.6134267449378967,
      "rewards/rollout_reward_func/std": 0.6279176473617554,
      "sampling/importance_sampling_ratio/max": 1.140057921409607,
      "sampling/importance_sampling_ratio/mean": 1.0109988451004028,
      "sampling/importance_sampling_ratio/min": 0.938127875328064,
      "sampling/sampling_logp_difference/max": 0.13187670707702637,
      "sampling/sampling_logp_difference/mean": 0.006204589735716581,
      "step": 485,
      "step_time": 10.317276081999808
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1789.0,
      "completions/max_terminated_length": 1789.0,
      "completions/mean_length": 639.5,
      "completions/mean_terminated_length": 639.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3711758777499199,
      "epoch": 0.00486,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.3677885830402374,
      "kl": 0.7351553123444319,
      "learning_rate": 2.8893798594044164e-05,
      "loss": -0.0202,
      "num_tokens": 11458287.0,
      "reward": 0.676061749458313,
      "reward_std": 0.3788810074329376,
      "rewards/rollout_reward_func/mean": 0.676061749458313,
      "rewards/rollout_reward_func/std": 0.6163548231124878,
      "sampling/importance_sampling_ratio/max": 1.0677070617675781,
      "sampling/importance_sampling_ratio/mean": 1.0016182661056519,
      "sampling/importance_sampling_ratio/min": 0.9456458687782288,
      "sampling/sampling_logp_difference/max": 0.14545154571533203,
      "sampling/sampling_logp_difference/mean": 0.00881893653422594,
      "step": 486,
      "step_time": 10.050044868000441
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 840.0,
      "completions/max_terminated_length": 840.0,
      "completions/mean_length": 360.8125,
      "completions/mean_terminated_length": 360.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.18486759811639786,
      "epoch": 0.00487,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.24606753885746002,
      "kl": 0.7503004297614098,
      "learning_rate": 2.8821943152548533e-05,
      "loss": -0.0053,
      "num_tokens": 11478272.0,
      "reward": 0.742428719997406,
      "reward_std": 0.5971807241439819,
      "rewards/rollout_reward_func/mean": 0.742428719997406,
      "rewards/rollout_reward_func/std": 0.5848068594932556,
      "sampling/importance_sampling_ratio/max": 1.00163996219635,
      "sampling/importance_sampling_ratio/mean": 0.9910687208175659,
      "sampling/importance_sampling_ratio/min": 0.8901053071022034,
      "sampling/sampling_logp_difference/max": 0.11600923538208008,
      "sampling/sampling_logp_difference/mean": 0.003298897063359618,
      "step": 487,
      "step_time": 7.0110813949995645
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1673.0,
      "completions/max_terminated_length": 1673.0,
      "completions/mean_length": 675.625,
      "completions/mean_terminated_length": 675.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2765193395316601,
      "epoch": 0.00488,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1262626200914383,
      "kl": 0.6532307993620634,
      "learning_rate": 2.8750080000550964e-05,
      "loss": -0.0116,
      "num_tokens": 11504588.0,
      "reward": 0.4821740984916687,
      "reward_std": 0.691818118095398,
      "rewards/rollout_reward_func/mean": 0.4821740984916687,
      "rewards/rollout_reward_func/std": 0.7354654669761658,
      "sampling/importance_sampling_ratio/max": 1.09874427318573,
      "sampling/importance_sampling_ratio/mean": 1.0064789056777954,
      "sampling/importance_sampling_ratio/min": 0.9905094504356384,
      "sampling/sampling_logp_difference/max": 0.0840153694152832,
      "sampling/sampling_logp_difference/mean": 0.002781198127195239,
      "step": 488,
      "step_time": 9.850569250998888
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1513.0,
      "completions/max_terminated_length": 1513.0,
      "completions/mean_length": 451.5625,
      "completions/mean_terminated_length": 451.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2275787005200982,
      "epoch": 0.00489,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.0685451552271843,
      "kl": 0.5276108775287867,
      "learning_rate": 2.8678209994542448e-05,
      "loss": 0.0029,
      "num_tokens": 11526138.0,
      "reward": 0.7980324029922485,
      "reward_std": 0.27898404002189636,
      "rewards/rollout_reward_func/mean": 0.7980324029922485,
      "rewards/rollout_reward_func/std": 0.4345076382160187,
      "sampling/importance_sampling_ratio/max": 1.0115927457809448,
      "sampling/importance_sampling_ratio/mean": 0.9918580651283264,
      "sampling/importance_sampling_ratio/min": 0.9178634285926819,
      "sampling/sampling_logp_difference/max": 0.11525511741638184,
      "sampling/sampling_logp_difference/mean": 0.004965056665241718,
      "step": 489,
      "step_time": 8.943795537001279
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1736.0,
      "completions/max_terminated_length": 1736.0,
      "completions/mean_length": 559.4375,
      "completions/mean_terminated_length": 533.1333618164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4829040803015232,
      "epoch": 0.0049,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.6817302107810974,
      "kl": 0.42791992891579866,
      "learning_rate": 2.860633399109565e-05,
      "loss": 0.1124,
      "num_tokens": 11550148.0,
      "reward": 0.6779893040657043,
      "reward_std": 0.3689769208431244,
      "rewards/rollout_reward_func/mean": 0.6779893040657043,
      "rewards/rollout_reward_func/std": 0.6039362549781799,
      "sampling/importance_sampling_ratio/max": 1.0319710969924927,
      "sampling/importance_sampling_ratio/mean": 1.001718521118164,
      "sampling/importance_sampling_ratio/min": 0.9752320647239685,
      "sampling/sampling_logp_difference/max": 0.22523760795593262,
      "sampling/sampling_logp_difference/mean": 0.011299995705485344,
      "step": 490,
      "step_time": 10.458377835999272
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1687.0,
      "completions/max_terminated_length": 1687.0,
      "completions/mean_length": 567.5625,
      "completions/mean_terminated_length": 540.2000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8212212026119232,
      "epoch": 0.00491,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.439546138048172,
      "kl": 0.5481934435665607,
      "learning_rate": 2.8534452846854713e-05,
      "loss": -0.1273,
      "num_tokens": 11574180.0,
      "reward": 0.5404597520828247,
      "reward_std": 0.6228877305984497,
      "rewards/rollout_reward_func/mean": 0.5404597520828247,
      "rewards/rollout_reward_func/std": 0.6450592279434204,
      "sampling/importance_sampling_ratio/max": 1.2175414562225342,
      "sampling/importance_sampling_ratio/mean": 0.9268731474876404,
      "sampling/importance_sampling_ratio/min": 1.0531110381990504e-18,
      "sampling/sampling_logp_difference/max": 12.911629676818848,
      "sampling/sampling_logp_difference/mean": 0.21678461134433746,
      "step": 491,
      "step_time": 10.692139663001399
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1791.0,
      "completions/max_terminated_length": 1791.0,
      "completions/mean_length": 627.0625,
      "completions/mean_terminated_length": 627.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3367028795182705,
      "epoch": 0.00492,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13094055652618408,
      "kl": 0.5632230676710606,
      "learning_rate": 2.8462567418525038e-05,
      "loss": 0.009,
      "num_tokens": 11599200.0,
      "reward": 0.6139419674873352,
      "reward_std": 0.6197312474250793,
      "rewards/rollout_reward_func/mean": 0.6139419674873352,
      "rewards/rollout_reward_func/std": 0.6240188479423523,
      "sampling/importance_sampling_ratio/max": 1.0164754390716553,
      "sampling/importance_sampling_ratio/mean": 0.999435544013977,
      "sampling/importance_sampling_ratio/min": 0.9763484597206116,
      "sampling/sampling_logp_difference/max": 0.03178119659423828,
      "sampling/sampling_logp_difference/mean": 0.0028899116441607475,
      "step": 492,
      "step_time": 9.968914789998962
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 929.0,
      "completions/max_terminated_length": 929.0,
      "completions/mean_length": 534.875,
      "completions/mean_terminated_length": 534.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5791830737143755,
      "epoch": 0.00493,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09785910695791245,
      "kl": 0.6444708555936813,
      "learning_rate": 2.8390678562863116e-05,
      "loss": -0.0028,
      "num_tokens": 11622727.0,
      "reward": 0.7278112173080444,
      "reward_std": 0.5040991306304932,
      "rewards/rollout_reward_func/mean": 0.7278112173080444,
      "rewards/rollout_reward_func/std": 0.4870731234550476,
      "sampling/importance_sampling_ratio/max": 1.0204254388809204,
      "sampling/importance_sampling_ratio/mean": 0.929658055305481,
      "sampling/importance_sampling_ratio/min": 3.2780204874301166e-13,
      "sampling/sampling_logp_difference/max": 20.274709701538086,
      "sampling/sampling_logp_difference/mean": 0.33075934648513794,
      "step": 493,
      "step_time": 7.894163112999195
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 871.0,
      "completions/max_terminated_length": 871.0,
      "completions/mean_length": 468.3125,
      "completions/mean_terminated_length": 468.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2663636696524918,
      "epoch": 0.00494,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.09024450927972794,
      "kl": 0.8879168331623077,
      "learning_rate": 2.831878713666626e-05,
      "loss": -0.0074,
      "num_tokens": 11644608.0,
      "reward": 0.7945862412452698,
      "reward_std": 0.44734084606170654,
      "rewards/rollout_reward_func/mean": 0.7945862412452698,
      "rewards/rollout_reward_func/std": 0.4417072534561157,
      "sampling/importance_sampling_ratio/max": 1.1249420642852783,
      "sampling/importance_sampling_ratio/mean": 1.0043171644210815,
      "sampling/importance_sampling_ratio/min": 0.894777774810791,
      "sampling/sampling_logp_difference/max": 0.11690878868103027,
      "sampling/sampling_logp_difference/mean": 0.007620139047503471,
      "step": 494,
      "step_time": 7.330106097998396
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1800.0,
      "completions/max_terminated_length": 1800.0,
      "completions/mean_length": 526.0625,
      "completions/mean_terminated_length": 526.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6430220031179488,
      "epoch": 0.00495,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.40981030464172363,
      "kl": 0.818458927795291,
      "learning_rate": 2.8246893996762422e-05,
      "loss": -0.0382,
      "num_tokens": 11667556.0,
      "reward": 0.40693581104278564,
      "reward_std": 0.6596423387527466,
      "rewards/rollout_reward_func/mean": 0.40693581104278564,
      "rewards/rollout_reward_func/std": 0.6501211524009705,
      "sampling/importance_sampling_ratio/max": 1.0477410554885864,
      "sampling/importance_sampling_ratio/mean": 0.998422384262085,
      "sampling/importance_sampling_ratio/min": 0.8596113324165344,
      "sampling/sampling_logp_difference/max": 0.2756006717681885,
      "sampling/sampling_logp_difference/mean": 0.022689089179039,
      "step": 495,
      "step_time": 10.095499081998241
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1801.0,
      "completions/max_terminated_length": 1801.0,
      "completions/mean_length": 690.6875,
      "completions/mean_terminated_length": 690.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6013439111411572,
      "epoch": 0.00496,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.43253520131111145,
      "kl": 1.1333320289850235,
      "learning_rate": 2.8174999999999994e-05,
      "loss": 0.0276,
      "num_tokens": 11693523.0,
      "reward": 0.4759520888328552,
      "reward_std": 0.6474426984786987,
      "rewards/rollout_reward_func/mean": 0.4759520888328552,
      "rewards/rollout_reward_func/std": 0.6463296413421631,
      "sampling/importance_sampling_ratio/max": 1.042036771774292,
      "sampling/importance_sampling_ratio/mean": 0.9759160280227661,
      "sampling/importance_sampling_ratio/min": 0.5619295239448547,
      "sampling/sampling_logp_difference/max": 0.19591856002807617,
      "sampling/sampling_logp_difference/mean": 0.015701422467827797,
      "step": 496,
      "step_time": 10.174612578998676
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 870.0,
      "completions/max_terminated_length": 870.0,
      "completions/mean_length": 333.1875,
      "completions/mean_terminated_length": 333.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4174681780859828,
      "epoch": 0.00497,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2594883143901825,
      "kl": 3.081635072827339,
      "learning_rate": 2.810310600323757e-05,
      "loss": 0.0081,
      "num_tokens": 11714001.0,
      "reward": 0.15337249636650085,
      "reward_std": 0.6976245641708374,
      "rewards/rollout_reward_func/mean": 0.15337249636650085,
      "rewards/rollout_reward_func/std": 0.7521743774414062,
      "sampling/importance_sampling_ratio/max": 1.1213948726654053,
      "sampling/importance_sampling_ratio/mean": 1.0075088739395142,
      "sampling/importance_sampling_ratio/min": 0.9775583148002625,
      "sampling/sampling_logp_difference/max": 0.11533546447753906,
      "sampling/sampling_logp_difference/mean": 0.0038656166289001703,
      "step": 497,
      "step_time": 7.470387280000068
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 972.0,
      "completions/max_terminated_length": 781.0,
      "completions/mean_length": 402.5625,
      "completions/mean_terminated_length": 364.6000061035156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9044255847111344,
      "epoch": 0.00498,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6075176000595093,
      "kl": 0.6264755055308342,
      "learning_rate": 2.803121286333373e-05,
      "loss": -0.0724,
      "num_tokens": 11735451.0,
      "reward": 0.5850565433502197,
      "reward_std": 0.5516167879104614,
      "rewards/rollout_reward_func/mean": 0.5850565433502197,
      "rewards/rollout_reward_func/std": 0.5533342957496643,
      "sampling/importance_sampling_ratio/max": 1.0126032829284668,
      "sampling/importance_sampling_ratio/mean": 0.98539137840271,
      "sampling/importance_sampling_ratio/min": 0.7983580231666565,
      "sampling/sampling_logp_difference/max": 0.11010122299194336,
      "sampling/sampling_logp_difference/mean": 0.020268751308321953,
      "step": 498,
      "step_time": 8.070700991000194
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 879.0,
      "completions/max_terminated_length": 879.0,
      "completions/mean_length": 421.1875,
      "completions/mean_terminated_length": 421.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7590763331390917,
      "epoch": 0.00499,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1672734171152115,
      "kl": 0.8923902101814747,
      "learning_rate": 2.7959321437136878e-05,
      "loss": 0.0012,
      "num_tokens": 11756960.0,
      "reward": 0.7424429655075073,
      "reward_std": 0.5625450611114502,
      "rewards/rollout_reward_func/mean": 0.7424429655075073,
      "rewards/rollout_reward_func/std": 0.5846875905990601,
      "sampling/importance_sampling_ratio/max": 1.026204228401184,
      "sampling/importance_sampling_ratio/mean": 0.9916017651557922,
      "sampling/importance_sampling_ratio/min": 0.933320939540863,
      "sampling/sampling_logp_difference/max": 0.09585857391357422,
      "sampling/sampling_logp_difference/mean": 0.008901124820113182,
      "step": 499,
      "step_time": 7.2512245980014995
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1693.0,
      "completions/max_terminated_length": 1693.0,
      "completions/mean_length": 436.9375,
      "completions/mean_terminated_length": 436.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4999217037111521,
      "epoch": 0.005,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07050195336341858,
      "kl": 0.5180555135011673,
      "learning_rate": 2.7887432581474953e-05,
      "loss": 0.0073,
      "num_tokens": 11778249.0,
      "reward": 0.5297715067863464,
      "reward_std": 0.5272229909896851,
      "rewards/rollout_reward_func/mean": 0.5297715067863464,
      "rewards/rollout_reward_func/std": 0.5512670278549194,
      "sampling/importance_sampling_ratio/max": 1.0365320444107056,
      "sampling/importance_sampling_ratio/mean": 1.0023356676101685,
      "sampling/importance_sampling_ratio/min": 0.9801058769226074,
      "sampling/sampling_logp_difference/max": 0.029670685529708862,
      "sampling/sampling_logp_difference/mean": 0.0029329475946724415,
      "step": 500,
      "step_time": 9.447935103999043
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1818.0,
      "completions/max_terminated_length": 1818.0,
      "completions/mean_length": 529.125,
      "completions/mean_terminated_length": 511.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8932434562593699,
      "epoch": 0.00501,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.15731525421142578,
      "kl": 0.8249597996473312,
      "learning_rate": 2.7815547153145278e-05,
      "loss": 0.0061,
      "num_tokens": 11800894.0,
      "reward": 0.6112844944000244,
      "reward_std": 0.3595355749130249,
      "rewards/rollout_reward_func/mean": 0.6112844944000244,
      "rewards/rollout_reward_func/std": 0.6344046592712402,
      "sampling/importance_sampling_ratio/max": 1.2072412967681885,
      "sampling/importance_sampling_ratio/mean": 1.0168917179107666,
      "sampling/importance_sampling_ratio/min": 0.8550694584846497,
      "sampling/sampling_logp_difference/max": 0.2056751251220703,
      "sampling/sampling_logp_difference/mean": 0.02801589109003544,
      "step": 501,
      "step_time": 10.341634451999198
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1682.0,
      "completions/max_terminated_length": 1682.0,
      "completions/mean_length": 577.0625,
      "completions/mean_terminated_length": 560.2667236328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1953909248113632,
      "epoch": 0.00502,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.2154904454946518,
      "kl": 0.46674451418220997,
      "learning_rate": 2.774366600890434e-05,
      "loss": -0.0305,
      "num_tokens": 11825241.0,
      "reward": 0.8600000143051147,
      "reward_std": 0.2592296302318573,
      "rewards/rollout_reward_func/mean": 0.8600000143051147,
      "rewards/rollout_reward_func/std": 0.382552832365036,
      "sampling/importance_sampling_ratio/max": 1.0665875673294067,
      "sampling/importance_sampling_ratio/mean": 0.9780542850494385,
      "sampling/importance_sampling_ratio/min": 0.7547071576118469,
      "sampling/sampling_logp_difference/max": 0.17217063903808594,
      "sampling/sampling_logp_difference/mean": 0.01958121545612812,
      "step": 502,
      "step_time": 10.353205082001296
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 835.0,
      "completions/max_terminated_length": 835.0,
      "completions/mean_length": 303.3125,
      "completions/mean_terminated_length": 303.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.206601527519524,
      "epoch": 0.00503,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5327816009521484,
      "kl": 0.6065811729058623,
      "learning_rate": 2.7671790005457543e-05,
      "loss": -0.046,
      "num_tokens": 11844381.0,
      "reward": 0.6004887223243713,
      "reward_std": 0.6423501968383789,
      "rewards/rollout_reward_func/mean": 0.6004887223243713,
      "rewards/rollout_reward_func/std": 0.6426416635513306,
      "sampling/importance_sampling_ratio/max": 1.0669827461242676,
      "sampling/importance_sampling_ratio/mean": 1.007955551147461,
      "sampling/importance_sampling_ratio/min": 0.9328991174697876,
      "sampling/sampling_logp_difference/max": 0.12491798400878906,
      "sampling/sampling_logp_difference/mean": 0.017570512369275093,
      "step": 503,
      "step_time": 7.916758014999687
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1817.0,
      "completions/max_terminated_length": 1817.0,
      "completions/mean_length": 721.0,
      "completions/mean_terminated_length": 671.0000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.280266523361206,
      "epoch": 0.00504,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3255436420440674,
      "kl": 0.7255903892219067,
      "learning_rate": 2.759991999944903e-05,
      "loss": 0.1386,
      "num_tokens": 11871477.0,
      "reward": 0.28869932889938354,
      "reward_std": 0.7173346281051636,
      "rewards/rollout_reward_func/mean": 0.28869932889938354,
      "rewards/rollout_reward_func/std": 0.7159432768821716,
      "sampling/importance_sampling_ratio/max": 1.396647572517395,
      "sampling/importance_sampling_ratio/mean": 1.0419824123382568,
      "sampling/importance_sampling_ratio/min": 0.7842727899551392,
      "sampling/sampling_logp_difference/max": 0.18808937072753906,
      "sampling/sampling_logp_difference/mean": 0.021584516391158104,
      "step": 504,
      "step_time": 11.400214304999281
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1799.0,
      "completions/max_terminated_length": 1799.0,
      "completions/mean_length": 520.375,
      "completions/mean_terminated_length": 520.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6960049625486135,
      "epoch": 0.00505,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.07011478394269943,
      "kl": 0.4125392371788621,
      "learning_rate": 2.7528056847451464e-05,
      "loss": -0.0006,
      "num_tokens": 11893466.0,
      "reward": 0.5967308282852173,
      "reward_std": 0.5375351309776306,
      "rewards/rollout_reward_func/mean": 0.5967308282852173,
      "rewards/rollout_reward_func/std": 0.5378601551055908,
      "sampling/importance_sampling_ratio/max": 1.1030281782150269,
      "sampling/importance_sampling_ratio/mean": 1.008255958557129,
      "sampling/importance_sampling_ratio/min": 0.9665822386741638,
      "sampling/sampling_logp_difference/max": 0.1041412353515625,
      "sampling/sampling_logp_difference/mean": 0.007584455423057079,
      "step": 505,
      "step_time": 9.957180950999827
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 844.0,
      "completions/max_terminated_length": 844.0,
      "completions/mean_length": 367.625,
      "completions/mean_terminated_length": 367.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0525492122396827,
      "epoch": 0.00506,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.6233291625976562,
      "kl": 0.6675339192152023,
      "learning_rate": 2.7456201405955826e-05,
      "loss": -0.0244,
      "num_tokens": 11913331.0,
      "reward": 0.7405893802642822,
      "reward_std": 0.38540756702423096,
      "rewards/rollout_reward_func/mean": 0.7405893802642822,
      "rewards/rollout_reward_func/std": 0.5908070206642151,
      "sampling/importance_sampling_ratio/max": 1.310989260673523,
      "sampling/importance_sampling_ratio/mean": 1.0100111961364746,
      "sampling/importance_sampling_ratio/min": 0.8909784555435181,
      "sampling/sampling_logp_difference/max": 0.20051097869873047,
      "sampling/sampling_logp_difference/mean": 0.021185901015996933,
      "step": 506,
      "step_time": 7.430653407998761
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1754.0,
      "completions/max_terminated_length": 1754.0,
      "completions/mean_length": 561.1875,
      "completions/mean_terminated_length": 495.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2408774439245462,
      "epoch": 0.00507,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.239328145980835,
      "kl": 0.5302010029554367,
      "learning_rate": 2.7384354531361192e-05,
      "loss": -0.0018,
      "num_tokens": 11937157.0,
      "reward": 0.40251705050468445,
      "reward_std": 0.6441358327865601,
      "rewards/rollout_reward_func/mean": 0.40251705050468445,
      "rewards/rollout_reward_func/std": 0.6542759537696838,
      "sampling/importance_sampling_ratio/max": 1.4992371797561646,
      "sampling/importance_sampling_ratio/mean": 1.046098232269287,
      "sampling/importance_sampling_ratio/min": 0.49767017364501953,
      "sampling/sampling_logp_difference/max": 0.20496797561645508,
      "sampling/sampling_logp_difference/mean": 0.025668229907751083,
      "step": 507,
      "step_time": 11.052996922999228
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1711.0,
      "completions/max_terminated_length": 1711.0,
      "completions/mean_length": 752.625,
      "completions/mean_terminated_length": 751.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7189773991703987,
      "epoch": 0.00508,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.937217652797699,
      "kl": 0.7905405387282372,
      "learning_rate": 2.7312517079964535e-05,
      "loss": 0.0771,
      "num_tokens": 11964713.0,
      "reward": 0.3469833731651306,
      "reward_std": 0.7315586805343628,
      "rewards/rollout_reward_func/mean": 0.3469833731651306,
      "rewards/rollout_reward_func/std": 0.7331360578536987,
      "sampling/importance_sampling_ratio/max": 1.231291651725769,
      "sampling/importance_sampling_ratio/mean": 0.9715734720230103,
      "sampling/importance_sampling_ratio/min": 8.381198296145143e-13,
      "sampling/sampling_logp_difference/max": 18.85352897644043,
      "sampling/sampling_logp_difference/mean": 0.17086854577064514,
      "step": 508,
      "step_time": 10.850750765999692
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1741.0,
      "completions/max_terminated_length": 1741.0,
      "completions/mean_length": 563.625,
      "completions/mean_terminated_length": 582.6428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.330325972288847,
      "epoch": 0.00509,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7797814011573792,
      "kl": 0.6142036449164152,
      "learning_rate": 2.724068990795053e-05,
      "loss": 0.1881,
      "num_tokens": 11988714.0,
      "reward": 0.486427903175354,
      "reward_std": 0.7621040344238281,
      "rewards/rollout_reward_func/mean": 0.486427903175354,
      "rewards/rollout_reward_func/std": 0.7362626791000366,
      "sampling/importance_sampling_ratio/max": 1.2059345245361328,
      "sampling/importance_sampling_ratio/mean": 0.9163919687271118,
      "sampling/importance_sampling_ratio/min": 2.7201372546152265e-16,
      "sampling/sampling_logp_difference/max": 16.54059600830078,
      "sampling/sampling_logp_difference/mean": 0.18369704484939575,
      "step": 509,
      "step_time": 10.787351083999965
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1539.0,
      "completions/max_terminated_length": 1539.0,
      "completions/mean_length": 448.75,
      "completions/mean_terminated_length": 448.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0788672612980008,
      "epoch": 0.0051,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.22840051352977753,
      "kl": 0.7272337730973959,
      "learning_rate": 2.7168873871381312e-05,
      "loss": -0.0366,
      "num_tokens": 12010272.0,
      "reward": 0.6060693860054016,
      "reward_std": 0.6493958830833435,
      "rewards/rollout_reward_func/mean": 0.6060693860054016,
      "rewards/rollout_reward_func/std": 0.6325777173042297,
      "sampling/importance_sampling_ratio/max": 1.1881694793701172,
      "sampling/importance_sampling_ratio/mean": 0.9517273306846619,
      "sampling/importance_sampling_ratio/min": 1.7100163299255122e-11,
      "sampling/sampling_logp_difference/max": 20.691312789916992,
      "sampling/sampling_logp_difference/mean": 0.285914808511734,
      "step": 510,
      "step_time": 9.81472598000073
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1634.0,
      "completions/max_terminated_length": 1634.0,
      "completions/mean_length": 622.0625,
      "completions/mean_terminated_length": 621.6666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.100554496049881,
      "epoch": 0.00511,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2381904423236847,
      "kl": 0.7202124465256929,
      "learning_rate": 2.709706982618633e-05,
      "loss": -0.053,
      "num_tokens": 12035423.0,
      "reward": 0.8649643659591675,
      "reward_std": 0.38193845748901367,
      "rewards/rollout_reward_func/mean": 0.8649643659591675,
      "rewards/rollout_reward_func/std": 0.3691708445549011,
      "sampling/importance_sampling_ratio/max": 1.1695090532302856,
      "sampling/importance_sampling_ratio/mean": 0.9788718223571777,
      "sampling/importance_sampling_ratio/min": 0.6809981465339661,
      "sampling/sampling_logp_difference/max": 0.37878894805908203,
      "sampling/sampling_logp_difference/mean": 0.0178239643573761,
      "step": 511,
      "step_time": 10.322795826000402
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1664.0,
      "completions/max_terminated_length": 1664.0,
      "completions/mean_length": 582.1875,
      "completions/mean_terminated_length": 582.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4872426204383373,
      "epoch": 0.00512,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6775252819061279,
      "kl": 0.5996307544410229,
      "learning_rate": 2.7025278628152112e-05,
      "loss": 0.0459,
      "num_tokens": 12059621.0,
      "reward": 0.4051712155342102,
      "reward_std": 0.55776447057724,
      "rewards/rollout_reward_func/mean": 0.4051712155342102,
      "rewards/rollout_reward_func/std": 0.5426474809646606,
      "sampling/importance_sampling_ratio/max": 1.353740930557251,
      "sampling/importance_sampling_ratio/mean": 1.008918285369873,
      "sampling/importance_sampling_ratio/min": 0.5563600659370422,
      "sampling/sampling_logp_difference/max": 0.2129530906677246,
      "sampling/sampling_logp_difference/mean": 0.029052414000034332,
      "step": 512,
      "step_time": 10.414606137001101
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1847.0,
      "completions/max_terminated_length": 1847.0,
      "completions/mean_length": 486.25,
      "completions/mean_terminated_length": 486.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.487437822856009,
      "epoch": 0.00513,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.39956143498420715,
      "kl": 0.3717654552310705,
      "learning_rate": 2.6953501132912048e-05,
      "loss": 0.0468,
      "num_tokens": 12081717.0,
      "reward": 0.6680237054824829,
      "reward_std": 0.2757265567779541,
      "rewards/rollout_reward_func/mean": 0.6680237054824829,
      "rewards/rollout_reward_func/std": 0.5093810558319092,
      "sampling/importance_sampling_ratio/max": 1.353229284286499,
      "sampling/importance_sampling_ratio/mean": 0.9715518355369568,
      "sampling/importance_sampling_ratio/min": 2.255081971852535e-12,
      "sampling/sampling_logp_difference/max": 22.251039505004883,
      "sampling/sampling_logp_difference/mean": 0.17759667336940765,
      "step": 513,
      "step_time": 10.607781228999556
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2653.0,
      "completions/max_terminated_length": 2653.0,
      "completions/mean_length": 780.875,
      "completions/mean_terminated_length": 780.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.486557312309742,
      "epoch": 0.00514,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3964758515357971,
      "kl": 0.6504312828183174,
      "learning_rate": 2.6881738195936235e-05,
      "loss": -0.0705,
      "num_tokens": 12108679.0,
      "reward": 0.6672953367233276,
      "reward_std": 0.47101926803588867,
      "rewards/rollout_reward_func/mean": 0.6672953367233276,
      "rewards/rollout_reward_func/std": 0.5104907155036926,
      "sampling/importance_sampling_ratio/max": 1.2031623125076294,
      "sampling/importance_sampling_ratio/mean": 1.018582820892334,
      "sampling/importance_sampling_ratio/min": 0.6628187894821167,
      "sampling/sampling_logp_difference/max": 0.21835041046142578,
      "sampling/sampling_logp_difference/mean": 0.02242954447865486,
      "step": 514,
      "step_time": 13.003094774999227
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1854.0,
      "completions/max_terminated_length": 1854.0,
      "completions/mean_length": 638.4375,
      "completions/mean_terminated_length": 638.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5242638438940048,
      "epoch": 0.00515,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 32965.51171875,
      "kl": 4908.258179767057,
      "learning_rate": 2.680999067252123e-05,
      "loss": 145.0176,
      "num_tokens": 12133791.0,
      "reward": 0.6743702292442322,
      "reward_std": 0.5691945552825928,
      "rewards/rollout_reward_func/mean": 0.6743702292442322,
      "rewards/rollout_reward_func/std": 0.6126084327697754,
      "sampling/importance_sampling_ratio/max": 1.5362476110458374,
      "sampling/importance_sampling_ratio/mean": 0.9398066997528076,
      "sampling/importance_sampling_ratio/min": 9.248972781112645e-18,
      "sampling/sampling_logp_difference/max": 13.446708679199219,
      "sampling/sampling_logp_difference/mean": 0.23525457084178925,
      "step": 515,
      "step_time": 10.913355206001143
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1788.0,
      "completions/max_terminated_length": 1788.0,
      "completions/mean_length": 455.5,
      "completions/mean_terminated_length": 455.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7521716137416661,
      "epoch": 0.00516,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2152564823627472,
      "kl": 0.6898723039776087,
      "learning_rate": 2.673825941777993e-05,
      "loss": -0.0461,
      "num_tokens": 12154948.0,
      "reward": 0.5894981026649475,
      "reward_std": 0.547359824180603,
      "rewards/rollout_reward_func/mean": 0.5894981026649475,
      "rewards/rollout_reward_func/std": 0.5475605726242065,
      "sampling/importance_sampling_ratio/max": 1.060836911201477,
      "sampling/importance_sampling_ratio/mean": 0.9900383949279785,
      "sampling/importance_sampling_ratio/min": 0.8827807307243347,
      "sampling/sampling_logp_difference/max": 0.1042633056640625,
      "sampling/sampling_logp_difference/mean": 0.008210563100874424,
      "step": 516,
      "step_time": 9.87368105800033
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 804.0,
      "completions/max_terminated_length": 804.0,
      "completions/mean_length": 222.8125,
      "completions/mean_terminated_length": 235.53334045410156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5569166140630841,
      "epoch": 0.00517,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.23673631250858307,
      "kl": 0.3596752565354109,
      "learning_rate": 2.6666545286631297e-05,
      "loss": -0.0761,
      "num_tokens": 12172675.0,
      "reward": 0.863089382648468,
      "reward_std": 0.38724178075790405,
      "rewards/rollout_reward_func/mean": 0.863089382648468,
      "rewards/rollout_reward_func/std": 0.3741616904735565,
      "sampling/importance_sampling_ratio/max": 1.309175729751587,
      "sampling/importance_sampling_ratio/mean": 0.876273512840271,
      "sampling/importance_sampling_ratio/min": 3.609164374858721e-18,
      "sampling/sampling_logp_difference/max": 21.198116302490234,
      "sampling/sampling_logp_difference/mean": 0.35617733001708984,
      "step": 517,
      "step_time": 8.09829592800088
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1921.0,
      "completions/max_terminated_length": 1815.0,
      "completions/mean_length": 660.625,
      "completions/mean_terminated_length": 576.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3734143800102174,
      "epoch": 0.00518,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8198644518852234,
      "kl": 0.6951352804899216,
      "learning_rate": 2.659484913379022e-05,
      "loss": 0.0103,
      "num_tokens": 12198367.0,
      "reward": 0.4667152464389801,
      "reward_std": 0.5705342292785645,
      "rewards/rollout_reward_func/mean": 0.4667152464389801,
      "rewards/rollout_reward_func/std": 0.5514736771583557,
      "sampling/importance_sampling_ratio/max": 1.2785083055496216,
      "sampling/importance_sampling_ratio/mean": 0.9707561135292053,
      "sampling/importance_sampling_ratio/min": 0.5110377669334412,
      "sampling/sampling_logp_difference/max": 0.2205367088317871,
      "sampling/sampling_logp_difference/mean": 0.029212772846221924,
      "step": 518,
      "step_time": 11.21799250100048
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 884.0,
      "completions/max_terminated_length": 884.0,
      "completions/mean_length": 387.8125,
      "completions/mean_terminated_length": 387.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1850406061857939,
      "epoch": 0.00519,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5732548236846924,
      "kl": 0.7601556219160557,
      "learning_rate": 2.6523171813757327e-05,
      "loss": 0.066,
      "num_tokens": 12219076.0,
      "reward": 0.5557318925857544,
      "reward_std": 0.7479475736618042,
      "rewards/rollout_reward_func/mean": 0.5557318925857544,
      "rewards/rollout_reward_func/std": 0.726023256778717,
      "sampling/importance_sampling_ratio/max": 1.0820235013961792,
      "sampling/importance_sampling_ratio/mean": 0.9604405164718628,
      "sampling/importance_sampling_ratio/min": 0.7983851432800293,
      "sampling/sampling_logp_difference/max": 0.1322002410888672,
      "sampling/sampling_logp_difference/mean": 0.022071031853556633,
      "step": 519,
      "step_time": 7.461832233002497
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1788.0,
      "completions/max_terminated_length": 1788.0,
      "completions/mean_length": 457.5625,
      "completions/mean_terminated_length": 457.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3117883251979947,
      "epoch": 0.0052,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.697210967540741,
      "kl": 0.5160862114280462,
      "learning_rate": 2.6451514180808782e-05,
      "loss": 0.0398,
      "num_tokens": 12240136.0,
      "reward": 0.7926955223083496,
      "reward_std": 0.45228463411331177,
      "rewards/rollout_reward_func/mean": 0.7926955223083496,
      "rewards/rollout_reward_func/std": 0.4457743167877197,
      "sampling/importance_sampling_ratio/max": 1.409623622894287,
      "sampling/importance_sampling_ratio/mean": 1.0395039319992065,
      "sampling/importance_sampling_ratio/min": 0.9806399345397949,
      "sampling/sampling_logp_difference/max": 0.15596389770507812,
      "sampling/sampling_logp_difference/mean": 0.022152196615934372,
      "step": 520,
      "step_time": 10.0177882830003
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1857.0,
      "completions/max_terminated_length": 1857.0,
      "completions/mean_length": 538.4375,
      "completions/mean_terminated_length": 610.7857666015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4332943046465516,
      "epoch": 0.00521,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8367098569869995,
      "kl": 0.3953321650624275,
      "learning_rate": 2.6379877088986095e-05,
      "loss": 0.3486,
      "num_tokens": 12263508.0,
      "reward": 0.4218920171260834,
      "reward_std": 0.6533591747283936,
      "rewards/rollout_reward_func/mean": 0.4218920171260834,
      "rewards/rollout_reward_func/std": 0.7260047793388367,
      "sampling/importance_sampling_ratio/max": 2.5345876216888428,
      "sampling/importance_sampling_ratio/mean": 1.1428991556167603,
      "sampling/importance_sampling_ratio/min": 0.8238805532455444,
      "sampling/sampling_logp_difference/max": 0.2569866180419922,
      "sampling/sampling_logp_difference/mean": 0.02530035562813282,
      "step": 521,
      "step_time": 10.457600961000935
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1827.0,
      "completions/max_terminated_length": 1827.0,
      "completions/mean_length": 553.5625,
      "completions/mean_terminated_length": 588.3333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2555285352282226,
      "epoch": 0.00522,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7988342046737671,
      "kl": 0.8207665085792542,
      "learning_rate": 2.630826139208599e-05,
      "loss": 0.0818,
      "num_tokens": 12287544.0,
      "reward": 0.6073197722434998,
      "reward_std": 0.6501792669296265,
      "rewards/rollout_reward_func/mean": 0.6073197722434998,
      "rewards/rollout_reward_func/std": 0.6350080966949463,
      "sampling/importance_sampling_ratio/max": 1.1609535217285156,
      "sampling/importance_sampling_ratio/mean": 1.0078225135803223,
      "sampling/importance_sampling_ratio/min": 0.8383359909057617,
      "sampling/sampling_logp_difference/max": 0.1778392791748047,
      "sampling/sampling_logp_difference/mean": 0.022850049659609795,
      "step": 522,
      "step_time": 10.453348520001782
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1824.0,
      "completions/max_terminated_length": 1824.0,
      "completions/mean_length": 507.0625,
      "completions/mean_terminated_length": 475.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4966867752373219,
      "epoch": 0.00523,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2444273233413696,
      "kl": 0.5000359695404768,
      "learning_rate": 2.6236667943650183e-05,
      "loss": 0.2776,
      "num_tokens": 12310009.0,
      "reward": 0.542832612991333,
      "reward_std": 0.5570040345191956,
      "rewards/rollout_reward_func/mean": 0.542832612991333,
      "rewards/rollout_reward_func/std": 0.6489003300666809,
      "sampling/importance_sampling_ratio/max": 1.262152075767517,
      "sampling/importance_sampling_ratio/mean": 0.9457108974456787,
      "sampling/importance_sampling_ratio/min": 5.508609540738618e-11,
      "sampling/sampling_logp_difference/max": 23.238245010375977,
      "sampling/sampling_logp_difference/mean": 0.18307118117809296,
      "step": 523,
      "step_time": 10.427911444998244
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1733.0,
      "completions/max_terminated_length": 1733.0,
      "completions/mean_length": 715.75,
      "completions/mean_terminated_length": 715.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1873166528530419,
      "epoch": 0.00524,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7813442349433899,
      "kl": 1.0371989496052265,
      "learning_rate": 2.6165097596955217e-05,
      "loss": 0.0377,
      "num_tokens": 12336192.0,
      "reward": 0.736572265625,
      "reward_std": 0.48805075883865356,
      "rewards/rollout_reward_func/mean": 0.736572265625,
      "rewards/rollout_reward_func/std": 0.47159314155578613,
      "sampling/importance_sampling_ratio/max": 1.5086814165115356,
      "sampling/importance_sampling_ratio/mean": 0.9504197239875793,
      "sampling/importance_sampling_ratio/min": 1.7091679183067754e-08,
      "sampling/sampling_logp_difference/max": 14.52955436706543,
      "sampling/sampling_logp_difference/mean": 0.11227541416883469,
      "step": 524,
      "step_time": 11.35225842500131
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 896.0,
      "completions/max_terminated_length": 896.0,
      "completions/mean_length": 380.0625,
      "completions/mean_terminated_length": 403.2666931152344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4008256681263447,
      "epoch": 0.00525,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8359005451202393,
      "kl": 0.6209121383726597,
      "learning_rate": 2.6093551205002315e-05,
      "loss": -0.0505,
      "num_tokens": 12357138.0,
      "reward": 0.2777320146560669,
      "reward_std": 0.7395191788673401,
      "rewards/rollout_reward_func/mean": 0.2777320146560669,
      "rewards/rollout_reward_func/std": 0.7188743352890015,
      "sampling/importance_sampling_ratio/max": 1.0154237747192383,
      "sampling/importance_sampling_ratio/mean": 0.9111983180046082,
      "sampling/importance_sampling_ratio/min": 0.492226779460907,
      "sampling/sampling_logp_difference/max": 0.18642187118530273,
      "sampling/sampling_logp_difference/mean": 0.026878656819462776,
      "step": 525,
      "step_time": 7.801684889999706
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2565.0,
      "completions/max_terminated_length": 2565.0,
      "completions/mean_length": 541.5625,
      "completions/mean_terminated_length": 514.0667114257812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4222534149885178,
      "epoch": 0.00526,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0282793045043945,
      "kl": 1.4862873181700706,
      "learning_rate": 2.6022029620507188e-05,
      "loss": 0.0265,
      "num_tokens": 12380885.0,
      "reward": 0.23544858396053314,
      "reward_std": 0.879543662071228,
      "rewards/rollout_reward_func/mean": 0.23544858396053314,
      "rewards/rollout_reward_func/std": 0.8497441411018372,
      "sampling/importance_sampling_ratio/max": 1.5159320831298828,
      "sampling/importance_sampling_ratio/mean": 1.0761419534683228,
      "sampling/importance_sampling_ratio/min": 0.9567909240722656,
      "sampling/sampling_logp_difference/max": 0.1314239501953125,
      "sampling/sampling_logp_difference/mean": 0.028232701122760773,
      "step": 526,
      "step_time": 12.298405591001028
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1726.0,
      "completions/max_terminated_length": 1726.0,
      "completions/mean_length": 557.5625,
      "completions/mean_terminated_length": 557.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0631724076811224,
      "epoch": 0.00527,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.48794668912887573,
      "kl": 0.7131907157599926,
      "learning_rate": 2.595053369588989e-05,
      "loss": 0.0173,
      "num_tokens": 12404357.0,
      "reward": 0.7266923785209656,
      "reward_std": 0.5060849785804749,
      "rewards/rollout_reward_func/mean": 0.7266923785209656,
      "rewards/rollout_reward_func/std": 0.4889782667160034,
      "sampling/importance_sampling_ratio/max": 1.054386019706726,
      "sampling/importance_sampling_ratio/mean": 0.9437083005905151,
      "sampling/importance_sampling_ratio/min": 0.5682156682014465,
      "sampling/sampling_logp_difference/max": 0.21667003631591797,
      "sampling/sampling_logp_difference/mean": 0.022044654935598373,
      "step": 527,
      "step_time": 10.040122038001755
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1558.0,
      "completions/max_terminated_length": 1558.0,
      "completions/mean_length": 352.0,
      "completions/mean_terminated_length": 352.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8553354709874839,
      "epoch": 0.00528,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2328607141971588,
      "kl": 0.4370038453489542,
      "learning_rate": 2.5879064283264642e-05,
      "loss": 0.0328,
      "num_tokens": 12423697.0,
      "reward": 0.6082050800323486,
      "reward_std": 0.5730134844779968,
      "rewards/rollout_reward_func/mean": 0.6082050800323486,
      "rewards/rollout_reward_func/std": 0.6350856423377991,
      "sampling/importance_sampling_ratio/max": 1.0556886196136475,
      "sampling/importance_sampling_ratio/mean": 1.008674144744873,
      "sampling/importance_sampling_ratio/min": 0.9584169387817383,
      "sampling/sampling_logp_difference/max": 0.11782145500183105,
      "sampling/sampling_logp_difference/mean": 0.01748243160545826,
      "step": 528,
      "step_time": 9.37418948100003
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2592.0,
      "completions/max_terminated_length": 2592.0,
      "completions/mean_length": 846.625,
      "completions/mean_terminated_length": 846.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.143167108297348,
      "epoch": 0.00529,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2595345973968506,
      "kl": 0.7458983566612005,
      "learning_rate": 2.58076222344297e-05,
      "loss": -0.0115,
      "num_tokens": 12452591.0,
      "reward": 0.8075994849205017,
      "reward_std": 0.4190846085548401,
      "rewards/rollout_reward_func/mean": 0.8075994849205017,
      "rewards/rollout_reward_func/std": 0.4146909713745117,
      "sampling/importance_sampling_ratio/max": 1.066960334777832,
      "sampling/importance_sampling_ratio/mean": 0.9836911559104919,
      "sampling/importance_sampling_ratio/min": 0.8560250997543335,
      "sampling/sampling_logp_difference/max": 0.17205095291137695,
      "sampling/sampling_logp_difference/mean": 0.017268424853682518,
      "step": 529,
      "step_time": 12.643491159000405
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 878.0,
      "completions/max_terminated_length": 878.0,
      "completions/mean_length": 371.3125,
      "completions/mean_terminated_length": 371.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.675945620983839,
      "epoch": 0.0053,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.12569227814674377,
      "kl": 0.3649990987032652,
      "learning_rate": 2.5736208400857168e-05,
      "loss": -0.0272,
      "num_tokens": 12473437.0,
      "reward": 0.5304707288742065,
      "reward_std": 0.6708939075469971,
      "rewards/rollout_reward_func/mean": 0.5304707288742065,
      "rewards/rollout_reward_func/std": 0.6566265821456909,
      "sampling/importance_sampling_ratio/max": 1.1197113990783691,
      "sampling/importance_sampling_ratio/mean": 1.0239882469177246,
      "sampling/importance_sampling_ratio/min": 0.9736427664756775,
      "sampling/sampling_logp_difference/max": 0.12157440185546875,
      "sampling/sampling_logp_difference/mean": 0.011223148554563522,
      "step": 530,
      "step_time": 7.608453129000736
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1683.0,
      "completions/max_terminated_length": 1683.0,
      "completions/mean_length": 601.3125,
      "completions/mean_terminated_length": 576.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0449076062068343,
      "epoch": 0.00531,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.497648686170578,
      "kl": 0.5593183720484376,
      "learning_rate": 2.5664823633682897e-05,
      "loss": -0.0588,
      "num_tokens": 12498035.0,
      "reward": 0.4167933762073517,
      "reward_std": 0.7400540709495544,
      "rewards/rollout_reward_func/mean": 0.4167933762073517,
      "rewards/rollout_reward_func/std": 0.731407642364502,
      "sampling/importance_sampling_ratio/max": 1.1337069272994995,
      "sampling/importance_sampling_ratio/mean": 0.9355716109275818,
      "sampling/importance_sampling_ratio/min": 2.016703454623195e-14,
      "sampling/sampling_logp_difference/max": 13.510005950927734,
      "sampling/sampling_logp_difference/mean": 0.22159571945667267,
      "step": 531,
      "step_time": 10.494458647000101
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1830.0,
      "completions/max_terminated_length": 1830.0,
      "completions/mean_length": 664.75,
      "completions/mean_terminated_length": 664.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.056113688275218,
      "epoch": 0.00532,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3355516493320465,
      "kl": 0.881293036043644,
      "learning_rate": 2.5593468783696286e-05,
      "loss": -0.0018,
      "num_tokens": 12523819.0,
      "reward": 0.2233395278453827,
      "reward_std": 0.6520013809204102,
      "rewards/rollout_reward_func/mean": 0.2233395278453827,
      "rewards/rollout_reward_func/std": 0.6889795064926147,
      "sampling/importance_sampling_ratio/max": 1.1076563596725464,
      "sampling/importance_sampling_ratio/mean": 0.9656431078910828,
      "sampling/importance_sampling_ratio/min": 0.6277097463607788,
      "sampling/sampling_logp_difference/max": 0.23987293243408203,
      "sampling/sampling_logp_difference/mean": 0.026280801743268967,
      "step": 532,
      "step_time": 10.390161550999437
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1661.0,
      "completions/max_terminated_length": 1661.0,
      "completions/mean_length": 516.875,
      "completions/mean_terminated_length": 516.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7897195983678102,
      "epoch": 0.00533,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4896773099899292,
      "kl": 0.6098141595721245,
      "learning_rate": 2.5522144701330197e-05,
      "loss": 0.0133,
      "num_tokens": 12546908.0,
      "reward": 0.6090041399002075,
      "reward_std": 0.6257617473602295,
      "rewards/rollout_reward_func/mean": 0.6090041399002075,
      "rewards/rollout_reward_func/std": 0.6331235766410828,
      "sampling/importance_sampling_ratio/max": 1.0002578496932983,
      "sampling/importance_sampling_ratio/mean": 0.9381527900695801,
      "sampling/importance_sampling_ratio/min": 0.7539975047111511,
      "sampling/sampling_logp_difference/max": 0.1796877384185791,
      "sampling/sampling_logp_difference/mean": 0.022149967029690742,
      "step": 533,
      "step_time": 9.929885826999453
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1640.0,
      "completions/max_terminated_length": 1640.0,
      "completions/mean_length": 418.1875,
      "completions/mean_terminated_length": 418.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9035346591845155,
      "epoch": 0.00534,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4437742531299591,
      "kl": 0.5895330272614956,
      "learning_rate": 2.545085223665077e-05,
      "loss": -0.0442,
      "num_tokens": 12567621.0,
      "reward": 0.7304576635360718,
      "reward_std": 0.46176040172576904,
      "rewards/rollout_reward_func/mean": 0.7304576635360718,
      "rewards/rollout_reward_func/std": 0.48290327191352844,
      "sampling/importance_sampling_ratio/max": 1.1600958108901978,
      "sampling/importance_sampling_ratio/mean": 1.0262854099273682,
      "sampling/importance_sampling_ratio/min": 0.9126690626144409,
      "sampling/sampling_logp_difference/max": 0.1338667869567871,
      "sampling/sampling_logp_difference/mean": 0.020677316933870316,
      "step": 534,
      "step_time": 9.574551290000272
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1754.0,
      "completions/max_terminated_length": 1754.0,
      "completions/mean_length": 475.9375,
      "completions/mean_terminated_length": 476.5000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2474545538425446,
      "epoch": 0.00535,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6554485559463501,
      "kl": 0.6805175486952066,
      "learning_rate": 2.5379592239347325e-05,
      "loss": 0.1326,
      "num_tokens": 12589263.0,
      "reward": 0.41441601514816284,
      "reward_std": 0.7558953166007996,
      "rewards/rollout_reward_func/mean": 0.41441601514816284,
      "rewards/rollout_reward_func/std": 0.734406590461731,
      "sampling/importance_sampling_ratio/max": 1.0841132402420044,
      "sampling/importance_sampling_ratio/mean": 0.9690288305282593,
      "sampling/importance_sampling_ratio/min": 0.6690371632575989,
      "sampling/sampling_logp_difference/max": 0.12838077545166016,
      "sampling/sampling_logp_difference/mean": 0.023626018315553665,
      "step": 535,
      "step_time": 10.036619287002395
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1730.0,
      "completions/max_terminated_length": 1730.0,
      "completions/mean_length": 501.125,
      "completions/mean_terminated_length": 501.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1636978685855865,
      "epoch": 0.00536,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 11.648630142211914,
      "kl": 71.57800440303981,
      "learning_rate": 2.5308365558722225e-05,
      "loss": 0.138,
      "num_tokens": 12612700.0,
      "reward": 0.5215747356414795,
      "reward_std": 0.574640154838562,
      "rewards/rollout_reward_func/mean": 0.5215747356414795,
      "rewards/rollout_reward_func/std": 0.5605029463768005,
      "sampling/importance_sampling_ratio/max": 1.0570502281188965,
      "sampling/importance_sampling_ratio/mean": 0.9645630121231079,
      "sampling/importance_sampling_ratio/min": 0.6272540092468262,
      "sampling/sampling_logp_difference/max": 0.12221479415893555,
      "sampling/sampling_logp_difference/mean": 0.017815591767430305,
      "step": 536,
      "step_time": 10.15975185499883
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2619.0,
      "completions/max_terminated_length": 2619.0,
      "completions/mean_length": 797.3125,
      "completions/mean_terminated_length": 788.4000244140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3079059422016144,
      "epoch": 0.00537,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7970654368400574,
      "kl": 0.8262802790850401,
      "learning_rate": 2.523717304368074e-05,
      "loss": 0.1876,
      "num_tokens": 12640225.0,
      "reward": 0.6244230270385742,
      "reward_std": 0.5514761805534363,
      "rewards/rollout_reward_func/mean": 0.6244230270385742,
      "rewards/rollout_reward_func/std": 0.6154450178146362,
      "sampling/importance_sampling_ratio/max": 2.1117238998413086,
      "sampling/importance_sampling_ratio/mean": 1.0741292238235474,
      "sampling/importance_sampling_ratio/min": 0.9134562611579895,
      "sampling/sampling_logp_difference/max": 0.23106956481933594,
      "sampling/sampling_logp_difference/mean": 0.02381753735244274,
      "step": 537,
      "step_time": 13.24066104799931
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 822.0,
      "completions/max_terminated_length": 822.0,
      "completions/mean_length": 273.0625,
      "completions/mean_terminated_length": 273.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5630022040568292,
      "epoch": 0.00538,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.03939440846443176,
      "kl": 0.44870556704699993,
      "learning_rate": 2.5166015542720966e-05,
      "loss": -0.0054,
      "num_tokens": 12658300.0,
      "reward": 0.860195517539978,
      "reward_std": 0.25886765122413635,
      "rewards/rollout_reward_func/mean": 0.860195517539978,
      "rewards/rollout_reward_func/std": 0.38201865553855896,
      "sampling/importance_sampling_ratio/max": 1.0544427633285522,
      "sampling/importance_sampling_ratio/mean": 1.0000927448272705,
      "sampling/importance_sampling_ratio/min": 0.959284245967865,
      "sampling/sampling_logp_difference/max": 0.0496983528137207,
      "sampling/sampling_logp_difference/mean": 0.005349138285964727,
      "step": 538,
      "step_time": 7.052046742998755
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1797.0,
      "completions/max_terminated_length": 1797.0,
      "completions/mean_length": 563.3125,
      "completions/mean_terminated_length": 462.857177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3602698324248195,
      "epoch": 0.00539,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4862782061100006,
      "kl": 0.5429798848927021,
      "learning_rate": 2.5094893903923658e-05,
      "loss": -0.1089,
      "num_tokens": 12681850.0,
      "reward": 0.6241888999938965,
      "reward_std": 0.7351925373077393,
      "rewards/rollout_reward_func/mean": 0.6241888999938965,
      "rewards/rollout_reward_func/std": 0.7104942202568054,
      "sampling/importance_sampling_ratio/max": 1.2304034233093262,
      "sampling/importance_sampling_ratio/mean": 1.0058586597442627,
      "sampling/importance_sampling_ratio/min": 0.8499550223350525,
      "sampling/sampling_logp_difference/max": 0.17574739456176758,
      "sampling/sampling_logp_difference/mean": 0.03333405777812004,
      "step": 539,
      "step_time": 10.479157690999273
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 3302.0,
      "completions/max_terminated_length": 3302.0,
      "completions/mean_length": 671.4375,
      "completions/mean_terminated_length": 697.7142944335938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7276536514982581,
      "epoch": 0.0054,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.034510612487793,
      "kl": 1.333363227546215,
      "learning_rate": 2.502380897494219e-05,
      "loss": -0.0185,
      "num_tokens": 12706994.0,
      "reward": 0.35497599840164185,
      "reward_std": 0.7336088418960571,
      "rewards/rollout_reward_func/mean": 0.35497599840164185,
      "rewards/rollout_reward_func/std": 0.7088828682899475,
      "sampling/importance_sampling_ratio/max": 1.0943602323532104,
      "sampling/importance_sampling_ratio/mean": 0.9531797170639038,
      "sampling/importance_sampling_ratio/min": 0.639563262462616,
      "sampling/sampling_logp_difference/max": 0.20615386962890625,
      "sampling/sampling_logp_difference/mean": 0.02872665412724018,
      "step": 540,
      "step_time": 14.955714935001197
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1613.0,
      "completions/max_terminated_length": 963.0,
      "completions/mean_length": 552.375,
      "completions/mean_terminated_length": 481.66668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2796281315386295,
      "epoch": 0.00541,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.098718523979187,
      "kl": 0.964159931987524,
      "learning_rate": 2.4952761602992387e-05,
      "loss": 0.011,
      "num_tokens": 12730508.0,
      "reward": 0.5944885015487671,
      "reward_std": 0.5387898087501526,
      "rewards/rollout_reward_func/mean": 0.5944885015487671,
      "rewards/rollout_reward_func/std": 0.5407010316848755,
      "sampling/importance_sampling_ratio/max": 1.1866731643676758,
      "sampling/importance_sampling_ratio/mean": 0.940095841884613,
      "sampling/importance_sampling_ratio/min": 0.6329286098480225,
      "sampling/sampling_logp_difference/max": 0.32470083236694336,
      "sampling/sampling_logp_difference/mean": 0.02970135398209095,
      "step": 541,
      "step_time": 10.403131936000136
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2444.0,
      "completions/max_terminated_length": 2444.0,
      "completions/mean_length": 546.875,
      "completions/mean_terminated_length": 546.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.268653355538845,
      "epoch": 0.00542,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2475711107254028,
      "kl": 0.7473979629576206,
      "learning_rate": 2.488175263484246e-05,
      "loss": 0.0867,
      "num_tokens": 12753885.0,
      "reward": 0.6612764596939087,
      "reward_std": 0.5311031341552734,
      "rewards/rollout_reward_func/mean": 0.6612764596939087,
      "rewards/rollout_reward_func/std": 0.5189508199691772,
      "sampling/importance_sampling_ratio/max": 1.539284348487854,
      "sampling/importance_sampling_ratio/mean": 1.0388654470443726,
      "sampling/importance_sampling_ratio/min": 0.771971583366394,
      "sampling/sampling_logp_difference/max": 0.1870737075805664,
      "sampling/sampling_logp_difference/mean": 0.031054137274622917,
      "step": 542,
      "step_time": 12.199658021000687
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1868.0,
      "completions/max_terminated_length": 1868.0,
      "completions/mean_length": 369.875,
      "completions/mean_terminated_length": 357.73333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0439331028610468,
      "epoch": 0.00543,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9744747281074524,
      "kl": 0.4355606585741043,
      "learning_rate": 2.481078291680291e-05,
      "loss": 0.2408,
      "num_tokens": 12774083.0,
      "reward": 0.593533456325531,
      "reward_std": 0.5373480319976807,
      "rewards/rollout_reward_func/mean": 0.593533456325531,
      "rewards/rollout_reward_func/std": 0.5425035357475281,
      "sampling/importance_sampling_ratio/max": 1.3507359027862549,
      "sampling/importance_sampling_ratio/mean": 0.8770681619644165,
      "sampling/importance_sampling_ratio/min": 3.407043891456851e-08,
      "sampling/sampling_logp_difference/max": 16.813507080078125,
      "sampling/sampling_logp_difference/mean": 0.1265895813703537,
      "step": 543,
      "step_time": 10.994257225999718
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1779.0,
      "completions/max_terminated_length": 1779.0,
      "completions/mean_length": 506.125,
      "completions/mean_terminated_length": 482.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.632188456133008,
      "epoch": 0.00544,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2693228721618652,
      "kl": 0.9960356205701828,
      "learning_rate": 2.4739853294716452e-05,
      "loss": 0.6572,
      "num_tokens": 12796353.0,
      "reward": 0.42602118849754333,
      "reward_std": 0.8188660144805908,
      "rewards/rollout_reward_func/mean": 0.42602118849754333,
      "rewards/rollout_reward_func/std": 0.8187212944030762,
      "sampling/importance_sampling_ratio/max": 2.638078212738037,
      "sampling/importance_sampling_ratio/mean": 1.0836058855056763,
      "sampling/importance_sampling_ratio/min": 0.4268307089805603,
      "sampling/sampling_logp_difference/max": 0.36266422271728516,
      "sampling/sampling_logp_difference/mean": 0.03239605948328972,
      "step": 544,
      "step_time": 11.353415413999755
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1743.0,
      "completions/max_terminated_length": 1743.0,
      "completions/mean_length": 428.8125,
      "completions/mean_terminated_length": 428.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.270268950611353,
      "epoch": 0.00545,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6342310905456543,
      "kl": 1.4020142685621977,
      "learning_rate": 2.466896461394792e-05,
      "loss": -0.0603,
      "num_tokens": 12817759.0,
      "reward": 0.5174800753593445,
      "reward_std": 0.5801866054534912,
      "rewards/rollout_reward_func/mean": 0.5174800753593445,
      "rewards/rollout_reward_func/std": 0.5653287172317505,
      "sampling/importance_sampling_ratio/max": 1.5185260772705078,
      "sampling/importance_sampling_ratio/mean": 1.0178322792053223,
      "sampling/importance_sampling_ratio/min": 0.7119832634925842,
      "sampling/sampling_logp_difference/max": 0.3138084411621094,
      "sampling/sampling_logp_difference/mean": 0.0182819701731205,
      "step": 545,
      "step_time": 9.737405712998225
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1813.0,
      "completions/max_terminated_length": 1813.0,
      "completions/mean_length": 536.375,
      "completions/mean_terminated_length": 536.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5073045529425144,
      "epoch": 0.00546,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7590393424034119,
      "kl": 1.88349777366966,
      "learning_rate": 2.459811771937417e-05,
      "loss": -0.0414,
      "num_tokens": 12841576.0,
      "reward": 0.2330772578716278,
      "reward_std": 0.838860034942627,
      "rewards/rollout_reward_func/mean": 0.2330772578716278,
      "rewards/rollout_reward_func/std": 0.8500470519065857,
      "sampling/importance_sampling_ratio/max": 1.4723201990127563,
      "sampling/importance_sampling_ratio/mean": 0.980765700340271,
      "sampling/importance_sampling_ratio/min": 0.6760978698730469,
      "sampling/sampling_logp_difference/max": 0.11932706832885742,
      "sampling/sampling_logp_difference/mean": 0.02445193938910961,
      "step": 546,
      "step_time": 10.333480598997994
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1658.0,
      "completions/max_terminated_length": 1658.0,
      "completions/mean_length": 412.6875,
      "completions/mean_terminated_length": 412.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0817281575873494,
      "epoch": 0.00547,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.6262773871421814,
      "kl": 1.0773707665503025,
      "learning_rate": 2.452731345537407e-05,
      "loss": 0.1204,
      "num_tokens": 12862151.0,
      "reward": 0.7435876727104187,
      "reward_std": 0.3818821310997009,
      "rewards/rollout_reward_func/mean": 0.7435876727104187,
      "rewards/rollout_reward_func/std": 0.5851101279258728,
      "sampling/importance_sampling_ratio/max": 1.2354474067687988,
      "sampling/importance_sampling_ratio/mean": 0.9964009523391724,
      "sampling/importance_sampling_ratio/min": 0.8216938376426697,
      "sampling/sampling_logp_difference/max": 0.13817977905273438,
      "sampling/sampling_logp_difference/mean": 0.02066924050450325,
      "step": 547,
      "step_time": 10.032346178998523
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1737.0,
      "completions/max_terminated_length": 1737.0,
      "completions/mean_length": 539.125,
      "completions/mean_terminated_length": 539.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.622030884027481,
      "epoch": 0.00548,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6508721709251404,
      "kl": 2.3896865025162697,
      "learning_rate": 2.445655266581839e-05,
      "loss": -0.0202,
      "num_tokens": 12886002.0,
      "reward": 0.5326532125473022,
      "reward_std": 0.6700919270515442,
      "rewards/rollout_reward_func/mean": 0.5326532125473022,
      "rewards/rollout_reward_func/std": 0.6587411761283875,
      "sampling/importance_sampling_ratio/max": 1.194414734840393,
      "sampling/importance_sampling_ratio/mean": 0.9552529454231262,
      "sampling/importance_sampling_ratio/min": 5.617953222444199e-15,
      "sampling/sampling_logp_difference/max": 25.511587142944336,
      "sampling/sampling_logp_difference/mean": 0.4232311546802521,
      "step": 548,
      "step_time": 9.999101087999406
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1513.0,
      "completions/max_terminated_length": 1513.0,
      "completions/mean_length": 504.3125,
      "completions/mean_terminated_length": 504.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.53586820140481,
      "epoch": 0.00549,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.675375759601593,
      "kl": 2.7414180524647236,
      "learning_rate": 2.4385836194059735e-05,
      "loss": -0.1018,
      "num_tokens": 12909427.0,
      "reward": 0.37195444107055664,
      "reward_std": 0.9074262380599976,
      "rewards/rollout_reward_func/mean": 0.37195444107055664,
      "rewards/rollout_reward_func/std": 0.8766622543334961,
      "sampling/importance_sampling_ratio/max": 1.5835307836532593,
      "sampling/importance_sampling_ratio/mean": 1.0690739154815674,
      "sampling/importance_sampling_ratio/min": 0.6199970245361328,
      "sampling/sampling_logp_difference/max": 0.21092796325683594,
      "sampling/sampling_logp_difference/mean": 0.02744690328836441,
      "step": 549,
      "step_time": 9.482350281002255
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 837.0,
      "completions/max_terminated_length": 837.0,
      "completions/mean_length": 455.5625,
      "completions/mean_terminated_length": 455.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9626604504883289,
      "epoch": 0.0055,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5593973994255066,
      "kl": 2.727639250457287,
      "learning_rate": 2.4315164882922545e-05,
      "loss": -0.0139,
      "num_tokens": 12931104.0,
      "reward": 0.6576071977615356,
      "reward_std": 0.5363452434539795,
      "rewards/rollout_reward_func/mean": 0.6576071977615356,
      "rewards/rollout_reward_func/std": 0.5246536731719971,
      "sampling/importance_sampling_ratio/max": 1.150420069694519,
      "sampling/importance_sampling_ratio/mean": 0.9868708848953247,
      "sampling/importance_sampling_ratio/min": 0.6563907861709595,
      "sampling/sampling_logp_difference/max": 0.20279169082641602,
      "sampling/sampling_logp_difference/mean": 0.02355813980102539,
      "step": 550,
      "step_time": 7.29614188099913
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2503.0,
      "completions/max_terminated_length": 2503.0,
      "completions/mean_length": 496.4375,
      "completions/mean_terminated_length": 496.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9728716015815735,
      "epoch": 0.00551,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6926240921020508,
      "kl": 1.7251118496060371,
      "learning_rate": 2.424453957469301e-05,
      "loss": 0.0812,
      "num_tokens": 12953698.0,
      "reward": 0.7261437773704529,
      "reward_std": 0.4728851020336151,
      "rewards/rollout_reward_func/mean": 0.7261437773704529,
      "rewards/rollout_reward_func/std": 0.49011626839637756,
      "sampling/importance_sampling_ratio/max": 1.2017550468444824,
      "sampling/importance_sampling_ratio/mean": 0.9179844856262207,
      "sampling/importance_sampling_ratio/min": 6.554222008875454e-12,
      "sampling/sampling_logp_difference/max": 17.596437454223633,
      "sampling/sampling_logp_difference/mean": 0.18634873628616333,
      "step": 551,
      "step_time": 13.17943623300016
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1828.0,
      "completions/max_terminated_length": 1828.0,
      "completions/mean_length": 600.375,
      "completions/mean_terminated_length": 600.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.973448071628809,
      "epoch": 0.00552,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6733105778694153,
      "kl": 3.0238305367529392,
      "learning_rate": 2.4173961111109018e-05,
      "loss": -0.0069,
      "num_tokens": 12978146.0,
      "reward": 0.6029388904571533,
      "reward_std": 0.6359779834747314,
      "rewards/rollout_reward_func/mean": 0.6029388904571533,
      "rewards/rollout_reward_func/std": 0.6370154619216919,
      "sampling/importance_sampling_ratio/max": 1.354170560836792,
      "sampling/importance_sampling_ratio/mean": 1.0181875228881836,
      "sampling/importance_sampling_ratio/min": 0.8845832347869873,
      "sampling/sampling_logp_difference/max": 0.20119667053222656,
      "sampling/sampling_logp_difference/mean": 0.020487971603870392,
      "step": 552,
      "step_time": 10.439832400998966
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2609.0,
      "completions/max_terminated_length": 1586.0,
      "completions/mean_length": 575.0625,
      "completions/mean_terminated_length": 444.0000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2765462771058083,
      "epoch": 0.00553,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5117926597595215,
      "kl": 2.718630276620388,
      "learning_rate": 2.410343033335018e-05,
      "loss": 0.0962,
      "num_tokens": 13001879.0,
      "reward": 0.7979937791824341,
      "reward_std": 0.4424179792404175,
      "rewards/rollout_reward_func/mean": 0.7979937791824341,
      "rewards/rollout_reward_func/std": 0.4345095753669739,
      "sampling/importance_sampling_ratio/max": 1.1542000770568848,
      "sampling/importance_sampling_ratio/mean": 1.0022094249725342,
      "sampling/importance_sampling_ratio/min": 0.840711772441864,
      "sampling/sampling_logp_difference/max": 0.2714349031448364,
      "sampling/sampling_logp_difference/mean": 0.02330547198653221,
      "step": 553,
      "step_time": 13.131576794999091
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1851.0,
      "completions/max_terminated_length": 1851.0,
      "completions/mean_length": 571.4375,
      "completions/mean_terminated_length": 571.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1269650608301163,
      "epoch": 0.00554,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8099320530891418,
      "kl": 4.255308359861374,
      "learning_rate": 2.4032948082027754e-05,
      "loss": 0.1478,
      "num_tokens": 13025965.0,
      "reward": 0.3123537302017212,
      "reward_std": 0.8863884210586548,
      "rewards/rollout_reward_func/mean": 0.3123537302017212,
      "rewards/rollout_reward_func/std": 0.8587961792945862,
      "sampling/importance_sampling_ratio/max": 1.1490260362625122,
      "sampling/importance_sampling_ratio/mean": 0.9348190426826477,
      "sampling/importance_sampling_ratio/min": 1.8046467075171396e-18,
      "sampling/sampling_logp_difference/max": 23.458932876586914,
      "sampling/sampling_logp_difference/mean": 0.3594074249267578,
      "step": 554,
      "step_time": 10.38929773199925
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1700.0,
      "completions/max_terminated_length": 1700.0,
      "completions/mean_length": 779.0625,
      "completions/mean_terminated_length": 779.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7811960056424141,
      "epoch": 0.00555,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.48723068833351135,
      "kl": 4.008253842592239,
      "learning_rate": 2.3962515197174634e-05,
      "loss": 0.009,
      "num_tokens": 13053286.0,
      "reward": 0.5032934546470642,
      "reward_std": 0.8116088509559631,
      "rewards/rollout_reward_func/mean": 0.5032934546470642,
      "rewards/rollout_reward_func/std": 0.7974763512611389,
      "sampling/importance_sampling_ratio/max": 1.177647590637207,
      "sampling/importance_sampling_ratio/mean": 1.0269012451171875,
      "sampling/importance_sampling_ratio/min": 0.9183833003044128,
      "sampling/sampling_logp_difference/max": 0.12884986400604248,
      "sampling/sampling_logp_difference/mean": 0.014399543404579163,
      "step": 555,
      "step_time": 10.328128802998435
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1676.0,
      "completions/max_terminated_length": 1676.0,
      "completions/mean_length": 507.9375,
      "completions/mean_terminated_length": 476.5714416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0810337364673615,
      "epoch": 0.00556,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1524604558944702,
      "kl": 2.99991774559021,
      "learning_rate": 2.389213251823537e-05,
      "loss": -0.0186,
      "num_tokens": 13076364.0,
      "reward": 0.553969144821167,
      "reward_std": 0.7016132473945618,
      "rewards/rollout_reward_func/mean": 0.553969144821167,
      "rewards/rollout_reward_func/std": 0.7290051579475403,
      "sampling/importance_sampling_ratio/max": 1.5198215246200562,
      "sampling/importance_sampling_ratio/mean": 1.0462846755981445,
      "sampling/importance_sampling_ratio/min": 0.9173221588134766,
      "sampling/sampling_logp_difference/max": 0.21445679664611816,
      "sampling/sampling_logp_difference/mean": 0.032629434019327164,
      "step": 556,
      "step_time": 10.16518229599933
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1725.0,
      "completions/max_terminated_length": 1725.0,
      "completions/mean_length": 557.0,
      "completions/mean_terminated_length": 543.6666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8870705906301737,
      "epoch": 0.00557,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.822985053062439,
      "kl": 3.073656715452671,
      "learning_rate": 2.382180088405611e-05,
      "loss": 0.0391,
      "num_tokens": 13099799.0,
      "reward": 0.875,
      "reward_std": 0.3535533845424652,
      "rewards/rollout_reward_func/mean": 0.875,
      "rewards/rollout_reward_func/std": 0.5,
      "sampling/importance_sampling_ratio/max": 1.873497724533081,
      "sampling/importance_sampling_ratio/mean": 1.067703127861023,
      "sampling/importance_sampling_ratio/min": 0.9249159693717957,
      "sampling/sampling_logp_difference/max": 0.20488262176513672,
      "sampling/sampling_logp_difference/mean": 0.022399919107556343,
      "step": 557,
      "step_time": 9.999179490000643
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 898.0,
      "completions/max_terminated_length": 898.0,
      "completions/mean_length": 361.8125,
      "completions/mean_terminated_length": 361.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8058498539030552,
      "epoch": 0.00558,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4197315573692322,
      "kl": 3.818153288215399,
      "learning_rate": 2.3751521132874653e-05,
      "loss": 0.0374,
      "num_tokens": 13119229.0,
      "reward": 0.6088957786560059,
      "reward_std": 0.6307508945465088,
      "rewards/rollout_reward_func/mean": 0.6088957786560059,
      "rewards/rollout_reward_func/std": 0.630672812461853,
      "sampling/importance_sampling_ratio/max": 1.2661539316177368,
      "sampling/importance_sampling_ratio/mean": 1.0179638862609863,
      "sampling/importance_sampling_ratio/min": 0.9407095313072205,
      "sampling/sampling_logp_difference/max": 0.11559057235717773,
      "sampling/sampling_logp_difference/mean": 0.015407845377922058,
      "step": 558,
      "step_time": 7.224917029000608
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2871.0,
      "completions/max_terminated_length": 2871.0,
      "completions/mean_length": 537.6875,
      "completions/mean_terminated_length": 518.7857666015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7883179485797882,
      "epoch": 0.00559,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9910441040992737,
      "kl": 3.637950085103512,
      "learning_rate": 2.3681294102310433e-05,
      "loss": 0.1296,
      "num_tokens": 13142233.0,
      "reward": 0.4925943911075592,
      "reward_std": 0.7222765684127808,
      "rewards/rollout_reward_func/mean": 0.4925943911075592,
      "rewards/rollout_reward_func/std": 0.7118977904319763,
      "sampling/importance_sampling_ratio/max": 1.5337437391281128,
      "sampling/importance_sampling_ratio/mean": 1.014978289604187,
      "sampling/importance_sampling_ratio/min": 0.5456930994987488,
      "sampling/sampling_logp_difference/max": 0.20423460006713867,
      "sampling/sampling_logp_difference/mean": 0.017003394663333893,
      "step": 559,
      "step_time": 14.020521272999758
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1901.0,
      "completions/max_terminated_length": 1901.0,
      "completions/mean_length": 489.75,
      "completions/mean_terminated_length": 489.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9078204482793808,
      "epoch": 0.0056,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.194460153579712,
      "kl": 3.155559688806534,
      "learning_rate": 2.3611120629354546e-05,
      "loss": 0.1231,
      "num_tokens": 13164493.0,
      "reward": 0.5961275696754456,
      "reward_std": 0.47010338306427,
      "rewards/rollout_reward_func/mean": 0.5961275696754456,
      "rewards/rollout_reward_func/std": 0.5387963056564331,
      "sampling/importance_sampling_ratio/max": 1.2530100345611572,
      "sampling/importance_sampling_ratio/mean": 1.027327060699463,
      "sampling/importance_sampling_ratio/min": 0.9412714242935181,
      "sampling/sampling_logp_difference/max": 0.14933252334594727,
      "sampling/sampling_logp_difference/mean": 0.017314955592155457,
      "step": 560,
      "step_time": 10.210166880001452
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1739.0,
      "completions/max_terminated_length": 1739.0,
      "completions/mean_length": 553.0625,
      "completions/mean_terminated_length": 579.4285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9976070988923311,
      "epoch": 0.00561,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4798259139060974,
      "kl": 2.0906727984547615,
      "learning_rate": 2.354100155035975e-05,
      "loss": -0.0536,
      "num_tokens": 13188243.0,
      "reward": 0.7329545021057129,
      "reward_std": 0.47016602754592896,
      "rewards/rollout_reward_func/mean": 0.7329545021057129,
      "rewards/rollout_reward_func/std": 0.4783327579498291,
      "sampling/importance_sampling_ratio/max": 1.1573050022125244,
      "sampling/importance_sampling_ratio/mean": 0.9368876218795776,
      "sampling/importance_sampling_ratio/min": 2.466313272314244e-24,
      "sampling/sampling_logp_difference/max": 24.73959732055664,
      "sampling/sampling_logp_difference/mean": 0.3202033042907715,
      "step": 561,
      "step_time": 10.63772206700014
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1601.0,
      "completions/max_terminated_length": 905.0,
      "completions/mean_length": 485.25,
      "completions/mean_terminated_length": 410.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9250057712197304,
      "epoch": 0.00562,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.36297836899757385,
      "kl": 2.789200071245432,
      "learning_rate": 2.347093770103054e-05,
      "loss": -0.0471,
      "num_tokens": 13210975.0,
      "reward": 0.5280978679656982,
      "reward_std": 0.4461696147918701,
      "rewards/rollout_reward_func/mean": 0.5280978679656982,
      "rewards/rollout_reward_func/std": 0.5530287623405457,
      "sampling/importance_sampling_ratio/max": 1.046120047569275,
      "sampling/importance_sampling_ratio/mean": 0.9183072447776794,
      "sampling/importance_sampling_ratio/min": 7.436449527453877e-15,
      "sampling/sampling_logp_difference/max": 26.786659240722656,
      "sampling/sampling_logp_difference/mean": 0.1995871514081955,
      "step": 562,
      "step_time": 10.248258591000194
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1577.0,
      "completions/max_terminated_length": 816.0,
      "completions/mean_length": 417.375,
      "completions/mean_terminated_length": 255.07693481445312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0865454599261284,
      "epoch": 0.00563,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.264197826385498,
      "kl": 1.493273925036192,
      "learning_rate": 2.3400929916413146e-05,
      "loss": 0.1016,
      "num_tokens": 13232247.0,
      "reward": 0.6624292731285095,
      "reward_std": 0.5319147109985352,
      "rewards/rollout_reward_func/mean": 0.6624292731285095,
      "rewards/rollout_reward_func/std": 0.5175023078918457,
      "sampling/importance_sampling_ratio/max": 1.5349406003952026,
      "sampling/importance_sampling_ratio/mean": 1.0673187971115112,
      "sampling/importance_sampling_ratio/min": 0.8204213976860046,
      "sampling/sampling_logp_difference/max": 0.15497207641601562,
      "sampling/sampling_logp_difference/mean": 0.023982608690857887,
      "step": 563,
      "step_time": 9.963229120000506
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 923.0,
      "completions/max_terminated_length": 923.0,
      "completions/mean_length": 317.8125,
      "completions/mean_terminated_length": 278.0666809082031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6331489086151123,
      "epoch": 0.00564,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2927844524383545,
      "kl": 1.2390188574790955,
      "learning_rate": 2.3330979030885613e-05,
      "loss": 0.0141,
      "num_tokens": 13252224.0,
      "reward": 0.21681366860866547,
      "reward_std": 0.8027514219284058,
      "rewards/rollout_reward_func/mean": 0.21681366860866547,
      "rewards/rollout_reward_func/std": 0.7865147590637207,
      "sampling/importance_sampling_ratio/max": 1.9741339683532715,
      "sampling/importance_sampling_ratio/mean": 1.0413146018981934,
      "sampling/importance_sampling_ratio/min": 0.5875877141952515,
      "sampling/sampling_logp_difference/max": 0.2450733184814453,
      "sampling/sampling_logp_difference/mean": 0.03157151862978935,
      "step": 564,
      "step_time": 8.82262768900091
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 940.0,
      "completions/max_terminated_length": 917.0,
      "completions/mean_length": 450.375,
      "completions/mean_terminated_length": 387.4285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2119767367839813,
      "epoch": 0.00565,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.422210454940796,
      "kl": 1.8059452772140503,
      "learning_rate": 2.326108587814783e-05,
      "loss": 0.2175,
      "num_tokens": 13273931.0,
      "reward": 0.4862895905971527,
      "reward_std": 0.7576062679290771,
      "rewards/rollout_reward_func/mean": 0.4862895905971527,
      "rewards/rollout_reward_func/std": 0.731959342956543,
      "sampling/importance_sampling_ratio/max": 1.5215963125228882,
      "sampling/importance_sampling_ratio/mean": 0.9513072967529297,
      "sampling/importance_sampling_ratio/min": 0.5532689690589905,
      "sampling/sampling_logp_difference/max": 0.22885143756866455,
      "sampling/sampling_logp_difference/mean": 0.027888784185051918,
      "step": 565,
      "step_time": 8.406898395000098
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1827.0,
      "completions/max_terminated_length": 847.0,
      "completions/mean_length": 512.8125,
      "completions/mean_terminated_length": 267.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0909974873065948,
      "epoch": 0.00566,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9867006540298462,
      "kl": 1.281192485243082,
      "learning_rate": 2.3191251291211613e-05,
      "loss": 0.1368,
      "num_tokens": 13296087.0,
      "reward": 0.8030237555503845,
      "reward_std": 0.4326300621032715,
      "rewards/rollout_reward_func/mean": 0.8030237555503845,
      "rewards/rollout_reward_func/std": 0.42394018173217773,
      "sampling/importance_sampling_ratio/max": 1.1572843790054321,
      "sampling/importance_sampling_ratio/mean": 0.9261052012443542,
      "sampling/importance_sampling_ratio/min": 0.5921273231506348,
      "sampling/sampling_logp_difference/max": 0.22025489807128906,
      "sampling/sampling_logp_difference/mean": 0.029737677425146103,
      "step": 566,
      "step_time": 11.115339219999441
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1626.0,
      "completions/max_terminated_length": 1626.0,
      "completions/mean_length": 532.625,
      "completions/mean_terminated_length": 558.357177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7963715195655823,
      "epoch": 0.00567,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6854095458984375,
      "kl": 1.555837631225586,
      "learning_rate": 2.312147610239076e-05,
      "loss": 0.1477,
      "num_tokens": 13319502.0,
      "reward": 0.6599801182746887,
      "reward_std": 0.4797225594520569,
      "rewards/rollout_reward_func/mean": 0.6599801182746887,
      "rewards/rollout_reward_func/std": 0.5210591554641724,
      "sampling/importance_sampling_ratio/max": 1.2276784181594849,
      "sampling/importance_sampling_ratio/mean": 0.9678752422332764,
      "sampling/importance_sampling_ratio/min": 0.5826553106307983,
      "sampling/sampling_logp_difference/max": 0.2165508270263672,
      "sampling/sampling_logp_difference/mean": 0.03361549600958824,
      "step": 567,
      "step_time": 10.817897906001235
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 982.0,
      "completions/max_terminated_length": 956.0,
      "completions/mean_length": 419.75,
      "completions/mean_terminated_length": 314.69232177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5168538242578506,
      "epoch": 0.00568,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6448848247528076,
      "kl": 1.3305737562477589,
      "learning_rate": 2.305176114329115e-05,
      "loss": 0.052,
      "num_tokens": 13339939.0,
      "reward": 0.5904667973518372,
      "reward_std": 0.5439466238021851,
      "rewards/rollout_reward_func/mean": 0.5904667973518372,
      "rewards/rollout_reward_func/std": 0.5461110472679138,
      "sampling/importance_sampling_ratio/max": 1.3275502920150757,
      "sampling/importance_sampling_ratio/mean": 0.8867374658584595,
      "sampling/importance_sampling_ratio/min": 1.5909418493720295e-07,
      "sampling/sampling_logp_difference/max": 5.754162311553955,
      "sampling/sampling_logp_difference/mean": 0.09813011437654495,
      "step": 568,
      "step_time": 9.009159067000837
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1709.0,
      "completions/max_terminated_length": 1709.0,
      "completions/mean_length": 596.5,
      "completions/mean_terminated_length": 462.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.865750715136528,
      "epoch": 0.00569,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4011133909225464,
      "kl": 1.2146862149238586,
      "learning_rate": 2.2982107244800827e-05,
      "loss": -0.137,
      "num_tokens": 13364366.0,
      "reward": 0.5302649736404419,
      "reward_std": 0.5273120999336243,
      "rewards/rollout_reward_func/mean": 0.5302649736404419,
      "rewards/rollout_reward_func/std": 0.5504822731018066,
      "sampling/importance_sampling_ratio/max": 1.5537084341049194,
      "sampling/importance_sampling_ratio/mean": 1.0408645868301392,
      "sampling/importance_sampling_ratio/min": 0.39935302734375,
      "sampling/sampling_logp_difference/max": 0.20558786392211914,
      "sampling/sampling_logp_difference/mean": 0.029017692431807518,
      "step": 569,
      "step_time": 11.856035033999433
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1749.0,
      "completions/max_terminated_length": 1724.0,
      "completions/mean_length": 651.6875,
      "completions/mean_terminated_length": 487.6363830566406,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6680370718240738,
      "epoch": 0.0057,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3853954076766968,
      "kl": 1.2595719546079636,
      "learning_rate": 2.2912515237080082e-05,
      "loss": -0.0229,
      "num_tokens": 13389839.0,
      "reward": 0.7304559946060181,
      "reward_std": 0.46978163719177246,
      "rewards/rollout_reward_func/mean": 0.7304559946060181,
      "rewards/rollout_reward_func/std": 0.48234474658966064,
      "sampling/importance_sampling_ratio/max": 1.16679048538208,
      "sampling/importance_sampling_ratio/mean": 0.9622118473052979,
      "sampling/importance_sampling_ratio/min": 0.7770766615867615,
      "sampling/sampling_logp_difference/max": 0.2221360206604004,
      "sampling/sampling_logp_difference/mean": 0.024335840716958046,
      "step": 570,
      "step_time": 12.294601866997255
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1669.0,
      "completions/max_terminated_length": 632.0,
      "completions/mean_length": 473.3125,
      "completions/mean_terminated_length": 175.10000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7118171826004982,
      "epoch": 0.00571,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 13.303802490234375,
      "kl": 9.866631887853146,
      "learning_rate": 2.2842985949551578e-05,
      "loss": 0.183,
      "num_tokens": 13412034.0,
      "reward": 0.8649643659591675,
      "reward_std": 0.25024524331092834,
      "rewards/rollout_reward_func/mean": 0.8649643659591675,
      "rewards/rollout_reward_func/std": 0.36925050616264343,
      "sampling/importance_sampling_ratio/max": 1.2599681615829468,
      "sampling/importance_sampling_ratio/mean": 0.9249535799026489,
      "sampling/importance_sampling_ratio/min": 0.49484243988990784,
      "sampling/sampling_logp_difference/max": 0.21425747871398926,
      "sampling/sampling_logp_difference/mean": 0.029436158016324043,
      "step": 571,
      "step_time": 11.602496072000577
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 892.0,
      "completions/max_terminated_length": 779.0,
      "completions/mean_length": 404.25,
      "completions/mean_terminated_length": 313.81817626953125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.597717933356762,
      "epoch": 0.00572,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.0445096492767334,
      "kl": 1.357462428510189,
      "learning_rate": 2.277352021089044e-05,
      "loss": 0.0441,
      "num_tokens": 13433264.0,
      "reward": 0.7939757108688354,
      "reward_std": 0.2843906581401825,
      "rewards/rollout_reward_func/mean": 0.7939757108688354,
      "rewards/rollout_reward_func/std": 0.442999005317688,
      "sampling/importance_sampling_ratio/max": 1.3350892066955566,
      "sampling/importance_sampling_ratio/mean": 0.898558497428894,
      "sampling/importance_sampling_ratio/min": 2.3742726422329e-18,
      "sampling/sampling_logp_difference/max": 24.363353729248047,
      "sampling/sampling_logp_difference/mean": 0.3157341778278351,
      "step": 572,
      "step_time": 8.997362234999855
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 886.0,
      "completions/max_terminated_length": 850.0,
      "completions/mean_length": 305.5,
      "completions/mean_terminated_length": 142.8181915283203,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5085598453879356,
      "epoch": 0.00573,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3061078786849976,
      "kl": 1.191207442432642,
      "learning_rate": 2.2704118849014426e-05,
      "loss": -0.0564,
      "num_tokens": 13452597.0,
      "reward": 0.26344457268714905,
      "reward_std": 0.5485521554946899,
      "rewards/rollout_reward_func/mean": 0.26344457268714905,
      "rewards/rollout_reward_func/std": 0.6306114792823792,
      "sampling/importance_sampling_ratio/max": 1.1874921321868896,
      "sampling/importance_sampling_ratio/mean": 0.962541401386261,
      "sampling/importance_sampling_ratio/min": 0.6516367793083191,
      "sampling/sampling_logp_difference/max": 0.18771839141845703,
      "sampling/sampling_logp_difference/mean": 0.02865973860025406,
      "step": 573,
      "step_time": 8.42848611000045
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 2428.0,
      "completions/max_terminated_length": 926.0,
      "completions/mean_length": 707.1875,
      "completions/mean_terminated_length": 461.1000061035156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5972290486097336,
      "epoch": 0.00574,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.433476209640503,
      "kl": 1.578532189130783,
      "learning_rate": 2.2634782691073985e-05,
      "loss": -0.086,
      "num_tokens": 13479341.0,
      "reward": 0.4841075539588928,
      "reward_std": 0.6904164552688599,
      "rewards/rollout_reward_func/mean": 0.4841075539588928,
      "rewards/rollout_reward_func/std": 0.7372131943702698,
      "sampling/importance_sampling_ratio/max": 1.6605989933013916,
      "sampling/importance_sampling_ratio/mean": 1.0300018787384033,
      "sampling/importance_sampling_ratio/min": 0.6051616668701172,
      "sampling/sampling_logp_difference/max": 0.4213838577270508,
      "sampling/sampling_logp_difference/mean": 0.030076568946242332,
      "step": 574,
      "step_time": 13.50664952399893
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1824.0,
      "completions/max_terminated_length": 823.0,
      "completions/mean_length": 521.4375,
      "completions/mean_terminated_length": 297.9090881347656,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4841725192964077,
      "epoch": 0.00575,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.702760934829712,
      "kl": 1.2308870814740658,
      "learning_rate": 2.256551256344248e-05,
      "loss": 0.1952,
      "num_tokens": 13502229.0,
      "reward": 0.7462824583053589,
      "reward_std": 0.590936541557312,
      "rewards/rollout_reward_func/mean": 0.7462824583053589,
      "rewards/rollout_reward_func/std": 0.580845057964325,
      "sampling/importance_sampling_ratio/max": 1.48490571975708,
      "sampling/importance_sampling_ratio/mean": 0.9619448781013489,
      "sampling/importance_sampling_ratio/min": 0.7101505398750305,
      "sampling/sampling_logp_difference/max": 0.2518887519836426,
      "sampling/sampling_logp_difference/mean": 0.029204828664660454,
      "step": 575,
      "step_time": 11.865203233001012
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1636.0,
      "completions/max_terminated_length": 1636.0,
      "completions/mean_length": 533.75,
      "completions/mean_terminated_length": 471.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3720168322324753,
      "epoch": 0.00576,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0969101190567017,
      "kl": 2.1063286513090134,
      "learning_rate": 2.249630929170629e-05,
      "loss": 0.0512,
      "num_tokens": 13525197.0,
      "reward": 0.6770831346511841,
      "reward_std": 0.6184979677200317,
      "rewards/rollout_reward_func/mean": 0.6770831346511841,
      "rewards/rollout_reward_func/std": 0.6121088266372681,
      "sampling/importance_sampling_ratio/max": 1.2789733409881592,
      "sampling/importance_sampling_ratio/mean": 0.9489634037017822,
      "sampling/importance_sampling_ratio/min": 0.6328025460243225,
      "sampling/sampling_logp_difference/max": 0.2261214256286621,
      "sampling/sampling_logp_difference/mean": 0.028420433402061462,
      "step": 576,
      "step_time": 10.529581845999928
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1656.0,
      "completions/max_terminated_length": 1656.0,
      "completions/mean_length": 493.375,
      "completions/mean_terminated_length": 343.4545593261719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8279696628451347,
      "epoch": 0.00577,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1420340538024902,
      "kl": 1.7110085859894753,
      "learning_rate": 2.2427173700654968e-05,
      "loss": -0.0282,
      "num_tokens": 13547951.0,
      "reward": 0.546736478805542,
      "reward_std": 0.7618032693862915,
      "rewards/rollout_reward_func/mean": 0.546736478805542,
      "rewards/rollout_reward_func/std": 0.7395371198654175,
      "sampling/importance_sampling_ratio/max": 1.2077327966690063,
      "sampling/importance_sampling_ratio/mean": 0.9520101547241211,
      "sampling/importance_sampling_ratio/min": 0.4581642746925354,
      "sampling/sampling_logp_difference/max": 0.3095378875732422,
      "sampling/sampling_logp_difference/mean": 0.030981533229351044,
      "step": 577,
      "step_time": 11.395136101999924
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1892.0,
      "completions/max_terminated_length": 780.0,
      "completions/mean_length": 643.6875,
      "completions/mean_terminated_length": 365.1000061035156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4229283146560192,
      "epoch": 0.00578,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6123243570327759,
      "kl": 1.6653584241867065,
      "learning_rate": 2.2358106614271432e-05,
      "loss": -0.1101,
      "num_tokens": 13573012.0,
      "reward": 0.5455800294876099,
      "reward_std": 0.6540078520774841,
      "rewards/rollout_reward_func/mean": 0.5455800294876099,
      "rewards/rollout_reward_func/std": 0.6425301432609558,
      "sampling/importance_sampling_ratio/max": 1.7746559381484985,
      "sampling/importance_sampling_ratio/mean": 1.0848283767700195,
      "sampling/importance_sampling_ratio/min": 0.6706281304359436,
      "sampling/sampling_logp_difference/max": 0.2980153560638428,
      "sampling/sampling_logp_difference/mean": 0.027461858466267586,
      "step": 578,
      "step_time": 12.563402591998965
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1746.0,
      "completions/max_terminated_length": 1746.0,
      "completions/mean_length": 672.0625,
      "completions/mean_terminated_length": 553.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7331793382763863,
      "epoch": 0.00579,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2445197105407715,
      "kl": 2.1172066032886505,
      "learning_rate": 2.228910885572215e-05,
      "loss": -0.0217,
      "num_tokens": 13599082.0,
      "reward": 0.7405202388763428,
      "reward_std": 0.5662730932235718,
      "rewards/rollout_reward_func/mean": 0.7405202388763428,
      "rewards/rollout_reward_func/std": 0.590585470199585,
      "sampling/importance_sampling_ratio/max": 1.2565866708755493,
      "sampling/importance_sampling_ratio/mean": 0.8660517930984497,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.31246161460876465,
      "sampling/sampling_logp_difference/mean": 0.03307899087667465,
      "step": 579,
      "step_time": 11.792298471000322
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1671.0,
      "completions/max_terminated_length": 1671.0,
      "completions/mean_length": 560.1875,
      "completions/mean_terminated_length": 541.0833740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.84732486307621,
      "epoch": 0.0058,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2209080457687378,
      "kl": 1.43480234593153,
      "learning_rate": 2.2220181247347294e-05,
      "loss": 0.0062,
      "num_tokens": 13622702.0,
      "reward": 0.7246373891830444,
      "reward_std": 0.48075729608535767,
      "rewards/rollout_reward_func/mean": 0.7246373891830444,
      "rewards/rollout_reward_func/std": 0.49264994263648987,
      "sampling/importance_sampling_ratio/max": 1.340293049812317,
      "sampling/importance_sampling_ratio/mean": 0.964871883392334,
      "sampling/importance_sampling_ratio/min": 0.5637698173522949,
      "sampling/sampling_logp_difference/max": 0.2054119110107422,
      "sampling/sampling_logp_difference/mean": 0.02911415323615074,
      "step": 580,
      "step_time": 11.045323579999604
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 928.0,
      "completions/max_terminated_length": 885.0,
      "completions/mean_length": 502.375,
      "completions/mean_terminated_length": 474.0000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7465177550911903,
      "epoch": 0.00581,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.940177857875824,
      "kl": 3.319881558418274,
      "learning_rate": 2.2151324610650976e-05,
      "loss": 0.0903,
      "num_tokens": 13645965.0,
      "reward": 0.5018174648284912,
      "reward_std": 0.7716896533966064,
      "rewards/rollout_reward_func/mean": 0.5018174648284912,
      "rewards/rollout_reward_func/std": 0.7987267374992371,
      "sampling/importance_sampling_ratio/max": 1.1730015277862549,
      "sampling/importance_sampling_ratio/mean": 0.988430380821228,
      "sampling/importance_sampling_ratio/min": 0.6871616244316101,
      "sampling/sampling_logp_difference/max": 0.1154029369354248,
      "sampling/sampling_logp_difference/mean": 0.016635455191135406,
      "step": 581,
      "step_time": 8.260957365999275
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1849.0,
      "completions/max_terminated_length": 924.0,
      "completions/mean_length": 780.3125,
      "completions/mean_terminated_length": 591.9091186523438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4032173603773117,
      "epoch": 0.00582,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4403680562973022,
      "kl": 2.5334279537200928,
      "learning_rate": 2.2082539766291436e-05,
      "loss": 0.1916,
      "num_tokens": 13673397.0,
      "reward": 0.25055599212646484,
      "reward_std": 0.752662718296051,
      "rewards/rollout_reward_func/mean": 0.25055599212646484,
      "rewards/rollout_reward_func/std": 0.8373469114303589,
      "sampling/importance_sampling_ratio/max": 1.2482988834381104,
      "sampling/importance_sampling_ratio/mean": 0.8628990650177002,
      "sampling/importance_sampling_ratio/min": 2.953794977708646e-13,
      "sampling/sampling_logp_difference/max": 19.44001007080078,
      "sampling/sampling_logp_difference/mean": 0.11240914463996887,
      "step": 582,
      "step_time": 12.003233620000174
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2751.0,
      "completions/max_terminated_length": 888.0,
      "completions/mean_length": 410.4375,
      "completions/mean_terminated_length": 182.4615478515625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3597935400903225,
      "epoch": 0.00583,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8382539749145508,
      "kl": 2.0789251141250134,
      "learning_rate": 2.2013827534071275e-05,
      "loss": 0.0659,
      "num_tokens": 13693518.0,
      "reward": 0.8014612197875977,
      "reward_std": 0.43126189708709717,
      "rewards/rollout_reward_func/mean": 0.8014612197875977,
      "rewards/rollout_reward_func/std": 0.42689791321754456,
      "sampling/importance_sampling_ratio/max": 1.086810827255249,
      "sampling/importance_sampling_ratio/mean": 0.9801361560821533,
      "sampling/importance_sampling_ratio/min": 0.892059862613678,
      "sampling/sampling_logp_difference/max": 0.15803813934326172,
      "sampling/sampling_logp_difference/mean": 0.024083495140075684,
      "step": 583,
      "step_time": 13.030310668999846
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 874.0,
      "completions/max_terminated_length": 793.0,
      "completions/mean_length": 608.625,
      "completions/mean_terminated_length": 518.8181762695312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2587800845503807,
      "epoch": 0.00584,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6147938370704651,
      "kl": 2.2855528444051743,
      "learning_rate": 2.1945188732927657e-05,
      "loss": 0.0116,
      "num_tokens": 13718401.0,
      "reward": 0.7993201017379761,
      "reward_std": 0.4385315179824829,
      "rewards/rollout_reward_func/mean": 0.7993201017379761,
      "rewards/rollout_reward_func/std": 0.43155911564826965,
      "sampling/importance_sampling_ratio/max": 1.1529583930969238,
      "sampling/importance_sampling_ratio/mean": 0.908941924571991,
      "sampling/importance_sampling_ratio/min": 2.034629892671638e-26,
      "sampling/sampling_logp_difference/max": 26.456314086914062,
      "sampling/sampling_logp_difference/mean": 0.23727014660835266,
      "step": 584,
      "step_time": 9.69913623299999
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 876.0,
      "completions/max_terminated_length": 876.0,
      "completions/mean_length": 440.4375,
      "completions/mean_terminated_length": 405.3571472167969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1542328968644142,
      "epoch": 0.00585,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9700044989585876,
      "kl": 3.1057949513196945,
      "learning_rate": 2.187662418092259e-05,
      "loss": -0.0837,
      "num_tokens": 13739868.0,
      "reward": 0.2388359159231186,
      "reward_std": 0.8282045125961304,
      "rewards/rollout_reward_func/mean": 0.2388359159231186,
      "rewards/rollout_reward_func/std": 0.8507248163223267,
      "sampling/importance_sampling_ratio/max": 1.4635441303253174,
      "sampling/importance_sampling_ratio/mean": 1.0049911737442017,
      "sampling/importance_sampling_ratio/min": 0.6654015779495239,
      "sampling/sampling_logp_difference/max": 0.18203258514404297,
      "sampling/sampling_logp_difference/mean": 0.02296440862119198,
      "step": 585,
      "step_time": 7.847535527000218
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1699.0,
      "completions/max_terminated_length": 1699.0,
      "completions/mean_length": 731.5,
      "completions/mean_terminated_length": 731.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.999927956610918,
      "epoch": 0.00586,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2754478454589844,
      "kl": 2.37445205450058,
      "learning_rate": 2.180813469523315e-05,
      "loss": 0.2826,
      "num_tokens": 13766789.0,
      "reward": 0.8063634634017944,
      "reward_std": 0.4223778247833252,
      "rewards/rollout_reward_func/mean": 0.8063634634017944,
      "rewards/rollout_reward_func/std": 0.4163627326488495,
      "sampling/importance_sampling_ratio/max": 1.1800395250320435,
      "sampling/importance_sampling_ratio/mean": 0.9952890872955322,
      "sampling/importance_sampling_ratio/min": 0.7920737266540527,
      "sampling/sampling_logp_difference/max": 0.1668229103088379,
      "sampling/sampling_logp_difference/mean": 0.02526860311627388,
      "step": 586,
      "step_time": 11.237717359998896
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1791.0,
      "completions/max_terminated_length": 1649.0,
      "completions/mean_length": 538.8125,
      "completions/mean_terminated_length": 410.3077087402344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.295318253338337,
      "epoch": 0.00587,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1674933433532715,
      "kl": 2.6676629967987537,
      "learning_rate": 2.1739721092141717e-05,
      "loss": 0.0325,
      "num_tokens": 13789834.0,
      "reward": 0.6605141162872314,
      "reward_std": 0.5334235429763794,
      "rewards/rollout_reward_func/mean": 0.6605141162872314,
      "rewards/rollout_reward_func/std": 0.5204169750213623,
      "sampling/importance_sampling_ratio/max": 1.190917730331421,
      "sampling/importance_sampling_ratio/mean": 0.9620643258094788,
      "sampling/importance_sampling_ratio/min": 7.271952367900371e-21,
      "sampling/sampling_logp_difference/max": 23.607830047607422,
      "sampling/sampling_logp_difference/mean": 0.19796687364578247,
      "step": 587,
      "step_time": 11.313493981997453
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1013.0,
      "completions/max_terminated_length": 1013.0,
      "completions/mean_length": 614.875,
      "completions/mean_terminated_length": 586.357177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1248888671398163,
      "epoch": 0.00588,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3718855381011963,
      "kl": 3.4282922223210335,
      "learning_rate": 2.167138418702632e-05,
      "loss": 0.2116,
      "num_tokens": 13814651.0,
      "reward": 0.7418703436851501,
      "reward_std": 0.5991032719612122,
      "rewards/rollout_reward_func/mean": 0.7418703436851501,
      "rewards/rollout_reward_func/std": 0.5872834324836731,
      "sampling/importance_sampling_ratio/max": 1.2524906396865845,
      "sampling/importance_sampling_ratio/mean": 0.9449061155319214,
      "sampling/importance_sampling_ratio/min": 2.070462380980579e-22,
      "sampling/sampling_logp_difference/max": 8.7053861618042,
      "sampling/sampling_logp_difference/mean": 0.2597668468952179,
      "step": 588,
      "step_time": 8.943089084001258
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1902.0,
      "completions/max_terminated_length": 1902.0,
      "completions/mean_length": 614.25,
      "completions/mean_terminated_length": 459.7857360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0866472609341145,
      "epoch": 0.00589,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.66602623462677,
      "kl": 2.03484545648098,
      "learning_rate": 2.1603124794350848e-05,
      "loss": 0.1285,
      "num_tokens": 13838381.0,
      "reward": 0.5062075853347778,
      "reward_std": 0.7433209419250488,
      "rewards/rollout_reward_func/mean": 0.5062075853347778,
      "rewards/rollout_reward_func/std": 0.7977257966995239,
      "sampling/importance_sampling_ratio/max": 1.362973690032959,
      "sampling/importance_sampling_ratio/mean": 1.003967523574829,
      "sampling/importance_sampling_ratio/min": 0.7815729975700378,
      "sampling/sampling_logp_difference/max": 0.2071552276611328,
      "sampling/sampling_logp_difference/mean": 0.024308761581778526,
      "step": 589,
      "step_time": 11.544230393999896
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 900.0,
      "completions/max_terminated_length": 829.0,
      "completions/mean_length": 459.8125,
      "completions/mean_terminated_length": 404.5714416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9882497452199459,
      "epoch": 0.0059,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6494209170341492,
      "kl": 2.949972216039896,
      "learning_rate": 2.1534943727655363e-05,
      "loss": -0.0616,
      "num_tokens": 13860734.0,
      "reward": 0.5369995832443237,
      "reward_std": 0.629214346408844,
      "rewards/rollout_reward_func/mean": 0.5369995832443237,
      "rewards/rollout_reward_func/std": 0.6504138112068176,
      "sampling/importance_sampling_ratio/max": 1.3569241762161255,
      "sampling/importance_sampling_ratio/mean": 1.0125453472137451,
      "sampling/importance_sampling_ratio/min": 0.7523367404937744,
      "sampling/sampling_logp_difference/max": 0.10883617401123047,
      "sampling/sampling_logp_difference/mean": 0.020246684551239014,
      "step": 590,
      "step_time": 8.061827260999053
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 851.0,
      "completions/max_terminated_length": 851.0,
      "completions/mean_length": 476.0625,
      "completions/mean_terminated_length": 476.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1603298410773277,
      "epoch": 0.00591,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2603288888931274,
      "kl": 3.0227127373218536,
      "learning_rate": 2.146684179954643e-05,
      "loss": 0.0666,
      "num_tokens": 13883410.0,
      "reward": 0.6215579509735107,
      "reward_std": 0.6421394348144531,
      "rewards/rollout_reward_func/mean": 0.6215579509735107,
      "rewards/rollout_reward_func/std": 0.7150582671165466,
      "sampling/importance_sampling_ratio/max": 1.4485291242599487,
      "sampling/importance_sampling_ratio/mean": 1.049468755722046,
      "sampling/importance_sampling_ratio/min": 0.9076827764511108,
      "sampling/sampling_logp_difference/max": 0.24439048767089844,
      "sampling/sampling_logp_difference/mean": 0.02072523720562458,
      "step": 591,
      "step_time": 7.967439489999379
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2494.0,
      "completions/max_terminated_length": 943.0,
      "completions/mean_length": 578.875,
      "completions/mean_terminated_length": 325.3846130371094,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9402407892048359,
      "epoch": 0.00592,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8629605174064636,
      "kl": 3.2452733293175697,
      "learning_rate": 2.1398819821687402e-05,
      "loss": -0.0089,
      "num_tokens": 13907573.0,
      "reward": 0.23060554265975952,
      "reward_std": 0.7994490265846252,
      "rewards/rollout_reward_func/mean": 0.23060554265975952,
      "rewards/rollout_reward_func/std": 0.7767879962921143,
      "sampling/importance_sampling_ratio/max": 2.3779571056365967,
      "sampling/importance_sampling_ratio/mean": 1.029057264328003,
      "sampling/importance_sampling_ratio/min": 0.5002110600471497,
      "sampling/sampling_logp_difference/max": 0.2325582504272461,
      "sampling/sampling_logp_difference/mean": 0.024426011368632317,
      "step": 592,
      "step_time": 13.255978840000353
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2659.0,
      "completions/max_terminated_length": 2659.0,
      "completions/mean_length": 540.625,
      "completions/mean_terminated_length": 513.0667114257812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0193525813519955,
      "epoch": 0.00593,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.09294593334198,
      "kl": 2.5910213105380535,
      "learning_rate": 2.133087860478874e-05,
      "loss": 0.0804,
      "num_tokens": 13930705.0,
      "reward": 0.6654731631278992,
      "reward_std": 0.5241061449050903,
      "rewards/rollout_reward_func/mean": 0.6654731631278992,
      "rewards/rollout_reward_func/std": 0.5127770900726318,
      "sampling/importance_sampling_ratio/max": 1.1142692565917969,
      "sampling/importance_sampling_ratio/mean": 0.9155042171478271,
      "sampling/importance_sampling_ratio/min": 2.7386465292431214e-18,
      "sampling/sampling_logp_difference/max": 14.25793743133545,
      "sampling/sampling_logp_difference/mean": 0.20508195459842682,
      "step": 593,
      "step_time": 13.408213925001291
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1788.0,
      "completions/max_terminated_length": 907.0,
      "completions/mean_length": 443.0,
      "completions/mean_terminated_length": 198.69232177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.157866157591343,
      "epoch": 0.00594,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.257294774055481,
      "kl": 0.9820532314479351,
      "learning_rate": 2.1263018958598393e-05,
      "loss": 0.1606,
      "num_tokens": 13951459.0,
      "reward": 0.4711407423019409,
      "reward_std": 0.6672821044921875,
      "rewards/rollout_reward_func/mean": 0.4711407423019409,
      "rewards/rollout_reward_func/std": 0.6498852372169495,
      "sampling/importance_sampling_ratio/max": 1.5738177299499512,
      "sampling/importance_sampling_ratio/mean": 1.0494228601455688,
      "sampling/importance_sampling_ratio/min": 0.7016568183898926,
      "sampling/sampling_logp_difference/max": 0.345367431640625,
      "sampling/sampling_logp_difference/mean": 0.02649499475955963,
      "step": 594,
      "step_time": 10.594913960999293
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1691.0,
      "completions/max_terminated_length": 916.0,
      "completions/mean_length": 519.5625,
      "completions/mean_terminated_length": 403.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0740090236067772,
      "epoch": 0.00595,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6102277636528015,
      "kl": 2.9374351650476456,
      "learning_rate": 2.1195241691892116e-05,
      "loss": -0.0236,
      "num_tokens": 13974493.0,
      "reward": 0.603215754032135,
      "reward_std": 0.5700127482414246,
      "rewards/rollout_reward_func/mean": 0.603215754032135,
      "rewards/rollout_reward_func/std": 0.6394416689872742,
      "sampling/importance_sampling_ratio/max": 1.1160764694213867,
      "sampling/importance_sampling_ratio/mean": 0.9283337593078613,
      "sampling/importance_sampling_ratio/min": 5.416054809787411e-18,
      "sampling/sampling_logp_difference/max": 20.39003562927246,
      "sampling/sampling_logp_difference/mean": 0.2750048041343689,
      "step": 595,
      "step_time": 10.412998033001713
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1823.0,
      "completions/max_terminated_length": 953.0,
      "completions/mean_length": 576.0,
      "completions/mean_terminated_length": 447.3846435546875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9365524016320705,
      "epoch": 0.00596,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.333216667175293,
      "kl": 2.5336154252290726,
      "learning_rate": 2.1127547612463807e-05,
      "loss": 0.3864,
      "num_tokens": 13998466.0,
      "reward": 0.5456976294517517,
      "reward_std": 0.6500083804130554,
      "rewards/rollout_reward_func/mean": 0.5456976294517517,
      "rewards/rollout_reward_func/std": 0.6379139423370361,
      "sampling/importance_sampling_ratio/max": 1.117544174194336,
      "sampling/importance_sampling_ratio/mean": 0.9427260160446167,
      "sampling/importance_sampling_ratio/min": 0.6408283710479736,
      "sampling/sampling_logp_difference/max": 0.1509103775024414,
      "sampling/sampling_logp_difference/mean": 0.021103549748659134,
      "step": 596,
      "step_time": 10.845983822001472
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 896.0,
      "completions/max_terminated_length": 896.0,
      "completions/mean_length": 494.0,
      "completions/mean_terminated_length": 431.0769348144531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.168479673564434,
      "epoch": 0.00597,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8350818753242493,
      "kl": 13.239249661564827,
      "learning_rate": 2.1059937527115955e-05,
      "loss": -0.0354,
      "num_tokens": 14021302.0,
      "reward": 0.6697325706481934,
      "reward_std": 0.6320565342903137,
      "rewards/rollout_reward_func/mean": 0.6697325706481934,
      "rewards/rollout_reward_func/std": 0.624255895614624,
      "sampling/importance_sampling_ratio/max": 1.2509099245071411,
      "sampling/importance_sampling_ratio/mean": 0.9672833681106567,
      "sampling/importance_sampling_ratio/min": 0.6539680361747742,
      "sampling/sampling_logp_difference/max": 0.2392568588256836,
      "sampling/sampling_logp_difference/mean": 0.02552494779229164,
      "step": 597,
      "step_time": 8.149917720998928
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1600.0,
      "completions/max_terminated_length": 1600.0,
      "completions/mean_length": 524.8125,
      "completions/mean_terminated_length": 524.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1496613826602697,
      "epoch": 0.00598,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1525787115097046,
      "kl": 2.205450512468815,
      "learning_rate": 2.099241224164994e-05,
      "loss": 0.0831,
      "num_tokens": 14044416.0,
      "reward": 0.3530844449996948,
      "reward_std": 0.6927666664123535,
      "rewards/rollout_reward_func/mean": 0.3530844449996948,
      "rewards/rollout_reward_func/std": 0.7308857440948486,
      "sampling/importance_sampling_ratio/max": 1.194742202758789,
      "sampling/importance_sampling_ratio/mean": 0.9619082808494568,
      "sampling/importance_sampling_ratio/min": 0.6830697655677795,
      "sampling/sampling_logp_difference/max": 0.32778334617614746,
      "sampling/sampling_logp_difference/mean": 0.022125601768493652,
      "step": 598,
      "step_time": 10.783286978000433
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1584.0,
      "completions/max_terminated_length": 1584.0,
      "completions/mean_length": 556.3125,
      "completions/mean_terminated_length": 512.2857666015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6792304404079914,
      "epoch": 0.00599,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6630924940109253,
      "kl": 1.990056797862053,
      "learning_rate": 2.0924972560856487e-05,
      "loss": 0.0666,
      "num_tokens": 14068246.0,
      "reward": 0.6703582406044006,
      "reward_std": 0.5826935172080994,
      "rewards/rollout_reward_func/mean": 0.6703582406044006,
      "rewards/rollout_reward_func/std": 0.6238357424736023,
      "sampling/importance_sampling_ratio/max": 1.4908597469329834,
      "sampling/importance_sampling_ratio/mean": 1.0478103160858154,
      "sampling/importance_sampling_ratio/min": 0.7509927153587341,
      "sampling/sampling_logp_difference/max": 0.21262693405151367,
      "sampling/sampling_logp_difference/mean": 0.0292524304240942,
      "step": 599,
      "step_time": 10.510523278000619
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 892.0,
      "completions/max_terminated_length": 819.0,
      "completions/mean_length": 280.25,
      "completions/mean_terminated_length": 239.4666748046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7019415833055973,
      "epoch": 0.006,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4444214701652527,
      "kl": 2.213603585958481,
      "learning_rate": 2.0857619288506032e-05,
      "loss": 0.0405,
      "num_tokens": 14086207.0,
      "reward": 0.4908999502658844,
      "reward_std": 0.754388689994812,
      "rewards/rollout_reward_func/mean": 0.4908999502658844,
      "rewards/rollout_reward_func/std": 0.7288393378257751,
      "sampling/importance_sampling_ratio/max": 1.0916309356689453,
      "sampling/importance_sampling_ratio/mean": 0.9821410775184631,
      "sampling/importance_sampling_ratio/min": 0.798694372177124,
      "sampling/sampling_logp_difference/max": 0.17668747901916504,
      "sampling/sampling_logp_difference/mean": 0.020010709762573242,
      "step": 600,
      "step_time": 7.229688613999315
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 899.0,
      "completions/max_terminated_length": 899.0,
      "completions/mean_length": 517.8125,
      "completions/mean_terminated_length": 517.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6902366057038307,
      "epoch": 0.00601,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.495932698249817,
      "kl": 7.075789093971252,
      "learning_rate": 2.0790353227339193e-05,
      "loss": 0.0059,
      "num_tokens": 14110056.0,
      "reward": 0.4716714918613434,
      "reward_std": 0.6690956950187683,
      "rewards/rollout_reward_func/mean": 0.4716714918613434,
      "rewards/rollout_reward_func/std": 0.6528115272521973,
      "sampling/importance_sampling_ratio/max": 1.1619176864624023,
      "sampling/importance_sampling_ratio/mean": 0.9889165163040161,
      "sampling/importance_sampling_ratio/min": 0.7197882533073425,
      "sampling/sampling_logp_difference/max": 0.23586368560791016,
      "sampling/sampling_logp_difference/mean": 0.020880460739135742,
      "step": 601,
      "step_time": 7.580693921998318
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 962.0,
      "completions/max_terminated_length": 962.0,
      "completions/mean_length": 531.3125,
      "completions/mean_terminated_length": 531.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8268904574215412,
      "epoch": 0.00602,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7088703513145447,
      "kl": 2.9078332632780075,
      "learning_rate": 2.0723175179057148e-05,
      "loss": 0.0271,
      "num_tokens": 14133189.0,
      "reward": 0.25641918182373047,
      "reward_std": 0.5302029848098755,
      "rewards/rollout_reward_func/mean": 0.25641918182373047,
      "rewards/rollout_reward_func/std": 0.5179427266120911,
      "sampling/importance_sampling_ratio/max": 1.1759432554244995,
      "sampling/importance_sampling_ratio/mean": 0.9900038242340088,
      "sampling/importance_sampling_ratio/min": 0.8648210167884827,
      "sampling/sampling_logp_difference/max": 0.25451767444610596,
      "sampling/sampling_logp_difference/mean": 0.019035711884498596,
      "step": 602,
      "step_time": 7.947131390999857
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 971.0,
      "completions/max_terminated_length": 971.0,
      "completions/mean_length": 587.125,
      "completions/mean_terminated_length": 603.1538696289062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9667786210775375,
      "epoch": 0.00603,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5629305243492126,
      "kl": 2.6020740531384945,
      "learning_rate": 2.0656085944312112e-05,
      "loss": -0.0699,
      "num_tokens": 14157975.0,
      "reward": 0.7279915809631348,
      "reward_std": 0.47321075201034546,
      "rewards/rollout_reward_func/mean": 0.7279915809631348,
      "rewards/rollout_reward_func/std": 0.4866364598274231,
      "sampling/importance_sampling_ratio/max": 1.3484758138656616,
      "sampling/importance_sampling_ratio/mean": 0.996452271938324,
      "sampling/importance_sampling_ratio/min": 0.5870487093925476,
      "sampling/sampling_logp_difference/max": 0.23509836196899414,
      "sampling/sampling_logp_difference/mean": 0.024260392412543297,
      "step": 603,
      "step_time": 8.641802758999802
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 792.0,
      "completions/max_terminated_length": 792.0,
      "completions/mean_length": 248.875,
      "completions/mean_terminated_length": 248.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.542995173484087,
      "epoch": 0.00604,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.1414765566587448,
      "kl": 1.6832859329879284,
      "learning_rate": 2.0589086322697805e-05,
      "loss": -0.0166,
      "num_tokens": 14175385.0,
      "reward": 0.7915946245193481,
      "reward_std": 0.45428431034088135,
      "rewards/rollout_reward_func/mean": 0.7915946245193481,
      "rewards/rollout_reward_func/std": 0.4480848014354706,
      "sampling/importance_sampling_ratio/max": 1.18201744556427,
      "sampling/importance_sampling_ratio/mean": 1.0005208253860474,
      "sampling/importance_sampling_ratio/min": 0.9252990484237671,
      "sampling/sampling_logp_difference/max": 0.11885237693786621,
      "sampling/sampling_logp_difference/mean": 0.012985472567379475,
      "step": 604,
      "step_time": 6.837579208000534
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1831.0,
      "completions/max_terminated_length": 934.0,
      "completions/mean_length": 347.25,
      "completions/mean_terminated_length": 248.33334350585938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8629022873938084,
      "epoch": 0.00605,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7953112721443176,
      "kl": 1.3158410973846912,
      "learning_rate": 2.052217711273989e-05,
      "loss": 0.1041,
      "num_tokens": 14195229.0,
      "reward": 0.5239043235778809,
      "reward_std": 0.5734065771102905,
      "rewards/rollout_reward_func/mean": 0.5239043235778809,
      "rewards/rollout_reward_func/std": 0.5579221844673157,
      "sampling/importance_sampling_ratio/max": 1.1085819005966187,
      "sampling/importance_sampling_ratio/mean": 1.0150554180145264,
      "sampling/importance_sampling_ratio/min": 0.9353727102279663,
      "sampling/sampling_logp_difference/max": 0.11303520202636719,
      "sampling/sampling_logp_difference/mean": 0.02232348546385765,
      "step": 605,
      "step_time": 10.414645790999202
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1812.0,
      "completions/max_terminated_length": 1812.0,
      "completions/mean_length": 488.1875,
      "completions/mean_terminated_length": 488.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9909847527742386,
      "epoch": 0.00606,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4297398030757904,
      "kl": 2.4697314091026783,
      "learning_rate": 2.045535911188646e-05,
      "loss": 0.0673,
      "num_tokens": 14217887.0,
      "reward": 0.59358811378479,
      "reward_std": 0.5614123344421387,
      "rewards/rollout_reward_func/mean": 0.59358811378479,
      "rewards/rollout_reward_func/std": 0.5424162745475769,
      "sampling/importance_sampling_ratio/max": 1.1533560752868652,
      "sampling/importance_sampling_ratio/mean": 0.9813312292098999,
      "sampling/importance_sampling_ratio/min": 0.4641493856906891,
      "sampling/sampling_logp_difference/max": 0.1572709083557129,
      "sampling/sampling_logp_difference/mean": 0.023230165243148804,
      "step": 606,
      "step_time": 10.206437961998745
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1879.0,
      "completions/max_terminated_length": 1879.0,
      "completions/mean_length": 418.375,
      "completions/mean_terminated_length": 418.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2869390919804573,
      "epoch": 0.00607,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7819638848304749,
      "kl": 1.837259110994637,
      "learning_rate": 2.0388633116498584e-05,
      "loss": 0.0892,
      "num_tokens": 14238505.0,
      "reward": 0.5373942852020264,
      "reward_std": 0.6635672450065613,
      "rewards/rollout_reward_func/mean": 0.5373942852020264,
      "rewards/rollout_reward_func/std": 0.6513210535049438,
      "sampling/importance_sampling_ratio/max": 1.4808622598648071,
      "sampling/importance_sampling_ratio/mean": 1.024658203125,
      "sampling/importance_sampling_ratio/min": 0.5336468815803528,
      "sampling/sampling_logp_difference/max": 0.39388179779052734,
      "sampling/sampling_logp_difference/mean": 0.02842026576399803,
      "step": 607,
      "step_time": 10.119217575998846
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1646.0,
      "completions/max_terminated_length": 1514.0,
      "completions/mean_length": 625.5,
      "completions/mean_terminated_length": 557.4666748046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.403266079723835,
      "epoch": 0.00608,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6181462407112122,
      "kl": 2.080050751566887,
      "learning_rate": 2.0321999921840726e-05,
      "loss": -0.1616,
      "num_tokens": 14264032.0,
      "reward": 0.6721682548522949,
      "reward_std": 0.6265424489974976,
      "rewards/rollout_reward_func/mean": 0.6721682548522949,
      "rewards/rollout_reward_func/std": 0.6182142496109009,
      "sampling/importance_sampling_ratio/max": 1.4406567811965942,
      "sampling/importance_sampling_ratio/mean": 1.0391013622283936,
      "sampling/importance_sampling_ratio/min": 0.8884077072143555,
      "sampling/sampling_logp_difference/max": 0.17791223526000977,
      "sampling/sampling_logp_difference/mean": 0.02867831103503704,
      "step": 608,
      "step_time": 10.477765500001624
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1658.0,
      "completions/max_terminated_length": 962.0,
      "completions/mean_length": 589.625,
      "completions/mean_terminated_length": 501.357177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9788664132356644,
      "epoch": 0.00609,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.376350998878479,
      "kl": 2.600955478847027,
      "learning_rate": 2.0255460322071353e-05,
      "loss": -0.031,
      "num_tokens": 14287945.0,
      "reward": 0.7302932739257812,
      "reward_std": 0.46984922885894775,
      "rewards/rollout_reward_func/mean": 0.7302932739257812,
      "rewards/rollout_reward_func/std": 0.4824662506580353,
      "sampling/importance_sampling_ratio/max": 1.0479470491409302,
      "sampling/importance_sampling_ratio/mean": 0.9796723127365112,
      "sampling/importance_sampling_ratio/min": 0.7711201906204224,
      "sampling/sampling_logp_difference/max": 0.22271156311035156,
      "sampling/sampling_logp_difference/mean": 0.019082840532064438,
      "step": 609,
      "step_time": 9.972094440001456
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 925.0,
      "completions/max_terminated_length": 925.0,
      "completions/mean_length": 435.375,
      "completions/mean_terminated_length": 435.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9085268247872591,
      "epoch": 0.0061,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4061870872974396,
      "kl": 1.972818523645401,
      "learning_rate": 2.0189015110233424e-05,
      "loss": 0.0057,
      "num_tokens": 14309420.0,
      "reward": 0.5907848477363586,
      "reward_std": 0.5648421049118042,
      "rewards/rollout_reward_func/mean": 0.5907848477363586,
      "rewards/rollout_reward_func/std": 0.5457659363746643,
      "sampling/importance_sampling_ratio/max": 1.107696294784546,
      "sampling/importance_sampling_ratio/mean": 0.9990531206130981,
      "sampling/importance_sampling_ratio/min": 0.9094392657279968,
      "sampling/sampling_logp_difference/max": 0.13893276453018188,
      "sampling/sampling_logp_difference/mean": 0.016368767246603966,
      "step": 610,
      "step_time": 7.622742223000387
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 929.0,
      "completions/max_terminated_length": 929.0,
      "completions/mean_length": 513.125,
      "completions/mean_terminated_length": 513.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4886610247194767,
      "epoch": 0.00611,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5417477488517761,
      "kl": 4.567134886980057,
      "learning_rate": 2.012266507824495e-05,
      "loss": -0.0178,
      "num_tokens": 14332968.0,
      "reward": 0.2210993766784668,
      "reward_std": 0.795758843421936,
      "rewards/rollout_reward_func/mean": 0.2210993766784668,
      "rewards/rollout_reward_func/std": 0.7736794948577881,
      "sampling/importance_sampling_ratio/max": 1.2220454216003418,
      "sampling/importance_sampling_ratio/mean": 0.9951769113540649,
      "sampling/importance_sampling_ratio/min": 0.6864623427391052,
      "sampling/sampling_logp_difference/max": 0.21149826049804688,
      "sampling/sampling_logp_difference/mean": 0.02804221771657467,
      "step": 611,
      "step_time": 7.903136291000919
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 765.0,
      "completions/max_terminated_length": 765.0,
      "completions/mean_length": 399.6875,
      "completions/mean_terminated_length": 377.5333557128906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0259610451757908,
      "epoch": 0.00612,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.45559048652648926,
      "kl": 1.6750146131962538,
      "learning_rate": 2.0056411016889552e-05,
      "loss": -0.0777,
      "num_tokens": 14353909.0,
      "reward": 0.7905893921852112,
      "reward_std": 0.45564812421798706,
      "rewards/rollout_reward_func/mean": 0.7905893921852112,
      "rewards/rollout_reward_func/std": 0.4502219259738922,
      "sampling/importance_sampling_ratio/max": 1.695064902305603,
      "sampling/importance_sampling_ratio/mean": 1.0830081701278687,
      "sampling/importance_sampling_ratio/min": 0.8082618713378906,
      "sampling/sampling_logp_difference/max": 0.18473243713378906,
      "sampling/sampling_logp_difference/mean": 0.026298491284251213,
      "step": 612,
      "step_time": 7.626202151998768
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1630.0,
      "completions/max_terminated_length": 1630.0,
      "completions/mean_length": 548.4375,
      "completions/mean_terminated_length": 543.7333374023438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8441150188446045,
      "epoch": 0.00613,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5978630781173706,
      "kl": 1.2981442101299763,
      "learning_rate": 1.9990253715807038e-05,
      "loss": -0.113,
      "num_tokens": 14378045.0,
      "reward": 0.6680893898010254,
      "reward_std": 0.6352653503417969,
      "rewards/rollout_reward_func/mean": 0.6680893898010254,
      "rewards/rollout_reward_func/std": 0.6267180442810059,
      "sampling/importance_sampling_ratio/max": 1.1531709432601929,
      "sampling/importance_sampling_ratio/mean": 1.0019423961639404,
      "sampling/importance_sampling_ratio/min": 0.6154673099517822,
      "sampling/sampling_logp_difference/max": 0.2028266191482544,
      "sampling/sampling_logp_difference/mean": 0.03362268581986427,
      "step": 613,
      "step_time": 10.398712123001133
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1857.0,
      "completions/max_terminated_length": 1857.0,
      "completions/mean_length": 467.1875,
      "completions/mean_terminated_length": 419.4285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5443383045494556,
      "epoch": 0.00614,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0306665897369385,
      "kl": 1.109802983701229,
      "learning_rate": 1.9924193963483995e-05,
      "loss": -0.0266,
      "num_tokens": 14400339.0,
      "reward": 0.10227563232183456,
      "reward_std": 0.819173276424408,
      "rewards/rollout_reward_func/mean": 0.10227563232183456,
      "rewards/rollout_reward_func/std": 0.8054653406143188,
      "sampling/importance_sampling_ratio/max": 1.0880545377731323,
      "sampling/importance_sampling_ratio/mean": 0.94884192943573,
      "sampling/importance_sampling_ratio/min": 0.6907558441162109,
      "sampling/sampling_logp_difference/max": 0.2758049964904785,
      "sampling/sampling_logp_difference/mean": 0.02769414894282818,
      "step": 614,
      "step_time": 10.739854616998855
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1569.0,
      "completions/max_terminated_length": 1569.0,
      "completions/mean_length": 390.0,
      "completions/mean_terminated_length": 351.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2512485422194004,
      "epoch": 0.00615,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.46150878071784973,
      "kl": 1.173178505152464,
      "learning_rate": 1.985823254724439e-05,
      "loss": -0.07,
      "num_tokens": 14421164.0,
      "reward": 0.21879902482032776,
      "reward_std": 0.758994460105896,
      "rewards/rollout_reward_func/mean": 0.21879902482032776,
      "rewards/rollout_reward_func/std": 0.7805491089820862,
      "sampling/importance_sampling_ratio/max": 1.213762640953064,
      "sampling/importance_sampling_ratio/mean": 1.0106863975524902,
      "sampling/importance_sampling_ratio/min": 0.8455726504325867,
      "sampling/sampling_logp_difference/max": 0.19272375106811523,
      "sampling/sampling_logp_difference/mean": 0.029097910970449448,
      "step": 615,
      "step_time": 9.575050008999824
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2475.0,
      "completions/max_terminated_length": 2475.0,
      "completions/mean_length": 657.75,
      "completions/mean_terminated_length": 639.800048828125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6308299396187067,
      "epoch": 0.00616,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.4150848388671875,
      "kl": 1.2367462292313576,
      "learning_rate": 1.9792370253240173e-05,
      "loss": 0.0142,
      "num_tokens": 14446802.0,
      "reward": 0.9325977563858032,
      "reward_std": 0.1906423568725586,
      "rewards/rollout_reward_func/mean": 0.9325977563858032,
      "rewards/rollout_reward_func/std": 0.2696090042591095,
      "sampling/importance_sampling_ratio/max": 1.1794155836105347,
      "sampling/importance_sampling_ratio/mean": 0.9658584594726562,
      "sampling/importance_sampling_ratio/min": 1.0450591587188826e-18,
      "sampling/sampling_logp_difference/max": 22.41102409362793,
      "sampling/sampling_logp_difference/mean": 0.4024604558944702,
      "step": 616,
      "step_time": 12.293242278000434
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1599.0,
      "completions/max_terminated_length": 1599.0,
      "completions/mean_length": 479.9375,
      "completions/mean_terminated_length": 434.5714416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.811702847480774,
      "epoch": 0.00617,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6636028289794922,
      "kl": 0.8233524858951569,
      "learning_rate": 1.9726607866441943e-05,
      "loss": -0.0801,
      "num_tokens": 14468868.0,
      "reward": 0.7428582310676575,
      "reward_std": 0.5982652902603149,
      "rewards/rollout_reward_func/mean": 0.7428582310676575,
      "rewards/rollout_reward_func/std": 0.5875125527381897,
      "sampling/importance_sampling_ratio/max": 1.1634608507156372,
      "sampling/importance_sampling_ratio/mean": 0.9807256460189819,
      "sampling/importance_sampling_ratio/min": 0.5972737669944763,
      "sampling/sampling_logp_difference/max": 0.26251697540283203,
      "sampling/sampling_logp_difference/mean": 0.03302546963095665,
      "step": 617,
      "step_time": 10.445719828999245
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1686.0,
      "completions/max_terminated_length": 1686.0,
      "completions/mean_length": 625.0,
      "completions/mean_terminated_length": 569.0833740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9175482261925936,
      "epoch": 0.00618,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.5116584897041321,
      "kl": 0.7329953443259001,
      "learning_rate": 1.966094617062954e-05,
      "loss": 0.0208,
      "num_tokens": 14494194.0,
      "reward": 0.7536830902099609,
      "reward_std": 0.4560902416706085,
      "rewards/rollout_reward_func/mean": 0.7536830902099609,
      "rewards/rollout_reward_func/std": 0.6730659008026123,
      "sampling/importance_sampling_ratio/max": 2.1432015895843506,
      "sampling/importance_sampling_ratio/mean": 1.097885251045227,
      "sampling/importance_sampling_ratio/min": 0.716617226600647,
      "sampling/sampling_logp_difference/max": 0.16228771209716797,
      "sampling/sampling_logp_difference/mean": 0.029443461447954178,
      "step": 618,
      "step_time": 11.390741646001516
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 890.0,
      "completions/max_terminated_length": 890.0,
      "completions/mean_length": 490.0,
      "completions/mean_terminated_length": 378.6363830566406,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.7422680258750916,
      "epoch": 0.00619,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6405684351921082,
      "kl": 1.5235631167888641,
      "learning_rate": 1.959538594838275e-05,
      "loss": -0.2283,
      "num_tokens": 14517589.0,
      "reward": 0.5610941052436829,
      "reward_std": 0.7064639329910278,
      "rewards/rollout_reward_func/mean": 0.5610941052436829,
      "rewards/rollout_reward_func/std": 0.8082947731018066,
      "sampling/importance_sampling_ratio/max": 1.4578988552093506,
      "sampling/importance_sampling_ratio/mean": 0.9538525342941284,
      "sampling/importance_sampling_ratio/min": 0.574160099029541,
      "sampling/sampling_logp_difference/max": 0.1503751277923584,
      "sampling/sampling_logp_difference/mean": 0.03182472661137581,
      "step": 619,
      "step_time": 8.76031194799998
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2509.0,
      "completions/max_terminated_length": 2509.0,
      "completions/mean_length": 542.375,
      "completions/mean_terminated_length": 519.4000244140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6142706386744976,
      "epoch": 0.0062,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.091845750808716,
      "kl": 0.8481117766350508,
      "learning_rate": 1.952992798107197e-05,
      "loss": 0.3721,
      "num_tokens": 14540457.0,
      "reward": 0.7502443194389343,
      "reward_std": 0.5441944599151611,
      "rewards/rollout_reward_func/mean": 0.7502443194389343,
      "rewards/rollout_reward_func/std": 0.5614885687828064,
      "sampling/importance_sampling_ratio/max": 1.3682987689971924,
      "sampling/importance_sampling_ratio/mean": 1.022026538848877,
      "sampling/importance_sampling_ratio/min": 0.7964625954627991,
      "sampling/sampling_logp_difference/max": 0.15885496139526367,
      "sampling/sampling_logp_difference/mean": 0.02641475573182106,
      "step": 620,
      "step_time": 12.610601682000379
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 3513.0,
      "completions/max_terminated_length": 1744.0,
      "completions/mean_length": 745.375,
      "completions/mean_terminated_length": 490.14288330078125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5299981366842985,
      "epoch": 0.00621,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6603609323501587,
      "kl": 0.6776915192604065,
      "learning_rate": 1.9464573048848857e-05,
      "loss": -0.0021,
      "num_tokens": 14567045.0,
      "reward": 0.48443731665611267,
      "reward_std": 0.609032392501831,
      "rewards/rollout_reward_func/mean": 0.48443731665611267,
      "rewards/rollout_reward_func/std": 0.7354660034179688,
      "sampling/importance_sampling_ratio/max": 1.217870831489563,
      "sampling/importance_sampling_ratio/mean": 0.9843665361404419,
      "sampling/importance_sampling_ratio/min": 0.5317865610122681,
      "sampling/sampling_logp_difference/max": 0.5027866363525391,
      "sampling/sampling_logp_difference/mean": 0.03572823852300644,
      "step": 621,
      "step_time": 15.907793700999719
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2615.0,
      "completions/max_terminated_length": 968.0,
      "completions/mean_length": 667.1875,
      "completions/mean_terminated_length": 519.4285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9090721309185028,
      "epoch": 0.00622,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9564177989959717,
      "kl": 0.812016362324357,
      "learning_rate": 1.939932193063708e-05,
      "loss": 0.0834,
      "num_tokens": 14592296.0,
      "reward": 0.679958164691925,
      "reward_std": 0.5633326172828674,
      "rewards/rollout_reward_func/mean": 0.679958164691925,
      "rewards/rollout_reward_func/std": 0.6039814352989197,
      "sampling/importance_sampling_ratio/max": 1.1025248765945435,
      "sampling/importance_sampling_ratio/mean": 0.9111260771751404,
      "sampling/importance_sampling_ratio/min": 2.566043514826144e-12,
      "sampling/sampling_logp_difference/max": 15.128560066223145,
      "sampling/sampling_logp_difference/mean": 0.15312494337558746,
      "step": 622,
      "step_time": 13.326158222000231
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 806.0,
      "completions/max_terminated_length": 806.0,
      "completions/mean_length": 350.75,
      "completions/mean_terminated_length": 333.73333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1168551333248615,
      "epoch": 0.00623,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.2737251818180084,
      "kl": 0.9551336392760277,
      "learning_rate": 1.9334175404123014e-05,
      "loss": -0.0343,
      "num_tokens": 14611935.0,
      "reward": 0.8056870698928833,
      "reward_std": 0.37872445583343506,
      "rewards/rollout_reward_func/mean": 0.8056870698928833,
      "rewards/rollout_reward_func/std": 0.554990828037262,
      "sampling/importance_sampling_ratio/max": 1.1051219701766968,
      "sampling/importance_sampling_ratio/mean": 0.9712210893630981,
      "sampling/importance_sampling_ratio/min": 0.7441492676734924,
      "sampling/sampling_logp_difference/max": 0.17717194557189941,
      "sampling/sampling_logp_difference/mean": 0.024945463985204697,
      "step": 623,
      "step_time": 7.418414550999842
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1662.0,
      "completions/max_terminated_length": 963.0,
      "completions/mean_length": 537.375,
      "completions/mean_terminated_length": 426.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7591120898723602,
      "epoch": 0.00624,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.1346184015274048,
      "kl": 1.0161640448495746,
      "learning_rate": 1.9269134245746465e-05,
      "loss": 0.0939,
      "num_tokens": 14634929.0,
      "reward": 0.7309411764144897,
      "reward_std": 0.28776705265045166,
      "rewards/rollout_reward_func/mean": 0.7309411764144897,
      "rewards/rollout_reward_func/std": 0.4814532697200775,
      "sampling/importance_sampling_ratio/max": 1.8576422929763794,
      "sampling/importance_sampling_ratio/mean": 0.9456809759140015,
      "sampling/importance_sampling_ratio/min": 1.2758862542872063e-11,
      "sampling/sampling_logp_difference/max": 16.536481857299805,
      "sampling/sampling_logp_difference/mean": 0.2343236804008484,
      "step": 624,
      "step_time": 10.474562792000143
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1638.0,
      "completions/max_terminated_length": 1638.0,
      "completions/mean_length": 684.625,
      "completions/mean_terminated_length": 662.6154174804688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.898536056280136,
      "epoch": 0.00625,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2388147115707397,
      "kl": 0.8716497123241425,
      "learning_rate": 1.920419923069143e-05,
      "loss": 0.0097,
      "num_tokens": 14661037.0,
      "reward": 0.3797062039375305,
      "reward_std": 0.8778296113014221,
      "rewards/rollout_reward_func/mean": 0.3797062039375305,
      "rewards/rollout_reward_func/std": 0.8664759993553162,
      "sampling/importance_sampling_ratio/max": 1.2678208351135254,
      "sampling/importance_sampling_ratio/mean": 1.0447735786437988,
      "sampling/importance_sampling_ratio/min": 0.7936925292015076,
      "sampling/sampling_logp_difference/max": 0.20116925239562988,
      "sampling/sampling_logp_difference/mean": 0.03212275356054306,
      "step": 625,
      "step_time": 10.859644024000772
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 887.0,
      "completions/max_terminated_length": 811.0,
      "completions/mean_length": 378.25,
      "completions/mean_terminated_length": 283.6153869628906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3258216995745897,
      "epoch": 0.00626,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2795703411102295,
      "kl": 0.9336279332637787,
      "learning_rate": 1.913937113287685e-05,
      "loss": 0.1162,
      "num_tokens": 14680860.0,
      "reward": 0.2987927496433258,
      "reward_std": 0.8794794082641602,
      "rewards/rollout_reward_func/mean": 0.2987927496433258,
      "rewards/rollout_reward_func/std": 0.8743470907211304,
      "sampling/importance_sampling_ratio/max": 1.227598786354065,
      "sampling/importance_sampling_ratio/mean": 0.9686786532402039,
      "sampling/importance_sampling_ratio/min": 0.5423810482025146,
      "sampling/sampling_logp_difference/max": 0.14370441436767578,
      "sampling/sampling_logp_difference/mean": 0.027979187667369843,
      "step": 626,
      "step_time": 8.55347711000013
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1578.0,
      "completions/max_terminated_length": 1578.0,
      "completions/mean_length": 494.6875,
      "completions/mean_terminated_length": 426.4285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.588543340563774,
      "epoch": 0.00627,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6414699554443359,
      "kl": 0.7764008436352015,
      "learning_rate": 1.9074650724947377e-05,
      "loss": 0.01,
      "num_tokens": 14703383.0,
      "reward": 0.6053858995437622,
      "reward_std": 0.6332288384437561,
      "rewards/rollout_reward_func/mean": 0.6053858995437622,
      "rewards/rollout_reward_func/std": 0.636553168296814,
      "sampling/importance_sampling_ratio/max": 1.2657897472381592,
      "sampling/importance_sampling_ratio/mean": 0.9780345559120178,
      "sampling/importance_sampling_ratio/min": 5.654217147821328e-06,
      "sampling/sampling_logp_difference/max": 7.087726593017578,
      "sampling/sampling_logp_difference/mean": 0.1877935528755188,
      "step": 627,
      "step_time": 10.114033426998503
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1733.0,
      "completions/max_terminated_length": 1733.0,
      "completions/mean_length": 564.25,
      "completions/mean_terminated_length": 512.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.33095370978117,
      "epoch": 0.00628,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1995278596878052,
      "kl": 1.340434055775404,
      "learning_rate": 1.9010038778264192e-05,
      "loss": -0.1299,
      "num_tokens": 14727703.0,
      "reward": 0.5384578704833984,
      "reward_std": 0.6614294052124023,
      "rewards/rollout_reward_func/mean": 0.5384578704833984,
      "rewards/rollout_reward_func/std": 0.6491928100585938,
      "sampling/importance_sampling_ratio/max": 1.2190344333648682,
      "sampling/importance_sampling_ratio/mean": 0.9796577095985413,
      "sampling/importance_sampling_ratio/min": 0.5257272720336914,
      "sampling/sampling_logp_difference/max": 0.2485109567642212,
      "sampling/sampling_logp_difference/mean": 0.035916589200496674,
      "step": 628,
      "step_time": 10.913784411999586
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2460.0,
      "completions/max_terminated_length": 2460.0,
      "completions/mean_length": 639.625,
      "completions/mean_terminated_length": 627.0000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6484442949295044,
      "epoch": 0.00629,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7125098705291748,
      "kl": 1.0779589097946882,
      "learning_rate": 1.89455360628958e-05,
      "loss": -0.0046,
      "num_tokens": 14752974.0,
      "reward": 0.420556902885437,
      "reward_std": 0.7534891366958618,
      "rewards/rollout_reward_func/mean": 0.420556902885437,
      "rewards/rollout_reward_func/std": 0.730453610420227,
      "sampling/importance_sampling_ratio/max": 1.302915334701538,
      "sampling/importance_sampling_ratio/mean": 1.0163111686706543,
      "sampling/importance_sampling_ratio/min": 0.7886053323745728,
      "sampling/sampling_logp_difference/max": 0.27883100509643555,
      "sampling/sampling_logp_difference/mean": 0.027383694425225258,
      "step": 629,
      "step_time": 12.20618453499901
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2689.0,
      "completions/max_terminated_length": 2689.0,
      "completions/mean_length": 728.125,
      "completions/mean_terminated_length": 718.2307739257812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8406692072749138,
      "epoch": 0.0063,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9126558303833008,
      "kl": 1.0161752626299858,
      "learning_rate": 1.8881143347608815e-05,
      "loss": -0.0436,
      "num_tokens": 14780251.0,
      "reward": 0.21589252352714539,
      "reward_std": 0.6462846994400024,
      "rewards/rollout_reward_func/mean": 0.21589252352714539,
      "rewards/rollout_reward_func/std": 0.6908215284347534,
      "sampling/importance_sampling_ratio/max": 1.2941497564315796,
      "sampling/importance_sampling_ratio/mean": 0.9465447664260864,
      "sampling/importance_sampling_ratio/min": 9.172463640932741e-22,
      "sampling/sampling_logp_difference/max": 20.115589141845703,
      "sampling/sampling_logp_difference/mean": 0.1973837912082672,
      "step": 630,
      "step_time": 13.826169745001607
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2731.0,
      "completions/max_terminated_length": 1746.0,
      "completions/mean_length": 727.1875,
      "completions/mean_terminated_length": 507.0769348144531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3867240212857723,
      "epoch": 0.00631,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3013274669647217,
      "kl": 0.5858388915657997,
      "learning_rate": 1.8816861399858856e-05,
      "loss": 0.1095,
      "num_tokens": 14806747.0,
      "reward": 0.601686954498291,
      "reward_std": 0.5260728597640991,
      "rewards/rollout_reward_func/mean": 0.601686954498291,
      "rewards/rollout_reward_func/std": 0.5323289036750793,
      "sampling/importance_sampling_ratio/max": 1.1936323642730713,
      "sampling/importance_sampling_ratio/mean": 0.8794849514961243,
      "sampling/importance_sampling_ratio/min": 4.4148612543537524e-10,
      "sampling/sampling_logp_difference/max": 9.190637588500977,
      "sampling/sampling_logp_difference/mean": 0.10790122300386429,
      "step": 631,
      "step_time": 14.214247155999146
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1701.0,
      "completions/max_terminated_length": 1701.0,
      "completions/mean_length": 600.5625,
      "completions/mean_terminated_length": 504.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3324820064008236,
      "epoch": 0.00632,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9582905173301697,
      "kl": 0.9242780730128288,
      "learning_rate": 1.8752690985781393e-05,
      "loss": 0.0338,
      "num_tokens": 14830689.0,
      "reward": 0.5524204969406128,
      "reward_std": 0.6409472227096558,
      "rewards/rollout_reward_func/mean": 0.5524204969406128,
      "rewards/rollout_reward_func/std": 0.6291042566299438,
      "sampling/importance_sampling_ratio/max": 1.337801218032837,
      "sampling/importance_sampling_ratio/mean": 1.0071183443069458,
      "sampling/importance_sampling_ratio/min": 0.4627330005168915,
      "sampling/sampling_logp_difference/max": 0.26346302032470703,
      "sampling/sampling_logp_difference/mean": 0.030971860513091087,
      "step": 632,
      "step_time": 11.129228657000567
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1770.0,
      "completions/max_terminated_length": 1770.0,
      "completions/mean_length": 781.5625,
      "completions/mean_terminated_length": 706.923095703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.877289455384016,
      "epoch": 0.00633,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2388124465942383,
      "kl": 1.176084402948618,
      "learning_rate": 1.868863287018255e-05,
      "loss": -0.2223,
      "num_tokens": 14858173.0,
      "reward": 0.42355966567993164,
      "reward_std": 0.755851149559021,
      "rewards/rollout_reward_func/mean": 0.42355966567993164,
      "rewards/rollout_reward_func/std": 0.7345213294029236,
      "sampling/importance_sampling_ratio/max": 1.3556761741638184,
      "sampling/importance_sampling_ratio/mean": 1.0354138612747192,
      "sampling/importance_sampling_ratio/min": 0.6750184893608093,
      "sampling/sampling_logp_difference/max": 0.2270495891571045,
      "sampling/sampling_logp_difference/mean": 0.03234861046075821,
      "step": 633,
      "step_time": 11.403465846000472
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1748.0,
      "completions/max_terminated_length": 1748.0,
      "completions/mean_length": 510.75,
      "completions/mean_terminated_length": 437.13336181640625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6751102432608604,
      "epoch": 0.00634,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8165377378463745,
      "kl": 1.5136263817548752,
      "learning_rate": 1.8624687816530077e-05,
      "loss": -0.0726,
      "num_tokens": 14880737.0,
      "reward": 0.32888200879096985,
      "reward_std": 0.6575926542282104,
      "rewards/rollout_reward_func/mean": 0.32888200879096985,
      "rewards/rollout_reward_func/std": 0.6504969596862793,
      "sampling/importance_sampling_ratio/max": 1.3697590827941895,
      "sampling/importance_sampling_ratio/mean": 0.9632593393325806,
      "sampling/importance_sampling_ratio/min": 5.962472811489594e-13,
      "sampling/sampling_logp_difference/max": 12.333945274353027,
      "sampling/sampling_logp_difference/mean": 0.16753119230270386,
      "step": 634,
      "step_time": 10.70241332400019
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 977.0,
      "completions/max_terminated_length": 889.0,
      "completions/mean_length": 390.0625,
      "completions/mean_terminated_length": 282.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6455042213201523,
      "epoch": 0.00635,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.9227047562599182,
      "kl": 0.9080795757472515,
      "learning_rate": 1.8560856586944193e-05,
      "loss": 0.0871,
      "num_tokens": 14900659.0,
      "reward": 0.8650977611541748,
      "reward_std": 0.2497905194759369,
      "rewards/rollout_reward_func/mean": 0.8650977611541748,
      "rewards/rollout_reward_func/std": 0.3686232268810272,
      "sampling/importance_sampling_ratio/max": 1.0788944959640503,
      "sampling/importance_sampling_ratio/mean": 0.9233052134513855,
      "sampling/importance_sampling_ratio/min": 1.6006969083548483e-09,
      "sampling/sampling_logp_difference/max": 9.343572616577148,
      "sampling/sampling_logp_difference/mean": 0.17189492285251617,
      "step": 635,
      "step_time": 8.37037705000057
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1014.0,
      "completions/max_terminated_length": 846.0,
      "completions/mean_length": 408.1875,
      "completions/mean_terminated_length": 347.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3181692967191339,
      "epoch": 0.00636,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9330150485038757,
      "kl": 1.3342108614742756,
      "learning_rate": 1.8497139942188516e-05,
      "loss": -0.0004,
      "num_tokens": 14921627.0,
      "reward": 0.6289923787117004,
      "reward_std": 0.7081799507141113,
      "rewards/rollout_reward_func/mean": 0.6289923787117004,
      "rewards/rollout_reward_func/std": 0.69873046875,
      "sampling/importance_sampling_ratio/max": 1.2598652839660645,
      "sampling/importance_sampling_ratio/mean": 0.8694257140159607,
      "sampling/importance_sampling_ratio/min": 5.984265410681824e-14,
      "sampling/sampling_logp_difference/max": 23.454160690307617,
      "sampling/sampling_logp_difference/mean": 0.29179808497428894,
      "step": 636,
      "step_time": 8.475848992001374
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1477.0,
      "completions/max_terminated_length": 1477.0,
      "completions/mean_length": 488.625,
      "completions/mean_terminated_length": 431.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3382335444912314,
      "epoch": 0.00637,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4880225658416748,
      "kl": 0.7427102997899055,
      "learning_rate": 1.8433538641661017e-05,
      "loss": 0.4008,
      "num_tokens": 14943712.0,
      "reward": 0.6074801087379456,
      "reward_std": 0.5671790838241577,
      "rewards/rollout_reward_func/mean": 0.6074801087379456,
      "rewards/rollout_reward_func/std": 0.6385104656219482,
      "sampling/importance_sampling_ratio/max": 1.271945834159851,
      "sampling/importance_sampling_ratio/mean": 1.0197434425354004,
      "sampling/importance_sampling_ratio/min": 0.7674380540847778,
      "sampling/sampling_logp_difference/max": 0.15520620346069336,
      "sampling/sampling_logp_difference/mean": 0.024981975555419922,
      "step": 637,
      "step_time": 10.061538544999166
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1809.0,
      "completions/max_terminated_length": 1809.0,
      "completions/mean_length": 516.75,
      "completions/mean_terminated_length": 470.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9426455795764923,
      "epoch": 0.00638,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9328568577766418,
      "kl": 1.4203741289675236,
      "learning_rate": 1.8370053443384946e-05,
      "loss": -0.0935,
      "num_tokens": 14966238.0,
      "reward": 0.49067386984825134,
      "reward_std": 0.7553443908691406,
      "rewards/rollout_reward_func/mean": 0.49067386984825134,
      "rewards/rollout_reward_func/std": 0.7299003005027771,
      "sampling/importance_sampling_ratio/max": 1.1908602714538574,
      "sampling/importance_sampling_ratio/mean": 0.9855950474739075,
      "sampling/importance_sampling_ratio/min": 0.7776402235031128,
      "sampling/sampling_logp_difference/max": 0.3148502707481384,
      "sampling/sampling_logp_difference/mean": 0.031090443953871727,
      "step": 638,
      "step_time": 10.906672310999056
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2710.0,
      "completions/max_terminated_length": 2710.0,
      "completions/mean_length": 813.9375,
      "completions/mean_terminated_length": 653.4166870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8249298194423318,
      "epoch": 0.00639,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2176610231399536,
      "kl": 0.48473675549030304,
      "learning_rate": 1.83066851039998e-05,
      "loss": 0.1661,
      "num_tokens": 14994018.0,
      "reward": 0.4694528579711914,
      "reward_std": 0.5681131482124329,
      "rewards/rollout_reward_func/mean": 0.4694528579711914,
      "rewards/rollout_reward_func/std": 0.5498501658439636,
      "sampling/importance_sampling_ratio/max": 1.344929814338684,
      "sampling/importance_sampling_ratio/mean": 0.9864699244499207,
      "sampling/importance_sampling_ratio/min": 0.5630996823310852,
      "sampling/sampling_logp_difference/max": 0.2261366844177246,
      "sampling/sampling_logp_difference/mean": 0.0358780175447464,
      "step": 639,
      "step_time": 13.868120129001
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 976.0,
      "completions/max_terminated_length": 942.0,
      "completions/mean_length": 396.5625,
      "completions/mean_terminated_length": 272.5833435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.468953083269298,
      "epoch": 0.0064,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2802150249481201,
      "kl": 0.6850388720631599,
      "learning_rate": 1.8243434378752306e-05,
      "loss": 0.1364,
      "num_tokens": 15014584.0,
      "reward": 0.7296956777572632,
      "reward_std": 0.500534176826477,
      "rewards/rollout_reward_func/mean": 0.7296956777572632,
      "rewards/rollout_reward_func/std": 0.4835648834705353,
      "sampling/importance_sampling_ratio/max": 1.3576226234436035,
      "sampling/importance_sampling_ratio/mean": 0.930569052696228,
      "sampling/importance_sampling_ratio/min": 6.396291959803575e-19,
      "sampling/sampling_logp_difference/max": 12.82080078125,
      "sampling/sampling_logp_difference/mean": 0.19144970178604126,
      "step": 640,
      "step_time": 8.949307147999207
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 894.0,
      "completions/max_terminated_length": 873.0,
      "completions/mean_length": 386.75,
      "completions/mean_terminated_length": 352.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6009106002748013,
      "epoch": 0.00641,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.182218074798584,
      "kl": 0.571442786604166,
      "learning_rate": 1.818030202148745e-05,
      "loss": 0.1804,
      "num_tokens": 15035097.0,
      "reward": 0.6062915325164795,
      "reward_std": 0.6541149616241455,
      "rewards/rollout_reward_func/mean": 0.6062915325164795,
      "rewards/rollout_reward_func/std": 0.6392048001289368,
      "sampling/importance_sampling_ratio/max": 1.1053448915481567,
      "sampling/importance_sampling_ratio/mean": 0.9681377410888672,
      "sampling/importance_sampling_ratio/min": 0.5165489912033081,
      "sampling/sampling_logp_difference/max": 0.2874774932861328,
      "sampling/sampling_logp_difference/mean": 0.03245335817337036,
      "step": 641,
      "step_time": 8.00457690300118
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2723.0,
      "completions/max_terminated_length": 834.0,
      "completions/mean_length": 493.1875,
      "completions/mean_terminated_length": 344.5333557128906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3221938386559486,
      "epoch": 0.00642,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9894315600395203,
      "kl": 0.8151009455323219,
      "learning_rate": 1.811728878463942e-05,
      "loss": -0.0083,
      "num_tokens": 15057493.0,
      "reward": 0.6695370674133301,
      "reward_std": 0.6296558380126953,
      "rewards/rollout_reward_func/mean": 0.6695370674133301,
      "rewards/rollout_reward_func/std": 0.6220418214797974,
      "sampling/importance_sampling_ratio/max": 1.3933353424072266,
      "sampling/importance_sampling_ratio/mean": 1.0326049327850342,
      "sampling/importance_sampling_ratio/min": 0.8340443968772888,
      "sampling/sampling_logp_difference/max": 0.19800305366516113,
      "sampling/sampling_logp_difference/mean": 0.0220269113779068,
      "step": 642,
      "step_time": 12.515109395001673
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1752.0,
      "completions/max_terminated_length": 915.0,
      "completions/mean_length": 537.25,
      "completions/mean_terminated_length": 349.3333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4723791852593422,
      "epoch": 0.00643,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.45256468653678894,
      "kl": 0.6516029350459576,
      "learning_rate": 1.8054395419222726e-05,
      "loss": -0.0409,
      "num_tokens": 15080991.0,
      "reward": 0.5355192422866821,
      "reward_std": 0.5521217584609985,
      "rewards/rollout_reward_func/mean": 0.5355192422866821,
      "rewards/rollout_reward_func/std": 0.6514862775802612,
      "sampling/importance_sampling_ratio/max": 1.347635269165039,
      "sampling/importance_sampling_ratio/mean": 0.9565499424934387,
      "sampling/importance_sampling_ratio/min": 1.7636717874763584e-23,
      "sampling/sampling_logp_difference/max": 17.375648498535156,
      "sampling/sampling_logp_difference/mean": 0.2488829344511032,
      "step": 643,
      "step_time": 10.846195179999995
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2477.0,
      "completions/max_terminated_length": 2477.0,
      "completions/mean_length": 681.8125,
      "completions/mean_terminated_length": 666.5333862304688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1758274855092168,
      "epoch": 0.00644,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.237399101257324,
      "kl": 0.49270761851221323,
      "learning_rate": 1.799162267482319e-05,
      "loss": 0.3052,
      "num_tokens": 15106960.0,
      "reward": 0.5009967088699341,
      "reward_std": 0.8267679214477539,
      "rewards/rollout_reward_func/mean": 0.5009967088699341,
      "rewards/rollout_reward_func/std": 0.802628755569458,
      "sampling/importance_sampling_ratio/max": 2.345956563949585,
      "sampling/importance_sampling_ratio/mean": 1.0944342613220215,
      "sampling/importance_sampling_ratio/min": 0.8346579074859619,
      "sampling/sampling_logp_difference/max": 0.19153404235839844,
      "sampling/sampling_logp_difference/mean": 0.029576923698186874,
      "step": 644,
      "step_time": 12.372069930998805
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1531.0,
      "completions/max_terminated_length": 1531.0,
      "completions/mean_length": 446.3125,
      "completions/mean_terminated_length": 413.5000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3111185990273952,
      "epoch": 0.00645,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3980444669723511,
      "kl": 1.0804447643458843,
      "learning_rate": 1.792897129958902e-05,
      "loss": -0.1047,
      "num_tokens": 15128412.0,
      "reward": 0.8604916334152222,
      "reward_std": 0.39458930492401123,
      "rewards/rollout_reward_func/mean": 0.8604916334152222,
      "rewards/rollout_reward_func/std": 0.3812122046947479,
      "sampling/importance_sampling_ratio/max": 1.5917885303497314,
      "sampling/importance_sampling_ratio/mean": 1.027782678604126,
      "sampling/importance_sampling_ratio/min": 0.8463866710662842,
      "sampling/sampling_logp_difference/max": 0.18087005615234375,
      "sampling/sampling_logp_difference/mean": 0.027617864310741425,
      "step": 645,
      "step_time": 10.035736097000154
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1776.0,
      "completions/max_terminated_length": 1776.0,
      "completions/mean_length": 605.75,
      "completions/mean_terminated_length": 575.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.243613475933671,
      "epoch": 0.00646,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0736457109451294,
      "kl": 0.9466625563800335,
      "learning_rate": 1.7866442040221912e-05,
      "loss": -0.0151,
      "num_tokens": 15153144.0,
      "reward": 0.3623318672180176,
      "reward_std": 0.672317385673523,
      "rewards/rollout_reward_func/mean": 0.3623318672180176,
      "rewards/rollout_reward_func/std": 0.7102683782577515,
      "sampling/importance_sampling_ratio/max": 1.293568730354309,
      "sampling/importance_sampling_ratio/mean": 0.9691824913024902,
      "sampling/importance_sampling_ratio/min": 0.6254640817642212,
      "sampling/sampling_logp_difference/max": 0.22707319259643555,
      "sampling/sampling_logp_difference/mean": 0.026596536859869957,
      "step": 646,
      "step_time": 11.032193248001022
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1577.0,
      "completions/max_terminated_length": 1577.0,
      "completions/mean_length": 591.125,
      "completions/mean_terminated_length": 492.8333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6660608239471912,
      "epoch": 0.00647,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.760917067527771,
      "kl": 1.1887229643762112,
      "learning_rate": 1.780403564196816e-05,
      "loss": 0.0848,
      "num_tokens": 15177364.0,
      "reward": 0.7291710376739502,
      "reward_std": 0.4716780185699463,
      "rewards/rollout_reward_func/mean": 0.7291710376739502,
      "rewards/rollout_reward_func/std": 0.484624445438385,
      "sampling/importance_sampling_ratio/max": 1.4940012693405151,
      "sampling/importance_sampling_ratio/mean": 0.9518143534660339,
      "sampling/importance_sampling_ratio/min": 2.3234576463496476e-21,
      "sampling/sampling_logp_difference/max": 23.556686401367188,
      "sampling/sampling_logp_difference/mean": 0.3079221844673157,
      "step": 647,
      "step_time": 10.431407945998217
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1644.0,
      "completions/max_terminated_length": 1644.0,
      "completions/mean_length": 530.375,
      "completions/mean_terminated_length": 530.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9330603256821632,
      "epoch": 0.00648,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.0940791517496109,
      "kl": 1.0482177347876132,
      "learning_rate": 1.7741752848609718e-05,
      "loss": 0.0107,
      "num_tokens": 15200495.0,
      "reward": 0.681278645992279,
      "reward_std": 0.3669034242630005,
      "rewards/rollout_reward_func/mean": 0.681278645992279,
      "rewards/rollout_reward_func/std": 0.5997021794319153,
      "sampling/importance_sampling_ratio/max": 1.0639244318008423,
      "sampling/importance_sampling_ratio/mean": 0.9228510856628418,
      "sampling/importance_sampling_ratio/min": 2.6005251771432647e-11,
      "sampling/sampling_logp_difference/max": 23.003292083740234,
      "sampling/sampling_logp_difference/mean": 0.23917256295681,
      "step": 648,
      "step_time": 9.778853133999291
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1713.0,
      "completions/max_terminated_length": 1713.0,
      "completions/mean_length": 493.6875,
      "completions/mean_terminated_length": 493.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.042997770011425,
      "epoch": 0.00649,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.316899836063385,
      "kl": 1.2028714939951897,
      "learning_rate": 1.7679594402455392e-05,
      "loss": 0.0088,
      "num_tokens": 15223323.0,
      "reward": 0.39591777324676514,
      "reward_std": 0.6685217618942261,
      "rewards/rollout_reward_func/mean": 0.39591777324676514,
      "rewards/rollout_reward_func/std": 0.6585574746131897,
      "sampling/importance_sampling_ratio/max": 1.207318902015686,
      "sampling/importance_sampling_ratio/mean": 0.9203846454620361,
      "sampling/importance_sampling_ratio/min": 1.7236830851530272e-09,
      "sampling/sampling_logp_difference/max": 19.99073600769043,
      "sampling/sampling_logp_difference/mean": 0.18987402319908142,
      "step": 649,
      "step_time": 9.87423575000139
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1736.0,
      "completions/max_terminated_length": 1736.0,
      "completions/mean_length": 566.8125,
      "completions/mean_terminated_length": 566.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0803861394524574,
      "epoch": 0.0065,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5516784191131592,
      "kl": 0.6609821673482656,
      "learning_rate": 1.7617561044331985e-05,
      "loss": 0.0013,
      "num_tokens": 15247676.0,
      "reward": 0.4228076934814453,
      "reward_std": 0.7054752111434937,
      "rewards/rollout_reward_func/mean": 0.4228076934814453,
      "rewards/rollout_reward_func/std": 0.722152054309845,
      "sampling/importance_sampling_ratio/max": 1.2373120784759521,
      "sampling/importance_sampling_ratio/mean": 1.0075448751449585,
      "sampling/importance_sampling_ratio/min": 0.7912281155586243,
      "sampling/sampling_logp_difference/max": 0.1877892017364502,
      "sampling/sampling_logp_difference/mean": 0.023813702166080475,
      "step": 650,
      "step_time": 9.957399111000996
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1786.0,
      "completions/max_terminated_length": 919.0,
      "completions/mean_length": 581.5,
      "completions/mean_terminated_length": 402.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7696244539692998,
      "epoch": 0.00651,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3261908292770386,
      "kl": 0.6281221956014633,
      "learning_rate": 1.7555653513575422e-05,
      "loss": 0.2841,
      "num_tokens": 15271328.0,
      "reward": 0.46696579456329346,
      "reward_std": 0.5521270036697388,
      "rewards/rollout_reward_func/mean": 0.46696579456329346,
      "rewards/rollout_reward_func/std": 0.5507985353469849,
      "sampling/importance_sampling_ratio/max": 1.4344110488891602,
      "sampling/importance_sampling_ratio/mean": 1.0618937015533447,
      "sampling/importance_sampling_ratio/min": 0.7859833836555481,
      "sampling/sampling_logp_difference/max": 0.21578443050384521,
      "sampling/sampling_logp_difference/mean": 0.027821065858006477,
      "step": 651,
      "step_time": 11.104289531999711
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2535.0,
      "completions/max_terminated_length": 2535.0,
      "completions/mean_length": 600.4375,
      "completions/mean_terminated_length": 636.2000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7256199717521667,
      "epoch": 0.00652,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7260348200798035,
      "kl": 1.1187658477574587,
      "learning_rate": 1.7493872548022005e-05,
      "loss": -0.0118,
      "num_tokens": 15296229.0,
      "reward": 0.6596840023994446,
      "reward_std": 0.5340322256088257,
      "rewards/rollout_reward_func/mean": 0.6596840023994446,
      "rewards/rollout_reward_func/std": 0.5215359926223755,
      "sampling/importance_sampling_ratio/max": 1.1135330200195312,
      "sampling/importance_sampling_ratio/mean": 0.8933799266815186,
      "sampling/importance_sampling_ratio/min": 2.785258808195509e-24,
      "sampling/sampling_logp_difference/max": 13.496781349182129,
      "sampling/sampling_logp_difference/mean": 0.3660738170146942,
      "step": 652,
      "step_time": 13.036609676998523
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1720.0,
      "completions/max_terminated_length": 1720.0,
      "completions/mean_length": 621.5,
      "completions/mean_terminated_length": 580.6428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5559286698698997,
      "epoch": 0.00653,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0836131572723389,
      "kl": 1.3382497169077396,
      "learning_rate": 1.7432218883999567e-05,
      "loss": 0.0079,
      "num_tokens": 15320961.0,
      "reward": 0.4717545509338379,
      "reward_std": 0.6558353900909424,
      "rewards/rollout_reward_func/mean": 0.4717545509338379,
      "rewards/rollout_reward_func/std": 0.6524227857589722,
      "sampling/importance_sampling_ratio/max": 1.360147476196289,
      "sampling/importance_sampling_ratio/mean": 0.9647229313850403,
      "sampling/importance_sampling_ratio/min": 1.0784662052287786e-09,
      "sampling/sampling_logp_difference/max": 20.519474029541016,
      "sampling/sampling_logp_difference/mean": 0.1228787899017334,
      "step": 653,
      "step_time": 10.740224416998899
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1578.0,
      "completions/max_terminated_length": 1578.0,
      "completions/mean_length": 589.0625,
      "completions/mean_terminated_length": 519.7692260742188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4965738877654076,
      "epoch": 0.00654,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3046633005142212,
      "kl": 0.41652299743145704,
      "learning_rate": 1.737069325631873e-05,
      "loss": 0.0417,
      "num_tokens": 15345209.0,
      "reward": 0.35151684284210205,
      "reward_std": 0.6915802955627441,
      "rewards/rollout_reward_func/mean": 0.35151684284210205,
      "rewards/rollout_reward_func/std": 0.7328969240188599,
      "sampling/importance_sampling_ratio/max": 1.686136245727539,
      "sampling/importance_sampling_ratio/mean": 1.0007990598678589,
      "sampling/importance_sampling_ratio/min": 8.339400836554489e-10,
      "sampling/sampling_logp_difference/max": 20.1636962890625,
      "sampling/sampling_logp_difference/mean": 0.11904493719339371,
      "step": 654,
      "step_time": 10.558449904001463
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2668.0,
      "completions/max_terminated_length": 799.0,
      "completions/mean_length": 580.875,
      "completions/mean_terminated_length": 312.15386962890625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4965924248099327,
      "epoch": 0.00655,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5404235124588013,
      "kl": 0.47479764372110367,
      "learning_rate": 1.7309296398264127e-05,
      "loss": 0.0549,
      "num_tokens": 15368792.0,
      "reward": 0.5520155429840088,
      "reward_std": 0.7197190523147583,
      "rewards/rollout_reward_func/mean": 0.5520155429840088,
      "rewards/rollout_reward_func/std": 0.7281516790390015,
      "sampling/importance_sampling_ratio/max": 1.2313690185546875,
      "sampling/importance_sampling_ratio/mean": 0.8905102014541626,
      "sampling/importance_sampling_ratio/min": 1.7679132761688407e-14,
      "sampling/sampling_logp_difference/max": 19.290924072265625,
      "sampling/sampling_logp_difference/mean": 0.43282732367515564,
      "step": 655,
      "step_time": 13.171986975002255
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1699.0,
      "completions/max_terminated_length": 790.0,
      "completions/mean_length": 570.5,
      "completions/mean_terminated_length": 417.0833435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6918750256299973,
      "epoch": 0.00656,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1169461011886597,
      "kl": 0.6081432849168777,
      "learning_rate": 1.7248029041585656e-05,
      "loss": 0.0819,
      "num_tokens": 15393106.0,
      "reward": 0.5451374053955078,
      "reward_std": 0.6537582278251648,
      "rewards/rollout_reward_func/mean": 0.5451374053955078,
      "rewards/rollout_reward_func/std": 0.6436811089515686,
      "sampling/importance_sampling_ratio/max": 1.2586166858673096,
      "sampling/importance_sampling_ratio/mean": 0.9870303273200989,
      "sampling/importance_sampling_ratio/min": 0.6317666172981262,
      "sampling/sampling_logp_difference/max": 0.20720505714416504,
      "sampling/sampling_logp_difference/mean": 0.02929820492863655,
      "step": 656,
      "step_time": 11.121083986001395
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1815.0,
      "completions/max_terminated_length": 1815.0,
      "completions/mean_length": 653.8125,
      "completions/mean_terminated_length": 630.1428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.462371852248907,
      "epoch": 0.00657,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8570422530174255,
      "kl": 0.7137961611151695,
      "learning_rate": 1.718689191648978e-05,
      "loss": 0.0477,
      "num_tokens": 15418247.0,
      "reward": 0.47343915700912476,
      "reward_std": 0.5969130992889404,
      "rewards/rollout_reward_func/mean": 0.47343915700912476,
      "rewards/rollout_reward_func/std": 0.6469634175300598,
      "sampling/importance_sampling_ratio/max": 1.2186063528060913,
      "sampling/importance_sampling_ratio/mean": 0.9903364181518555,
      "sampling/importance_sampling_ratio/min": 0.3699502646923065,
      "sampling/sampling_logp_difference/max": 0.3190563917160034,
      "sampling/sampling_logp_difference/mean": 0.027124766260385513,
      "step": 657,
      "step_time": 10.795293269999092
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2603.0,
      "completions/max_terminated_length": 2603.0,
      "completions/mean_length": 572.1875,
      "completions/mean_terminated_length": 553.7142944335938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.334397442638874,
      "epoch": 0.00658,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4098859429359436,
      "kl": 0.6660514902323484,
      "learning_rate": 1.712588575163083e-05,
      "loss": -0.0947,
      "num_tokens": 15442525.0,
      "reward": 0.44305360317230225,
      "reward_std": 0.7159780263900757,
      "rewards/rollout_reward_func/mean": 0.44305360317230225,
      "rewards/rollout_reward_func/std": 0.8759298920631409,
      "sampling/importance_sampling_ratio/max": 1.1292169094085693,
      "sampling/importance_sampling_ratio/mean": 0.9515812993049622,
      "sampling/importance_sampling_ratio/min": 0.5170180797576904,
      "sampling/sampling_logp_difference/max": 0.18513202667236328,
      "sampling/sampling_logp_difference/mean": 0.025266334414482117,
      "step": 658,
      "step_time": 13.207738357999915
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1705.0,
      "completions/max_terminated_length": 1705.0,
      "completions/mean_length": 839.75,
      "completions/mean_terminated_length": 792.3846435546875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5324356406927109,
      "epoch": 0.00659,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0523827075958252,
      "kl": 1.1903462707996368,
      "learning_rate": 1.706501127410227e-05,
      "loss": -0.0787,
      "num_tokens": 15471322.0,
      "reward": 0.31659913063049316,
      "reward_std": 0.7752031683921814,
      "rewards/rollout_reward_func/mean": 0.31659913063049316,
      "rewards/rollout_reward_func/std": 0.7756358981132507,
      "sampling/importance_sampling_ratio/max": 1.1009678840637207,
      "sampling/importance_sampling_ratio/mean": 0.9359540939331055,
      "sampling/importance_sampling_ratio/min": 0.6603038311004639,
      "sampling/sampling_logp_difference/max": 0.43077611923217773,
      "sampling/sampling_logp_difference/mean": 0.03178316727280617,
      "step": 659,
      "step_time": 11.451160892999724
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2592.0,
      "completions/max_terminated_length": 2592.0,
      "completions/mean_length": 511.1875,
      "completions/mean_terminated_length": 500.66668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1271697171032429,
      "epoch": 0.0066,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6926183700561523,
      "kl": 0.5963338315486908,
      "learning_rate": 1.7004269209428104e-05,
      "loss": 0.0687,
      "num_tokens": 15493609.0,
      "reward": 0.42815014719963074,
      "reward_std": 0.7267354726791382,
      "rewards/rollout_reward_func/mean": 0.42815014719963074,
      "rewards/rollout_reward_func/std": 0.7192981243133545,
      "sampling/importance_sampling_ratio/max": 1.2506152391433716,
      "sampling/importance_sampling_ratio/mean": 1.0437874794006348,
      "sampling/importance_sampling_ratio/min": 0.9934878945350647,
      "sampling/sampling_logp_difference/max": 0.20145010948181152,
      "sampling/sampling_logp_difference/mean": 0.02311704307794571,
      "step": 660,
      "step_time": 12.327202058999319
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1616.0,
      "completions/max_terminated_length": 1616.0,
      "completions/mean_length": 335.5,
      "completions/mean_terminated_length": 335.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7692693583667278,
      "epoch": 0.00661,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5597454309463501,
      "kl": 0.6700050961226225,
      "learning_rate": 1.6943660281554178e-05,
      "loss": -0.0668,
      "num_tokens": 15512810.0,
      "reward": 0.5227375626564026,
      "reward_std": 0.573967695236206,
      "rewards/rollout_reward_func/mean": 0.5227375626564026,
      "rewards/rollout_reward_func/std": 0.5590690970420837,
      "sampling/importance_sampling_ratio/max": 1.1830486059188843,
      "sampling/importance_sampling_ratio/mean": 1.0200307369232178,
      "sampling/importance_sampling_ratio/min": 0.9713947772979736,
      "sampling/sampling_logp_difference/max": 0.11959195137023926,
      "sampling/sampling_logp_difference/mean": 0.015573815442621708,
      "step": 661,
      "step_time": 9.378545321001184
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2544.0,
      "completions/max_terminated_length": 2544.0,
      "completions/mean_length": 922.625,
      "completions/mean_terminated_length": 871.6428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5695377215743065,
      "epoch": 0.00662,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4582250118255615,
      "kl": 1.6935585662722588,
      "learning_rate": 1.6883185212839546e-05,
      "loss": 0.1736,
      "num_tokens": 15542989.0,
      "reward": 0.6802067756652832,
      "reward_std": 0.6095561385154724,
      "rewards/rollout_reward_func/mean": 0.6802067756652832,
      "rewards/rollout_reward_func/std": 0.6021771430969238,
      "sampling/importance_sampling_ratio/max": 1.2328811883926392,
      "sampling/importance_sampling_ratio/mean": 0.9073071479797363,
      "sampling/importance_sampling_ratio/min": 2.7145374836834435e-10,
      "sampling/sampling_logp_difference/max": 13.865955352783203,
      "sampling/sampling_logp_difference/mean": 0.2071029394865036,
      "step": 662,
      "step_time": 13.241113868000866
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 897.0,
      "completions/max_terminated_length": 897.0,
      "completions/mean_length": 313.0625,
      "completions/mean_terminated_length": 291.13336181640625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3131508640944958,
      "epoch": 0.00663,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.44031691551208496,
      "kl": 0.576003497466445,
      "learning_rate": 1.6822844724047912e-05,
      "loss": -0.017,
      "num_tokens": 15562322.0,
      "reward": 0.42543891072273254,
      "reward_std": 0.6560266613960266,
      "rewards/rollout_reward_func/mean": 0.42543891072273254,
      "rewards/rollout_reward_func/std": 0.8176174759864807,
      "sampling/importance_sampling_ratio/max": 1.1961946487426758,
      "sampling/importance_sampling_ratio/mean": 0.947827160358429,
      "sampling/importance_sampling_ratio/min": 1.8351147446082905e-06,
      "sampling/sampling_logp_difference/max": 14.83138656616211,
      "sampling/sampling_logp_difference/mean": 0.20160749554634094,
      "step": 663,
      "step_time": 7.435833260999971
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1929.0,
      "completions/max_terminated_length": 1855.0,
      "completions/mean_length": 442.875,
      "completions/mean_terminated_length": 343.8000183105469,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0513470321893692,
      "epoch": 0.00664,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6992607116699219,
      "kl": 0.9015768356621265,
      "learning_rate": 1.6762639534339e-05,
      "loss": 0.0642,
      "num_tokens": 15583129.0,
      "reward": 0.001103729009628296,
      "reward_std": 0.8639423847198486,
      "rewards/rollout_reward_func/mean": 0.001103729009628296,
      "rewards/rollout_reward_func/std": 0.8727154731750488,
      "sampling/importance_sampling_ratio/max": 1.1786980628967285,
      "sampling/importance_sampling_ratio/mean": 1.020542860031128,
      "sampling/importance_sampling_ratio/min": 0.8640682101249695,
      "sampling/sampling_logp_difference/max": 0.318561315536499,
      "sampling/sampling_logp_difference/mean": 0.029208656400442123,
      "step": 664,
      "step_time": 10.58670114800043
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1576.0,
      "completions/max_terminated_length": 1576.0,
      "completions/mean_length": 689.9375,
      "completions/mean_terminated_length": 675.0000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.939259096980095,
      "epoch": 0.00665,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0857726335525513,
      "kl": 1.0519740507006645,
      "learning_rate": 1.6702570361259965e-05,
      "loss": 0.029,
      "num_tokens": 15609185.0,
      "reward": 0.40714168548583984,
      "reward_std": 0.634304404258728,
      "rewards/rollout_reward_func/mean": 0.40714168548583984,
      "rewards/rollout_reward_func/std": 0.6464594602584839,
      "sampling/importance_sampling_ratio/max": 1.2589426040649414,
      "sampling/importance_sampling_ratio/mean": 0.9829325675964355,
      "sampling/importance_sampling_ratio/min": 0.7103419303894043,
      "sampling/sampling_logp_difference/max": 0.24526596069335938,
      "sampling/sampling_logp_difference/mean": 0.03218601644039154,
      "step": 665,
      "step_time": 10.71285971999987
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2583.0,
      "completions/max_terminated_length": 1766.0,
      "completions/mean_length": 688.3125,
      "completions/mean_terminated_length": 556.6428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.585175983607769,
      "epoch": 0.00666,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.916762113571167,
      "kl": 1.1101038679480553,
      "learning_rate": 1.6642637920736896e-05,
      "loss": -0.0972,
      "num_tokens": 15634942.0,
      "reward": 0.43690112233161926,
      "reward_std": 0.8154484629631042,
      "rewards/rollout_reward_func/mean": 0.43690112233161926,
      "rewards/rollout_reward_func/std": 0.7954718470573425,
      "sampling/importance_sampling_ratio/max": 1.2386853694915771,
      "sampling/importance_sampling_ratio/mean": 0.9224010109901428,
      "sampling/importance_sampling_ratio/min": 1.0121575799715288e-09,
      "sampling/sampling_logp_difference/max": 18.8210391998291,
      "sampling/sampling_logp_difference/mean": 0.14950013160705566,
      "step": 666,
      "step_time": 13.214295259999744
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1684.0,
      "completions/max_terminated_length": 920.0,
      "completions/mean_length": 510.75,
      "completions/mean_terminated_length": 329.41668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6793602854013443,
      "epoch": 0.00667,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9874573945999146,
      "kl": 0.6432331129908562,
      "learning_rate": 1.6582842927066253e-05,
      "loss": -0.0666,
      "num_tokens": 15658235.0,
      "reward": 0.4370177388191223,
      "reward_std": 0.8837131261825562,
      "rewards/rollout_reward_func/mean": 0.4370177388191223,
      "rewards/rollout_reward_func/std": 0.8867343664169312,
      "sampling/importance_sampling_ratio/max": 1.4515140056610107,
      "sampling/importance_sampling_ratio/mean": 0.8402682542800903,
      "sampling/importance_sampling_ratio/min": 7.560903019602767e-16,
      "sampling/sampling_logp_difference/max": 22.342849731445312,
      "sampling/sampling_logp_difference/mean": 0.17223362624645233,
      "step": 667,
      "step_time": 11.351292987999841
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1878.0,
      "completions/max_terminated_length": 1564.0,
      "completions/mean_length": 425.5625,
      "completions/mean_terminated_length": 302.5714416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5702407695353031,
      "epoch": 0.00668,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7659485936164856,
      "kl": 0.9683316387236118,
      "learning_rate": 1.6523186092906327e-05,
      "loss": -0.1582,
      "num_tokens": 15678900.0,
      "reward": 0.3593719005584717,
      "reward_std": 0.8224848508834839,
      "rewards/rollout_reward_func/mean": 0.3593719005584717,
      "rewards/rollout_reward_func/std": 0.8047971129417419,
      "sampling/importance_sampling_ratio/max": 1.2350655794143677,
      "sampling/importance_sampling_ratio/mean": 0.8963993787765503,
      "sampling/importance_sampling_ratio/min": 8.998472382257291e-17,
      "sampling/sampling_logp_difference/max": 15.818962097167969,
      "sampling/sampling_logp_difference/mean": 0.27989521622657776,
      "step": 668,
      "step_time": 10.938339600999825
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1674.0,
      "completions/max_terminated_length": 1615.0,
      "completions/mean_length": 677.5,
      "completions/mean_terminated_length": 569.7692260742188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7642676532268524,
      "epoch": 0.00669,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2160706520080566,
      "kl": 1.2913380078971386,
      "learning_rate": 1.6463668129268807e-05,
      "loss": 0.07,
      "num_tokens": 15704776.0,
      "reward": 0.6082610487937927,
      "reward_std": 0.6298823356628418,
      "rewards/rollout_reward_func/mean": 0.6082610487937927,
      "rewards/rollout_reward_func/std": 0.6288493871688843,
      "sampling/importance_sampling_ratio/max": 1.3978314399719238,
      "sampling/importance_sampling_ratio/mean": 0.8576462864875793,
      "sampling/importance_sampling_ratio/min": 7.568082827091871e-10,
      "sampling/sampling_logp_difference/max": 19.50074577331543,
      "sampling/sampling_logp_difference/mean": 0.09601635485887527,
      "step": 669,
      "step_time": 11.428074443000696
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1795.0,
      "completions/max_terminated_length": 867.0,
      "completions/mean_length": 309.5625,
      "completions/mean_terminated_length": 221.00001525878906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4065847676247358,
      "epoch": 0.0067,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4914484024047852,
      "kl": 1.1707097701728344,
      "learning_rate": 1.6404289745510263e-05,
      "loss": 0.2675,
      "num_tokens": 15722864.0,
      "reward": 0.6182333827018738,
      "reward_std": 0.7428498268127441,
      "rewards/rollout_reward_func/mean": 0.6182333827018738,
      "rewards/rollout_reward_func/std": 0.7177757620811462,
      "sampling/importance_sampling_ratio/max": 1.1595525741577148,
      "sampling/importance_sampling_ratio/mean": 0.9019491672515869,
      "sampling/importance_sampling_ratio/min": 4.401700307547643e-15,
      "sampling/sampling_logp_difference/max": 7.347288131713867,
      "sampling/sampling_logp_difference/mean": 0.23543883860111237,
      "step": 670,
      "step_time": 10.280208861000574
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 981.0,
      "completions/max_terminated_length": 727.0,
      "completions/mean_length": 415.1875,
      "completions/mean_terminated_length": 279.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8199209347367287,
      "epoch": 0.00671,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.52837872505188,
      "kl": 1.2563159205019474,
      "learning_rate": 1.634505164932368e-05,
      "loss": 0.4061,
      "num_tokens": 15744044.0,
      "reward": 0.6744654178619385,
      "reward_std": 0.6187928318977356,
      "rewards/rollout_reward_func/mean": 0.6744654178619385,
      "rewards/rollout_reward_func/std": 0.6101317405700684,
      "sampling/importance_sampling_ratio/max": 1.9446550607681274,
      "sampling/importance_sampling_ratio/mean": 1.10215425491333,
      "sampling/importance_sampling_ratio/min": 0.7511518001556396,
      "sampling/sampling_logp_difference/max": 0.32787132263183594,
      "sampling/sampling_logp_difference/mean": 0.03412650153040886,
      "step": 671,
      "step_time": 9.021345797000322
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1754.0,
      "completions/max_terminated_length": 1754.0,
      "completions/mean_length": 524.625,
      "completions/mean_terminated_length": 555.3333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6865895241498947,
      "epoch": 0.00672,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8963785767555237,
      "kl": 0.7030502893030643,
      "learning_rate": 1.6285954546730074e-05,
      "loss": -0.024,
      "num_tokens": 15767285.0,
      "reward": 0.3871366083621979,
      "reward_std": 0.881628692150116,
      "rewards/rollout_reward_func/mean": 0.3871366083621979,
      "rewards/rollout_reward_func/std": 0.8619232177734375,
      "sampling/importance_sampling_ratio/max": 1.6081324815750122,
      "sampling/importance_sampling_ratio/mean": 0.9430299997329712,
      "sampling/importance_sampling_ratio/min": 9.642727154951514e-13,
      "sampling/sampling_logp_difference/max": 7.14216423034668,
      "sampling/sampling_logp_difference/mean": 0.202286958694458,
      "step": 672,
      "step_time": 10.5048563570017
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1670.0,
      "completions/max_terminated_length": 937.0,
      "completions/mean_length": 569.9375,
      "completions/mean_terminated_length": 382.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6532134339213371,
      "epoch": 0.00673,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6800742745399475,
      "kl": 0.9451617114245892,
      "learning_rate": 1.6226999142070047e-05,
      "loss": 0.0519,
      "num_tokens": 15791534.0,
      "reward": 0.36374884843826294,
      "reward_std": 0.7846888303756714,
      "rewards/rollout_reward_func/mean": 0.36374884843826294,
      "rewards/rollout_reward_func/std": 0.8058448433876038,
      "sampling/importance_sampling_ratio/max": 1.0170291662216187,
      "sampling/importance_sampling_ratio/mean": 0.8561265468597412,
      "sampling/importance_sampling_ratio/min": 3.0544957674294437e-09,
      "sampling/sampling_logp_difference/max": 19.491958618164062,
      "sampling/sampling_logp_difference/mean": 0.10280874371528625,
      "step": 673,
      "step_time": 10.993285310000829
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1690.0,
      "completions/max_terminated_length": 1690.0,
      "completions/mean_length": 542.75,
      "completions/mean_terminated_length": 481.0769348144531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9372618570923805,
      "epoch": 0.00674,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4376753568649292,
      "kl": 0.7701126150786877,
      "learning_rate": 1.6168186137995366e-05,
      "loss": 0.1494,
      "num_tokens": 15815187.0,
      "reward": 0.2887886166572571,
      "reward_std": 0.549822986125946,
      "rewards/rollout_reward_func/mean": 0.2887886166572571,
      "rewards/rollout_reward_func/std": 0.7053626775741577,
      "sampling/importance_sampling_ratio/max": 1.1803287267684937,
      "sampling/importance_sampling_ratio/mean": 0.9168508052825928,
      "sampling/importance_sampling_ratio/min": 1.015657891123567e-09,
      "sampling/sampling_logp_difference/max": 21.092975616455078,
      "sampling/sampling_logp_difference/mean": 0.13277138769626617,
      "step": 674,
      "step_time": 10.777480022001328
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1825.0,
      "completions/max_terminated_length": 1825.0,
      "completions/mean_length": 413.5625,
      "completions/mean_terminated_length": 413.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0625906065106392,
      "epoch": 0.00675,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8254585862159729,
      "kl": 1.4977070279419422,
      "learning_rate": 1.6109516235460644e-05,
      "loss": -0.1056,
      "num_tokens": 15836313.0,
      "reward": 0.740588366985321,
      "reward_std": 0.5995748043060303,
      "rewards/rollout_reward_func/mean": 0.740588366985321,
      "rewards/rollout_reward_func/std": 0.586037814617157,
      "sampling/importance_sampling_ratio/max": 1.266194462776184,
      "sampling/importance_sampling_ratio/mean": 1.0497159957885742,
      "sampling/importance_sampling_ratio/min": 0.8584321141242981,
      "sampling/sampling_logp_difference/max": 0.24932479858398438,
      "sampling/sampling_logp_difference/mean": 0.023144492879509926,
      "step": 675,
      "step_time": 10.49066086899802
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1046.0,
      "completions/max_terminated_length": 1046.0,
      "completions/mean_length": 525.375,
      "completions/mean_terminated_length": 514.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4442017525434494,
      "epoch": 0.00676,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5178921818733215,
      "kl": 1.6952897161245346,
      "learning_rate": 1.605099013371495e-05,
      "loss": -0.011,
      "num_tokens": 15859957.0,
      "reward": 0.6792142391204834,
      "reward_std": 0.5677767395973206,
      "rewards/rollout_reward_func/mean": 0.6792142391204834,
      "rewards/rollout_reward_func/std": 0.6093670129776001,
      "sampling/importance_sampling_ratio/max": 1.0544252395629883,
      "sampling/importance_sampling_ratio/mean": 0.9134812951087952,
      "sampling/importance_sampling_ratio/min": 0.2927674353122711,
      "sampling/sampling_logp_difference/max": 0.17868423461914062,
      "sampling/sampling_logp_difference/mean": 0.026219552382826805,
      "step": 676,
      "step_time": 8.857892474000437
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1725.0,
      "completions/max_terminated_length": 1725.0,
      "completions/mean_length": 897.0625,
      "completions/mean_terminated_length": 932.4615478515625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.035911902785301,
      "epoch": 0.00677,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4399863481521606,
      "kl": 0.8014046624302864,
      "learning_rate": 1.599260853029348e-05,
      "loss": -0.233,
      "num_tokens": 15890036.0,
      "reward": 0.34552067518234253,
      "reward_std": 0.6439216136932373,
      "rewards/rollout_reward_func/mean": 0.34552067518234253,
      "rewards/rollout_reward_func/std": 0.6345322728157043,
      "sampling/importance_sampling_ratio/max": 1.5418018102645874,
      "sampling/importance_sampling_ratio/mean": 0.9995754361152649,
      "sampling/importance_sampling_ratio/min": 5.388668400385654e-10,
      "sampling/sampling_logp_difference/max": 20.933704376220703,
      "sampling/sampling_logp_difference/mean": 0.08636881411075592,
      "step": 677,
      "step_time": 12.329484920998766
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 905.0,
      "completions/max_terminated_length": 905.0,
      "completions/mean_length": 407.625,
      "completions/mean_terminated_length": 386.5333557128906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2475827857851982,
      "epoch": 0.00678,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7498685717582703,
      "kl": 1.700403768569231,
      "learning_rate": 1.5934372121009246e-05,
      "loss": -0.046,
      "num_tokens": 15911552.0,
      "reward": 0.5310155153274536,
      "reward_std": 0.6391137838363647,
      "rewards/rollout_reward_func/mean": 0.5310155153274536,
      "rewards/rollout_reward_func/std": 0.6592024564743042,
      "sampling/importance_sampling_ratio/max": 1.2238770723342896,
      "sampling/importance_sampling_ratio/mean": 0.9646122455596924,
      "sampling/importance_sampling_ratio/min": 0.4582350552082062,
      "sampling/sampling_logp_difference/max": 0.22925877571105957,
      "sampling/sampling_logp_difference/mean": 0.03052441030740738,
      "step": 678,
      "step_time": 8.425362430999485
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 803.0,
      "completions/max_terminated_length": 781.0,
      "completions/mean_length": 360.25,
      "completions/mean_terminated_length": 275.0769348144531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.550108090043068,
      "epoch": 0.00679,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6403111219406128,
      "kl": 1.1837421506643295,
      "learning_rate": 1.5876281599944794e-05,
      "loss": 0.0346,
      "num_tokens": 15931600.0,
      "reward": 0.6178237795829773,
      "reward_std": 0.7445057034492493,
      "rewards/rollout_reward_func/mean": 0.6178237795829773,
      "rewards/rollout_reward_func/std": 0.719297468662262,
      "sampling/importance_sampling_ratio/max": 1.9465534687042236,
      "sampling/importance_sampling_ratio/mean": 1.0805667638778687,
      "sampling/importance_sampling_ratio/min": 0.7045955061912537,
      "sampling/sampling_logp_difference/max": 0.22378110885620117,
      "sampling/sampling_logp_difference/mean": 0.030105452984571457,
      "step": 679,
      "step_time": 8.111964378001176
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 921.0,
      "completions/max_terminated_length": 921.0,
      "completions/mean_length": 557.125,
      "completions/mean_terminated_length": 540.6666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4573722630739212,
      "epoch": 0.0068,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8897140622138977,
      "kl": 1.4080004878342152,
      "learning_rate": 1.581833765944391e-05,
      "loss": -0.0366,
      "num_tokens": 15955784.0,
      "reward": 0.35969042778015137,
      "reward_std": 0.826954185962677,
      "rewards/rollout_reward_func/mean": 0.35969042778015137,
      "rewards/rollout_reward_func/std": 0.8115602135658264,
      "sampling/importance_sampling_ratio/max": 1.1847797632217407,
      "sampling/importance_sampling_ratio/mean": 0.9342596530914307,
      "sampling/importance_sampling_ratio/min": 1.661819661914754e-12,
      "sampling/sampling_logp_difference/max": 21.521066665649414,
      "sampling/sampling_logp_difference/mean": 0.16487188637256622,
      "step": 680,
      "step_time": 8.69450847799908
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1893.0,
      "completions/max_terminated_length": 1893.0,
      "completions/mean_length": 374.875,
      "completions/mean_terminated_length": 374.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.075052548199892,
      "epoch": 0.00681,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8252686858177185,
      "kl": 1.1079923398792744,
      "learning_rate": 1.576054099010338e-05,
      "loss": 0.143,
      "num_tokens": 15975646.0,
      "reward": 0.4592648148536682,
      "reward_std": 0.5599569082260132,
      "rewards/rollout_reward_func/mean": 0.4592648148536682,
      "rewards/rollout_reward_func/std": 0.5590508580207825,
      "sampling/importance_sampling_ratio/max": 1.5413812398910522,
      "sampling/importance_sampling_ratio/mean": 1.0040078163146973,
      "sampling/importance_sampling_ratio/min": 0.6755105257034302,
      "sampling/sampling_logp_difference/max": 0.18985629081726074,
      "sampling/sampling_logp_difference/mean": 0.028700899332761765,
      "step": 681,
      "step_time": 10.17337551699893
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1582.0,
      "completions/max_terminated_length": 1582.0,
      "completions/mean_length": 476.25,
      "completions/mean_terminated_length": 481.857177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3420511931180954,
      "epoch": 0.00682,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2659615278244019,
      "kl": 0.716154046356678,
      "learning_rate": 1.570289228076477e-05,
      "loss": 0.1174,
      "num_tokens": 15997690.0,
      "reward": 0.6127112507820129,
      "reward_std": 0.6483471393585205,
      "rewards/rollout_reward_func/mean": 0.6127112507820129,
      "rewards/rollout_reward_func/std": 0.6330800652503967,
      "sampling/importance_sampling_ratio/max": 1.2310723066329956,
      "sampling/importance_sampling_ratio/mean": 0.9015952348709106,
      "sampling/importance_sampling_ratio/min": 3.3583846271843408e-15,
      "sampling/sampling_logp_difference/max": 21.889667510986328,
      "sampling/sampling_logp_difference/mean": 0.3015367090702057,
      "step": 682,
      "step_time": 10.638689300998522
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 921.0,
      "completions/max_terminated_length": 921.0,
      "completions/mean_length": 440.4375,
      "completions/mean_terminated_length": 402.3571472167969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1756690368056297,
      "epoch": 0.00683,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8645580410957336,
      "kl": 1.4760319516062737,
      "learning_rate": 1.56453922185062e-05,
      "loss": -0.0251,
      "num_tokens": 16019476.0,
      "reward": 0.5390343070030212,
      "reward_std": 0.6618067026138306,
      "rewards/rollout_reward_func/mean": 0.5390343070030212,
      "rewards/rollout_reward_func/std": 0.6497278809547424,
      "sampling/importance_sampling_ratio/max": 1.0731407403945923,
      "sampling/importance_sampling_ratio/mean": 0.9548696875572205,
      "sampling/importance_sampling_ratio/min": 0.6586828231811523,
      "sampling/sampling_logp_difference/max": 0.22665977478027344,
      "sampling/sampling_logp_difference/mean": 0.02973107434809208,
      "step": 683,
      "step_time": 8.172434535999855
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1631.0,
      "completions/max_terminated_length": 1631.0,
      "completions/mean_length": 581.8125,
      "completions/mean_terminated_length": 561.857177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.32006274163723,
      "epoch": 0.00684,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6839733719825745,
      "kl": 1.4500991962850094,
      "learning_rate": 1.558804148863416e-05,
      "loss": -0.177,
      "num_tokens": 16043903.0,
      "reward": 0.6111427545547485,
      "reward_std": 0.6452494859695435,
      "rewards/rollout_reward_func/mean": 0.6111427545547485,
      "rewards/rollout_reward_func/std": 0.630233108997345,
      "sampling/importance_sampling_ratio/max": 1.3090561628341675,
      "sampling/importance_sampling_ratio/mean": 0.9364181756973267,
      "sampling/importance_sampling_ratio/min": 5.135005753942323e-09,
      "sampling/sampling_logp_difference/max": 13.708704948425293,
      "sampling/sampling_logp_difference/mean": 0.10387608408927917,
      "step": 684,
      "step_time": 10.76439902400125
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1666.0,
      "completions/max_terminated_length": 767.0,
      "completions/mean_length": 468.125,
      "completions/mean_terminated_length": 337.15386962890625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2208957001566887,
      "epoch": 0.00685,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4422285556793213,
      "kl": 1.4423366039991379,
      "learning_rate": 1.5530840774675334e-05,
      "loss": 0.1032,
      "num_tokens": 16066380.0,
      "reward": 0.4162796437740326,
      "reward_std": 0.7594186067581177,
      "rewards/rollout_reward_func/mean": 0.4162796437740326,
      "rewards/rollout_reward_func/std": 0.7371407747268677,
      "sampling/importance_sampling_ratio/max": 1.2299238443374634,
      "sampling/importance_sampling_ratio/mean": 0.9013667702674866,
      "sampling/importance_sampling_ratio/min": 1.3331573692210164e-11,
      "sampling/sampling_logp_difference/max": 16.03261947631836,
      "sampling/sampling_logp_difference/mean": 0.1259443759918213,
      "step": 685,
      "step_time": 11.116148570001315
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 930.0,
      "completions/max_terminated_length": 930.0,
      "completions/mean_length": 369.875,
      "completions/mean_terminated_length": 342.69232177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4483902528882027,
      "epoch": 0.00686,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5868804454803467,
      "kl": 0.9764808639883995,
      "learning_rate": 1.5473790758368476e-05,
      "loss": -0.0039,
      "num_tokens": 16086803.0,
      "reward": 0.4111071527004242,
      "reward_std": 0.7691795229911804,
      "rewards/rollout_reward_func/mean": 0.4111071527004242,
      "rewards/rollout_reward_func/std": 0.7456741333007812,
      "sampling/importance_sampling_ratio/max": 1.1200357675552368,
      "sampling/importance_sampling_ratio/mean": 0.8417237997055054,
      "sampling/importance_sampling_ratio/min": 7.181008446297682e-14,
      "sampling/sampling_logp_difference/max": 19.80322265625,
      "sampling/sampling_logp_difference/mean": 0.2678983211517334,
      "step": 686,
      "step_time": 8.47915383399777
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 946.0,
      "completions/max_terminated_length": 946.0,
      "completions/mean_length": 485.0,
      "completions/mean_terminated_length": 464.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.220985822379589,
      "epoch": 0.00687,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7172864079475403,
      "kl": 1.399572279304266,
      "learning_rate": 1.5416892119656275e-05,
      "loss": 0.0916,
      "num_tokens": 16108878.0,
      "reward": 0.4759719967842102,
      "reward_std": 0.6664689183235168,
      "rewards/rollout_reward_func/mean": 0.4759719967842102,
      "rewards/rollout_reward_func/std": 0.6504893898963928,
      "sampling/importance_sampling_ratio/max": 1.3279204368591309,
      "sampling/importance_sampling_ratio/mean": 0.933225154876709,
      "sampling/importance_sampling_ratio/min": 1.5609437570632267e-13,
      "sampling/sampling_logp_difference/max": 21.36569595336914,
      "sampling/sampling_logp_difference/mean": 0.4251546859741211,
      "step": 687,
      "step_time": 8.373947043002772
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1629.0,
      "completions/max_terminated_length": 1629.0,
      "completions/mean_length": 476.125,
      "completions/mean_terminated_length": 433.14288330078125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2422768324613571,
      "epoch": 0.00688,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4164313077926636,
      "kl": 1.0519587844610214,
      "learning_rate": 1.5360145536677218e-05,
      "loss": 0.1405,
      "num_tokens": 16130794.0,
      "reward": 0.740947425365448,
      "reward_std": 0.6017705202102661,
      "rewards/rollout_reward_func/mean": 0.740947425365448,
      "rewards/rollout_reward_func/std": 0.5902790427207947,
      "sampling/importance_sampling_ratio/max": 1.3088830709457397,
      "sampling/importance_sampling_ratio/mean": 0.9372510313987732,
      "sampling/importance_sampling_ratio/min": 4.0220299313276087e-11,
      "sampling/sampling_logp_difference/max": 21.263147354125977,
      "sampling/sampling_logp_difference/mean": 0.14568933844566345,
      "step": 688,
      "step_time": 10.226489279999441
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1916.0,
      "completions/max_terminated_length": 896.0,
      "completions/mean_length": 601.5625,
      "completions/mean_terminated_length": 529.84619140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6279980167746544,
      "epoch": 0.00689,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1701464653015137,
      "kl": 1.7955193929374218,
      "learning_rate": 1.5303551685757565e-05,
      "loss": 0.1172,
      "num_tokens": 16156235.0,
      "reward": 0.5965731143951416,
      "reward_std": 0.557382345199585,
      "rewards/rollout_reward_func/mean": 0.5965731143951416,
      "rewards/rollout_reward_func/std": 0.5387891530990601,
      "sampling/importance_sampling_ratio/max": 1.129630446434021,
      "sampling/importance_sampling_ratio/mean": 0.7254347205162048,
      "sampling/importance_sampling_ratio/min": 2.0393434594571762e-14,
      "sampling/sampling_logp_difference/max": 21.460853576660156,
      "sampling/sampling_logp_difference/mean": 0.3417104184627533,
      "step": 689,
      "step_time": 12.104364930999509
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1820.0,
      "completions/max_terminated_length": 1820.0,
      "completions/mean_length": 728.0625,
      "completions/mean_terminated_length": 573.2727661132812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4800849184393883,
      "epoch": 0.0069,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5018415451049805,
      "kl": 0.9843963757157326,
      "learning_rate": 1.5247111241403259e-05,
      "loss": 0.0924,
      "num_tokens": 16182902.0,
      "reward": 0.3451913595199585,
      "reward_std": 0.7317287921905518,
      "rewards/rollout_reward_func/mean": 0.3451913595199585,
      "rewards/rollout_reward_func/std": 0.7310022711753845,
      "sampling/importance_sampling_ratio/max": 1.433858036994934,
      "sampling/importance_sampling_ratio/mean": 1.0573289394378662,
      "sampling/importance_sampling_ratio/min": 0.8017138242721558,
      "sampling/sampling_logp_difference/max": 0.2149796485900879,
      "sampling/sampling_logp_difference/mean": 0.02916000969707966,
      "step": 690,
      "step_time": 12.031000816003143
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1726.0,
      "completions/max_terminated_length": 924.0,
      "completions/mean_length": 481.25,
      "completions/mean_terminated_length": 400.0769348144531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1350087076425552,
      "epoch": 0.00691,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7563462257385254,
      "kl": 0.9922362789511681,
      "learning_rate": 1.519082487629187e-05,
      "loss": -0.1785,
      "num_tokens": 16205985.0,
      "reward": -0.037388771772384644,
      "reward_std": 0.6577937006950378,
      "rewards/rollout_reward_func/mean": -0.037388771772384644,
      "rewards/rollout_reward_func/std": 0.7290431261062622,
      "sampling/importance_sampling_ratio/max": 1.2527647018432617,
      "sampling/importance_sampling_ratio/mean": 0.9612501263618469,
      "sampling/importance_sampling_ratio/min": 9.20134628212832e-13,
      "sampling/sampling_logp_difference/max": 16.600141525268555,
      "sampling/sampling_logp_difference/mean": 0.12294116616249084,
      "step": 691,
      "step_time": 10.8970279210007
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1735.0,
      "completions/max_terminated_length": 1582.0,
      "completions/mean_length": 660.3125,
      "completions/mean_terminated_length": 589.9166870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5836433619260788,
      "epoch": 0.00692,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1740033626556396,
      "kl": 1.1711075901985168,
      "learning_rate": 1.5134693261264616e-05,
      "loss": 0.0619,
      "num_tokens": 16231650.0,
      "reward": 0.6315708160400391,
      "reward_std": 0.719792366027832,
      "rewards/rollout_reward_func/mean": 0.6315708160400391,
      "rewards/rollout_reward_func/std": 0.6954714059829712,
      "sampling/importance_sampling_ratio/max": 1.441172480583191,
      "sampling/importance_sampling_ratio/mean": 0.7280750274658203,
      "sampling/importance_sampling_ratio/min": 5.534106384469785e-22,
      "sampling/sampling_logp_difference/max": 24.142091751098633,
      "sampling/sampling_logp_difference/mean": 0.3537870943546295,
      "step": 692,
      "step_time": 12.481537988000127
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 962.0,
      "completions/max_terminated_length": 886.0,
      "completions/mean_length": 543.5625,
      "completions/mean_terminated_length": 499.0714416503906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2969263419508934,
      "epoch": 0.00693,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.4383161664009094,
      "kl": 1.5649254936724901,
      "learning_rate": 1.5078717065318332e-05,
      "loss": 0.071,
      "num_tokens": 16255158.0,
      "reward": 0.6767088174819946,
      "reward_std": 0.3749246597290039,
      "rewards/rollout_reward_func/mean": 0.6767088174819946,
      "rewards/rollout_reward_func/std": 0.6114568710327148,
      "sampling/importance_sampling_ratio/max": 1.220386028289795,
      "sampling/importance_sampling_ratio/mean": 0.971204400062561,
      "sampling/importance_sampling_ratio/min": 0.6620696783065796,
      "sampling/sampling_logp_difference/max": 0.29119110107421875,
      "sampling/sampling_logp_difference/mean": 0.023584114387631416,
      "step": 693,
      "step_time": 8.54206246099966
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1851.0,
      "completions/max_terminated_length": 1851.0,
      "completions/mean_length": 653.125,
      "completions/mean_terminated_length": 628.7692260742188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.536184087395668,
      "epoch": 0.00694,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1309144496917725,
      "kl": 1.9746625125408173,
      "learning_rate": 1.502289695559752e-05,
      "loss": 0.0883,
      "num_tokens": 16281204.0,
      "reward": 0.35753610730171204,
      "reward_std": 0.8274136781692505,
      "rewards/rollout_reward_func/mean": 0.35753610730171204,
      "rewards/rollout_reward_func/std": 0.8102326989173889,
      "sampling/importance_sampling_ratio/max": 1.2096456289291382,
      "sampling/importance_sampling_ratio/mean": 0.9431241750717163,
      "sampling/importance_sampling_ratio/min": 0.6049433946609497,
      "sampling/sampling_logp_difference/max": 0.30788326263427734,
      "sampling/sampling_logp_difference/mean": 0.03042389452457428,
      "step": 694,
      "step_time": 11.066275493003559
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2406.0,
      "completions/max_terminated_length": 2406.0,
      "completions/mean_length": 599.5,
      "completions/mean_terminated_length": 599.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.163339901715517,
      "epoch": 0.00695,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1871544122695923,
      "kl": 1.0556674748659134,
      "learning_rate": 1.4967233597386388e-05,
      "loss": 0.0644,
      "num_tokens": 16305698.0,
      "reward": 0.8043465614318848,
      "reward_std": 0.42210233211517334,
      "rewards/rollout_reward_func/mean": 0.8043465614318848,
      "rewards/rollout_reward_func/std": 0.42112430930137634,
      "sampling/importance_sampling_ratio/max": 1.2585638761520386,
      "sampling/importance_sampling_ratio/mean": 0.9921461939811707,
      "sampling/importance_sampling_ratio/min": 0.8626058101654053,
      "sampling/sampling_logp_difference/max": 0.18103742599487305,
      "sampling/sampling_logp_difference/mean": 0.02151741459965706,
      "step": 695,
      "step_time": 12.125617132003754
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1680.0,
      "completions/max_terminated_length": 1680.0,
      "completions/mean_length": 470.9375,
      "completions/mean_terminated_length": 470.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1896422319114208,
      "epoch": 0.00696,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7166163921356201,
      "kl": 1.9508598782122135,
      "learning_rate": 1.4911727654100936e-05,
      "loss": -0.048,
      "num_tokens": 16327544.0,
      "reward": 0.6164870858192444,
      "reward_std": 0.642276406288147,
      "rewards/rollout_reward_func/mean": 0.6164870858192444,
      "rewards/rollout_reward_func/std": 0.6285626888275146,
      "sampling/importance_sampling_ratio/max": 1.0045812129974365,
      "sampling/importance_sampling_ratio/mean": 0.8776983618736267,
      "sampling/importance_sampling_ratio/min": 1.2418698014382556e-10,
      "sampling/sampling_logp_difference/max": 21.480478286743164,
      "sampling/sampling_logp_difference/mean": 0.18945303559303284,
      "step": 696,
      "step_time": 10.076945456999965
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 829.0,
      "completions/max_terminated_length": 829.0,
      "completions/mean_length": 490.0,
      "completions/mean_terminated_length": 503.0000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3520624563097954,
      "epoch": 0.00697,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.123781442642212,
      "kl": 1.1766479760408401,
      "learning_rate": 1.4856379787281e-05,
      "loss": -0.1203,
      "num_tokens": 16350861.0,
      "reward": 0.4667368531227112,
      "reward_std": 0.6623847484588623,
      "rewards/rollout_reward_func/mean": 0.4667368531227112,
      "rewards/rollout_reward_func/std": 0.658941924571991,
      "sampling/importance_sampling_ratio/max": 1.490035057067871,
      "sampling/importance_sampling_ratio/mean": 1.0538616180419922,
      "sampling/importance_sampling_ratio/min": 0.6578003168106079,
      "sampling/sampling_logp_difference/max": 0.18489885330200195,
      "sampling/sampling_logp_difference/mean": 0.029105495661497116,
      "step": 697,
      "step_time": 8.079749669999728
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1728.0,
      "completions/max_terminated_length": 1728.0,
      "completions/mean_length": 654.625,
      "completions/mean_terminated_length": 654.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0784658240154386,
      "epoch": 0.00698,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.111570119857788,
      "kl": 1.1433400884270668,
      "learning_rate": 1.4801190656582445e-05,
      "loss": -0.0851,
      "num_tokens": 16376178.0,
      "reward": 0.5535483360290527,
      "reward_std": 0.623715341091156,
      "rewards/rollout_reward_func/mean": 0.5535483360290527,
      "rewards/rollout_reward_func/std": 0.7259810566902161,
      "sampling/importance_sampling_ratio/max": 1.928082823753357,
      "sampling/importance_sampling_ratio/mean": 1.002072811126709,
      "sampling/importance_sampling_ratio/min": 3.752467669926318e-09,
      "sampling/sampling_logp_difference/max": 13.9297456741333,
      "sampling/sampling_logp_difference/mean": 0.12300223857164383,
      "step": 698,
      "step_time": 11.333345023998845
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 973.0,
      "completions/max_terminated_length": 973.0,
      "completions/mean_length": 545.0,
      "completions/mean_terminated_length": 514.3077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3893871158361435,
      "epoch": 0.00699,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.719835102558136,
      "kl": 0.9843301996588707,
      "learning_rate": 1.474616091976925e-05,
      "loss": -0.0897,
      "num_tokens": 16399943.0,
      "reward": 0.5927575826644897,
      "reward_std": 0.5622844696044922,
      "rewards/rollout_reward_func/mean": 0.5927575826644897,
      "rewards/rollout_reward_func/std": 0.5432231426239014,
      "sampling/importance_sampling_ratio/max": 1.3103532791137695,
      "sampling/importance_sampling_ratio/mean": 0.9614405632019043,
      "sampling/importance_sampling_ratio/min": 3.262217163069181e-08,
      "sampling/sampling_logp_difference/max": 14.316007614135742,
      "sampling/sampling_logp_difference/mean": 0.10725592821836472,
      "step": 699,
      "step_time": 8.675829417998102
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1928.0,
      "completions/max_terminated_length": 1726.0,
      "completions/mean_length": 792.8125,
      "completions/mean_terminated_length": 756.5385131835938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6562182158231735,
      "epoch": 0.007,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3721543550491333,
      "kl": 0.9210209026932716,
      "learning_rate": 1.4691291232705654e-05,
      "loss": 0.0054,
      "num_tokens": 16428146.0,
      "reward": 0.7406536936759949,
      "reward_std": 0.480239599943161,
      "rewards/rollout_reward_func/mean": 0.7406536936759949,
      "rewards/rollout_reward_func/std": 0.46410658955574036,
      "sampling/importance_sampling_ratio/max": 1.8446707725524902,
      "sampling/importance_sampling_ratio/mean": 0.8942121267318726,
      "sampling/importance_sampling_ratio/min": 2.3406282275776936e-20,
      "sampling/sampling_logp_difference/max": 17.50212860107422,
      "sampling/sampling_logp_difference/mean": 0.191732719540596,
      "step": 700,
      "step_time": 13.073169418999896
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1662.0,
      "completions/max_terminated_length": 894.0,
      "completions/mean_length": 434.0625,
      "completions/mean_terminated_length": 286.8461608886719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3907948359847069,
      "epoch": 0.00701,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.357908844947815,
      "kl": 0.845918346196413,
      "learning_rate": 1.4636582249348399e-05,
      "loss": -0.1616,
      "num_tokens": 16449966.0,
      "reward": 0.33031708002090454,
      "reward_std": 0.6491563320159912,
      "rewards/rollout_reward_func/mean": 0.33031708002090454,
      "rewards/rollout_reward_func/std": 0.6454206705093384,
      "sampling/importance_sampling_ratio/max": 1.378141164779663,
      "sampling/importance_sampling_ratio/mean": 0.8920959830284119,
      "sampling/importance_sampling_ratio/min": 5.742792909504658e-19,
      "sampling/sampling_logp_difference/max": 25.339630126953125,
      "sampling/sampling_logp_difference/mean": 0.28778746724128723,
      "step": 701,
      "step_time": 10.663540681998711
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1900.0,
      "completions/max_terminated_length": 780.0,
      "completions/mean_length": 673.25,
      "completions/mean_terminated_length": 308.91668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4922561720013618,
      "epoch": 0.00702,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.6618748903274536,
      "kl": 0.5332339555025101,
      "learning_rate": 1.4582034621738892e-05,
      "loss": -0.1346,
      "num_tokens": 16475050.0,
      "reward": 0.8639968633651733,
      "reward_std": 0.25197380781173706,
      "rewards/rollout_reward_func/mean": 0.8639968633651733,
      "rewards/rollout_reward_func/std": 0.37181463837623596,
      "sampling/importance_sampling_ratio/max": 1.262352705001831,
      "sampling/importance_sampling_ratio/mean": 0.9712091684341431,
      "sampling/importance_sampling_ratio/min": 0.41024312376976013,
      "sampling/sampling_logp_difference/max": 0.25640296936035156,
      "sampling/sampling_logp_difference/mean": 0.031989626586437225,
      "step": 702,
      "step_time": 12.332565322001756
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1842.0,
      "completions/max_terminated_length": 881.0,
      "completions/mean_length": 646.0625,
      "completions/mean_terminated_length": 405.18182373046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3757896944880486,
      "epoch": 0.00703,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.307963252067566,
      "kl": 1.025855090469122,
      "learning_rate": 1.4527648999995431e-05,
      "loss": 0.1794,
      "num_tokens": 16500720.0,
      "reward": 0.3575502932071686,
      "reward_std": 0.7303059101104736,
      "rewards/rollout_reward_func/mean": 0.3575502932071686,
      "rewards/rollout_reward_func/std": 0.71901935338974,
      "sampling/importance_sampling_ratio/max": 1.2504680156707764,
      "sampling/importance_sampling_ratio/mean": 0.9704500436782837,
      "sampling/importance_sampling_ratio/min": 0.5449524521827698,
      "sampling/sampling_logp_difference/max": 0.36342883110046387,
      "sampling/sampling_logp_difference/mean": 0.02601504884660244,
      "step": 703,
      "step_time": 11.814979606999259
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1890.0,
      "completions/max_terminated_length": 1890.0,
      "completions/mean_length": 653.5625,
      "completions/mean_terminated_length": 517.8181762695312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4462593719363213,
      "epoch": 0.00704,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3948261737823486,
      "kl": 0.9775891043245792,
      "learning_rate": 1.4473426032305481e-05,
      "loss": 0.1387,
      "num_tokens": 16525850.0,
      "reward": 0.4269237518310547,
      "reward_std": 0.7549272775650024,
      "rewards/rollout_reward_func/mean": 0.4269237518310547,
      "rewards/rollout_reward_func/std": 0.7319669127464294,
      "sampling/importance_sampling_ratio/max": 1.1050164699554443,
      "sampling/importance_sampling_ratio/mean": 0.8867297768592834,
      "sampling/importance_sampling_ratio/min": 2.579633034960338e-11,
      "sampling/sampling_logp_difference/max": 23.59089469909668,
      "sampling/sampling_logp_difference/mean": 0.10473604500293732,
      "step": 704,
      "step_time": 11.816250660998776
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 950.0,
      "completions/max_terminated_length": 819.0,
      "completions/mean_length": 452.0,
      "completions/mean_terminated_length": 265.2727355957031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.624134361743927,
      "epoch": 0.00705,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.1454594135284424,
      "kl": 0.8867531940340996,
      "learning_rate": 1.4419366364917952e-05,
      "loss": 0.2696,
      "num_tokens": 16547844.0,
      "reward": 0.9328975081443787,
      "reward_std": 0.18979451060295105,
      "rewards/rollout_reward_func/mean": 0.9328975081443787,
      "rewards/rollout_reward_func/std": 0.26840999722480774,
      "sampling/importance_sampling_ratio/max": 1.4838175773620605,
      "sampling/importance_sampling_ratio/mean": 0.8136758804321289,
      "sampling/importance_sampling_ratio/min": 1.72956397615026e-25,
      "sampling/sampling_logp_difference/max": 21.65777587890625,
      "sampling/sampling_logp_difference/mean": 0.5429930090904236,
      "step": 705,
      "step_time": 9.986222370996984
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1708.0,
      "completions/max_terminated_length": 1708.0,
      "completions/mean_length": 376.5625,
      "completions/mean_terminated_length": 376.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1823937147855759,
      "epoch": 0.00706,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6538615822792053,
      "kl": 1.2140935249626637,
      "learning_rate": 1.4365470642135443e-05,
      "loss": -0.0984,
      "num_tokens": 16568459.0,
      "reward": 0.3792353868484497,
      "reward_std": 0.8967363834381104,
      "rewards/rollout_reward_func/mean": 0.3792353868484497,
      "rewards/rollout_reward_func/std": 0.8663355708122253,
      "sampling/importance_sampling_ratio/max": 1.0825272798538208,
      "sampling/importance_sampling_ratio/mean": 0.9815031886100769,
      "sampling/importance_sampling_ratio/min": 0.8258065581321716,
      "sampling/sampling_logp_difference/max": 0.17076635360717773,
      "sampling/sampling_logp_difference/mean": 0.023801688104867935,
      "step": 706,
      "step_time": 9.836614323001413
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 1822.0,
      "completions/max_terminated_length": 1706.0,
      "completions/mean_length": 723.1875,
      "completions/mean_terminated_length": 406.8888854980469,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7955635488033295,
      "epoch": 0.00707,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.708747148513794,
      "kl": 0.9942348077893257,
      "learning_rate": 1.4311739506306633e-05,
      "loss": 0.1394,
      "num_tokens": 16594793.0,
      "reward": 0.42570674419403076,
      "reward_std": 0.7189317941665649,
      "rewards/rollout_reward_func/mean": 0.42570674419403076,
      "rewards/rollout_reward_func/std": 0.7245398163795471,
      "sampling/importance_sampling_ratio/max": 1.6519193649291992,
      "sampling/importance_sampling_ratio/mean": 0.8820178508758545,
      "sampling/importance_sampling_ratio/min": 7.553483515564993e-19,
      "sampling/sampling_logp_difference/max": 11.620370864868164,
      "sampling/sampling_logp_difference/mean": 0.1599031388759613,
      "step": 707,
      "step_time": 13.150678265999886
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1700.0,
      "completions/max_terminated_length": 1700.0,
      "completions/mean_length": 517.6875,
      "completions/mean_terminated_length": 461.8461608886719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7156246230006218,
      "epoch": 0.00708,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5454522967338562,
      "kl": 0.9495202396064997,
      "learning_rate": 1.4258173597818598e-05,
      "loss": -0.1322,
      "num_tokens": 16617724.0,
      "reward": 0.7964968681335449,
      "reward_std": 0.4428926408290863,
      "rewards/rollout_reward_func/mean": 0.7964968681335449,
      "rewards/rollout_reward_func/std": 0.4376775920391083,
      "sampling/importance_sampling_ratio/max": 1.2766051292419434,
      "sampling/importance_sampling_ratio/mean": 0.9380710124969482,
      "sampling/importance_sampling_ratio/min": 1.0912246612362669e-20,
      "sampling/sampling_logp_difference/max": 23.896278381347656,
      "sampling/sampling_logp_difference/mean": 0.18220603466033936,
      "step": 708,
      "step_time": 11.083045516998027
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1776.0,
      "completions/max_terminated_length": 1776.0,
      "completions/mean_length": 517.8125,
      "completions/mean_terminated_length": 456.3077087402344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5389274805784225,
      "epoch": 0.00709,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9504926204681396,
      "kl": 0.8267150484025478,
      "learning_rate": 1.4204773555089145e-05,
      "loss": -0.1229,
      "num_tokens": 16640775.0,
      "reward": 0.5219003558158875,
      "reward_std": 0.5377866625785828,
      "rewards/rollout_reward_func/mean": 0.5219003558158875,
      "rewards/rollout_reward_func/std": 0.5601289868354797,
      "sampling/importance_sampling_ratio/max": 1.4434703588485718,
      "sampling/importance_sampling_ratio/mean": 0.9166728258132935,
      "sampling/importance_sampling_ratio/min": 3.6712441975339516e-09,
      "sampling/sampling_logp_difference/max": 11.459701538085938,
      "sampling/sampling_logp_difference/mean": 0.14061549305915833,
      "step": 709,
      "step_time": 11.431732481996733
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1656.0,
      "completions/max_terminated_length": 903.0,
      "completions/mean_length": 693.8125,
      "completions/mean_terminated_length": 607.2000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.599947765469551,
      "epoch": 0.0071,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0334323644638062,
      "kl": 0.9434119686484337,
      "learning_rate": 1.4151540014559254e-05,
      "loss": 0.026,
      "num_tokens": 16667481.0,
      "reward": 0.4301490783691406,
      "reward_std": 0.7580746412277222,
      "rewards/rollout_reward_func/mean": 0.4301490783691406,
      "rewards/rollout_reward_func/std": 0.808670163154602,
      "sampling/importance_sampling_ratio/max": 1.4717339277267456,
      "sampling/importance_sampling_ratio/mean": 0.8250753879547119,
      "sampling/importance_sampling_ratio/min": 1.1503064145352562e-18,
      "sampling/sampling_logp_difference/max": 21.3045654296875,
      "sampling/sampling_logp_difference/mean": 0.2549584209918976,
      "step": 710,
      "step_time": 11.556789353000568
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1717.0,
      "completions/max_terminated_length": 940.0,
      "completions/mean_length": 610.0625,
      "completions/mean_terminated_length": 392.8333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5362486727535725,
      "epoch": 0.00711,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7814918756484985,
      "kl": 0.6296849995851517,
      "learning_rate": 1.4098473610685488e-05,
      "loss": 0.0409,
      "num_tokens": 16691860.0,
      "reward": 0.48300495743751526,
      "reward_std": 0.7032579183578491,
      "rewards/rollout_reward_func/mean": 0.48300495743751526,
      "rewards/rollout_reward_func/std": 0.7339441180229187,
      "sampling/importance_sampling_ratio/max": 1.9606149196624756,
      "sampling/importance_sampling_ratio/mean": 1.0997477769851685,
      "sampling/importance_sampling_ratio/min": 0.4182867407798767,
      "sampling/sampling_logp_difference/max": 0.3381448984146118,
      "sampling/sampling_logp_difference/mean": 0.03253721818327904,
      "step": 711,
      "step_time": 11.243264537999494
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1974.0,
      "completions/max_terminated_length": 1002.0,
      "completions/mean_length": 575.1875,
      "completions/mean_terminated_length": 456.2857360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4410144332796335,
      "epoch": 0.00712,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5734072923660278,
      "kl": 1.3055023178458214,
      "learning_rate": 1.4045574975932377e-05,
      "loss": 0.0827,
      "num_tokens": 16715623.0,
      "reward": 0.4712754487991333,
      "reward_std": 0.6571270823478699,
      "rewards/rollout_reward_func/mean": 0.4712754487991333,
      "rewards/rollout_reward_func/std": 0.653060257434845,
      "sampling/importance_sampling_ratio/max": 1.1488077640533447,
      "sampling/importance_sampling_ratio/mean": 0.8082647919654846,
      "sampling/importance_sampling_ratio/min": 3.689025981731919e-23,
      "sampling/sampling_logp_difference/max": 22.042875289916992,
      "sampling/sampling_logp_difference/mean": 0.33532506227493286,
      "step": 712,
      "step_time": 12.65742191299978
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 3472.0,
      "completions/max_terminated_length": 786.0,
      "completions/mean_length": 780.1875,
      "completions/mean_terminated_length": 289.6000061035156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5461645796895027,
      "epoch": 0.00713,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6921166777610779,
      "kl": 0.69517882168293,
      "learning_rate": 1.3992844740764944e-05,
      "loss": -0.1748,
      "num_tokens": 16743182.0,
      "reward": 0.6712243556976318,
      "reward_std": 0.5792440176010132,
      "rewards/rollout_reward_func/mean": 0.6712243556976318,
      "rewards/rollout_reward_func/std": 0.6184772849082947,
      "sampling/importance_sampling_ratio/max": 1.637166142463684,
      "sampling/importance_sampling_ratio/mean": 0.9999202489852905,
      "sampling/importance_sampling_ratio/min": 0.5146482586860657,
      "sampling/sampling_logp_difference/max": 0.2439255714416504,
      "sampling/sampling_logp_difference/mean": 0.03103027679026127,
      "step": 713,
      "step_time": 16.532241532004264
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 884.0,
      "completions/max_terminated_length": 884.0,
      "completions/mean_length": 292.125,
      "completions/mean_terminated_length": 260.73333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1248899158090353,
      "epoch": 0.00714,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6931241750717163,
      "kl": 1.1628527417778969,
      "learning_rate": 1.3940283533641161e-05,
      "loss": -0.028,
      "num_tokens": 16761752.0,
      "reward": 0.5320744514465332,
      "reward_std": 0.6331992149353027,
      "rewards/rollout_reward_func/mean": 0.5320744514465332,
      "rewards/rollout_reward_func/std": 0.6556718945503235,
      "sampling/importance_sampling_ratio/max": 1.0008292198181152,
      "sampling/importance_sampling_ratio/mean": 0.9258711338043213,
      "sampling/importance_sampling_ratio/min": 0.696967363357544,
      "sampling/sampling_logp_difference/max": 0.1529247760772705,
      "sampling/sampling_logp_difference/mean": 0.029324276372790337,
      "step": 714,
      "step_time": 7.388006919003601
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1784.0,
      "completions/max_terminated_length": 1784.0,
      "completions/mean_length": 544.0,
      "completions/mean_terminated_length": 542.2307739257812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.1579250022768974,
      "epoch": 0.00715,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.734899878501892,
      "kl": 1.0630651377141476,
      "learning_rate": 1.3887891981004452e-05,
      "loss": 0.0986,
      "num_tokens": 16785294.0,
      "reward": 0.4206351637840271,
      "reward_std": 0.7512409687042236,
      "rewards/rollout_reward_func/mean": 0.4206351637840271,
      "rewards/rollout_reward_func/std": 0.7291008234024048,
      "sampling/importance_sampling_ratio/max": 1.599147915840149,
      "sampling/importance_sampling_ratio/mean": 0.9453530311584473,
      "sampling/importance_sampling_ratio/min": 1.3027953108807457e-28,
      "sampling/sampling_logp_difference/max": 21.390958786010742,
      "sampling/sampling_logp_difference/mean": 0.2509622573852539,
      "step": 715,
      "step_time": 11.619461353995575
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1709.0,
      "completions/max_terminated_length": 1709.0,
      "completions/mean_length": 723.625,
      "completions/mean_terminated_length": 692.923095703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.542530607432127,
      "epoch": 0.00716,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0488451719284058,
      "kl": 0.9204613007605076,
      "learning_rate": 1.3835670707276248e-05,
      "loss": 0.0864,
      "num_tokens": 16811705.0,
      "reward": 0.6217412352561951,
      "reward_std": 0.6025893092155457,
      "rewards/rollout_reward_func/mean": 0.6217412352561951,
      "rewards/rollout_reward_func/std": 0.6059678792953491,
      "sampling/importance_sampling_ratio/max": 1.126154899597168,
      "sampling/importance_sampling_ratio/mean": 0.7224040031433105,
      "sampling/importance_sampling_ratio/min": 1.236608459553263e-21,
      "sampling/sampling_logp_difference/max": 18.42071533203125,
      "sampling/sampling_logp_difference/mean": 0.3200171887874603,
      "step": 716,
      "step_time": 11.807903707996957
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 936.0,
      "completions/max_terminated_length": 911.0,
      "completions/mean_length": 458.125,
      "completions/mean_terminated_length": 341.2727355957031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.437521293759346,
      "epoch": 0.00717,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8635270595550537,
      "kl": 0.6230896301567554,
      "learning_rate": 1.378362033484854e-05,
      "loss": 0.3487,
      "num_tokens": 16833810.0,
      "reward": 0.21997112035751343,
      "reward_std": 0.7954052090644836,
      "rewards/rollout_reward_func/mean": 0.21997112035751343,
      "rewards/rollout_reward_func/std": 0.7800721526145935,
      "sampling/importance_sampling_ratio/max": 1.5898783206939697,
      "sampling/importance_sampling_ratio/mean": 0.9630542397499084,
      "sampling/importance_sampling_ratio/min": 3.3532665986842095e-36,
      "sampling/sampling_logp_difference/max": 24.13275718688965,
      "sampling/sampling_logp_difference/mean": 0.2961810827255249,
      "step": 717,
      "step_time": 9.01472804399964
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 892.0,
      "completions/max_terminated_length": 862.0,
      "completions/mean_length": 484.25,
      "completions/mean_terminated_length": 414.15386962890625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5035808756947517,
      "epoch": 0.00718,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3266481161117554,
      "kl": 1.3570725359022617,
      "learning_rate": 1.3731741484076437e-05,
      "loss": 0.1144,
      "num_tokens": 16856199.0,
      "reward": 0.6750286817550659,
      "reward_std": 0.6216333508491516,
      "rewards/rollout_reward_func/mean": 0.6750286817550659,
      "rewards/rollout_reward_func/std": 0.6147534251213074,
      "sampling/importance_sampling_ratio/max": 1.1925420761108398,
      "sampling/importance_sampling_ratio/mean": 0.9561296701431274,
      "sampling/importance_sampling_ratio/min": 0.5428045392036438,
      "sampling/sampling_logp_difference/max": 0.19607830047607422,
      "sampling/sampling_logp_difference/mean": 0.03389275074005127,
      "step": 718,
      "step_time": 8.40682454600028
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 2878.0,
      "completions/max_terminated_length": 921.0,
      "completions/mean_length": 711.6875,
      "completions/mean_terminated_length": 408.70001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5958425160497427,
      "epoch": 0.00719,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1087982654571533,
      "kl": 0.9756546095013618,
      "learning_rate": 1.3680034773270805e-05,
      "loss": -0.1645,
      "num_tokens": 16882357.0,
      "reward": 0.6014577150344849,
      "reward_std": 0.5509649515151978,
      "rewards/rollout_reward_func/mean": 0.6014577150344849,
      "rewards/rollout_reward_func/std": 0.5324959754943848,
      "sampling/importance_sampling_ratio/max": 1.2004213333129883,
      "sampling/importance_sampling_ratio/mean": 0.9644163846969604,
      "sampling/importance_sampling_ratio/min": 0.30072057247161865,
      "sampling/sampling_logp_difference/max": 0.3687915802001953,
      "sampling/sampling_logp_difference/mean": 0.030057312920689583,
      "step": 719,
      "step_time": 15.376248401997145
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2689.0,
      "completions/max_terminated_length": 2648.0,
      "completions/mean_length": 779.4375,
      "completions/mean_terminated_length": 627.1538696289062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3939874190837145,
      "epoch": 0.0072,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1826224327087402,
      "kl": 0.5379478558897972,
      "learning_rate": 1.3628500818690886e-05,
      "loss": -0.2506,
      "num_tokens": 16910161.0,
      "reward": 0.5371236801147461,
      "reward_std": 0.6314152479171753,
      "rewards/rollout_reward_func/mean": 0.5371236801147461,
      "rewards/rollout_reward_func/std": 0.6513983011245728,
      "sampling/importance_sampling_ratio/max": 1.4625983238220215,
      "sampling/importance_sampling_ratio/mean": 0.9043285846710205,
      "sampling/importance_sampling_ratio/min": 4.5732144274678354e-18,
      "sampling/sampling_logp_difference/max": 21.822586059570312,
      "sampling/sampling_logp_difference/mean": 0.21521428227424622,
      "step": 720,
      "step_time": 15.054215104997638
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1874.0,
      "completions/max_terminated_length": 815.0,
      "completions/mean_length": 465.875,
      "completions/mean_terminated_length": 293.91668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.764878287911415,
      "epoch": 0.00721,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.765400767326355,
      "kl": 0.9870773516595364,
      "learning_rate": 1.3577140234536935e-05,
      "loss": 0.315,
      "num_tokens": 16932143.0,
      "reward": 0.7466549873352051,
      "reward_std": 0.5476303100585938,
      "rewards/rollout_reward_func/mean": 0.7466549873352051,
      "rewards/rollout_reward_func/std": 0.5740799903869629,
      "sampling/importance_sampling_ratio/max": 1.5102543830871582,
      "sampling/importance_sampling_ratio/mean": 0.9621061086654663,
      "sampling/importance_sampling_ratio/min": 1.677587313508866e-08,
      "sampling/sampling_logp_difference/max": 14.062419891357422,
      "sampling/sampling_logp_difference/mean": 0.08487087488174438,
      "step": 721,
      "step_time": 11.329886789002558
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1605.0,
      "completions/max_terminated_length": 1605.0,
      "completions/mean_length": 396.625,
      "completions/mean_terminated_length": 259.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5561937056481838,
      "epoch": 0.00722,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4347245693206787,
      "kl": 0.7461302019655704,
      "learning_rate": 1.352595363294293e-05,
      "loss": 0.1033,
      "num_tokens": 16952413.0,
      "reward": 0.6115221977233887,
      "reward_std": 0.5568301677703857,
      "rewards/rollout_reward_func/mean": 0.6115221977233887,
      "rewards/rollout_reward_func/std": 0.6228585243225098,
      "sampling/importance_sampling_ratio/max": 1.2789316177368164,
      "sampling/importance_sampling_ratio/mean": 0.8730728626251221,
      "sampling/importance_sampling_ratio/min": 7.830143377215245e-20,
      "sampling/sampling_logp_difference/max": 18.767879486083984,
      "sampling/sampling_logp_difference/mean": 0.2887651026248932,
      "step": 722,
      "step_time": 10.966019290997792
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 980.0,
      "completions/max_terminated_length": 945.0,
      "completions/mean_length": 450.25,
      "completions/mean_terminated_length": 374.9090881347656,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5539072677493095,
      "epoch": 0.00723,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9223983287811279,
      "kl": 1.116535734385252,
      "learning_rate": 1.3474941623969268e-05,
      "loss": 0.1167,
      "num_tokens": 16975103.0,
      "reward": 0.7280535697937012,
      "reward_std": 0.503871738910675,
      "rewards/rollout_reward_func/mean": 0.7280535697937012,
      "rewards/rollout_reward_func/std": 0.48681992292404175,
      "sampling/importance_sampling_ratio/max": 1.33633291721344,
      "sampling/importance_sampling_ratio/mean": 0.7038499116897583,
      "sampling/importance_sampling_ratio/min": 4.9691866413779575e-30,
      "sampling/sampling_logp_difference/max": 21.0473575592041,
      "sampling/sampling_logp_difference/mean": 0.3998992443084717,
      "step": 723,
      "step_time": 9.99919209699874
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1751.0,
      "completions/max_terminated_length": 859.0,
      "completions/mean_length": 623.1875,
      "completions/mean_terminated_length": 395.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3311140295118093,
      "epoch": 0.00724,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3462871313095093,
      "kl": 1.0258608609437943,
      "learning_rate": 1.3424104815595465e-05,
      "loss": -0.0139,
      "num_tokens": 16999314.0,
      "reward": 0.735025942325592,
      "reward_std": 0.49079185724258423,
      "rewards/rollout_reward_func/mean": 0.735025942325592,
      "rewards/rollout_reward_func/std": 0.47426074743270874,
      "sampling/importance_sampling_ratio/max": 1.2621561288833618,
      "sampling/importance_sampling_ratio/mean": 0.8745005130767822,
      "sampling/importance_sampling_ratio/min": 8.996992875102278e-14,
      "sampling/sampling_logp_difference/max": 19.94701385498047,
      "sampling/sampling_logp_difference/mean": 0.12907318770885468,
      "step": 724,
      "step_time": 12.269873610002833
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1700.0,
      "completions/max_terminated_length": 1700.0,
      "completions/mean_length": 622.3125,
      "completions/mean_terminated_length": 428.0909118652344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4999584816396236,
      "epoch": 0.00725,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4803268909454346,
      "kl": 0.7888909615576267,
      "learning_rate": 1.3373443813712959e-05,
      "loss": -0.2027,
      "num_tokens": 17024051.0,
      "reward": 0.5394012928009033,
      "reward_std": 0.6306537389755249,
      "rewards/rollout_reward_func/mean": 0.5394012928009033,
      "rewards/rollout_reward_func/std": 0.6496259570121765,
      "sampling/importance_sampling_ratio/max": 1.5666933059692383,
      "sampling/importance_sampling_ratio/mean": 0.973326563835144,
      "sampling/importance_sampling_ratio/min": 5.536274179447601e-17,
      "sampling/sampling_logp_difference/max": 16.181249618530273,
      "sampling/sampling_logp_difference/mean": 0.12189991027116776,
      "step": 725,
      "step_time": 11.941231341997991
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 942.0,
      "completions/max_terminated_length": 846.0,
      "completions/mean_length": 509.9375,
      "completions/mean_terminated_length": 310.70001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 2.126358225941658,
      "epoch": 0.00726,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8347178101539612,
      "kl": 0.8283669799566269,
      "learning_rate": 1.3322959222117833e-05,
      "loss": 0.1482,
      "num_tokens": 17047057.0,
      "reward": 0.3360403776168823,
      "reward_std": 0.6341661810874939,
      "rewards/rollout_reward_func/mean": 0.3360403776168823,
      "rewards/rollout_reward_func/std": 0.6438053250312805,
      "sampling/importance_sampling_ratio/max": 1.3144587278366089,
      "sampling/importance_sampling_ratio/mean": 0.9390147924423218,
      "sampling/importance_sampling_ratio/min": 6.19407136360692e-16,
      "sampling/sampling_logp_difference/max": 9.0570650100708,
      "sampling/sampling_logp_difference/mean": 0.1313377469778061,
      "step": 726,
      "step_time": 10.092168783001398
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1035.0,
      "completions/max_terminated_length": 944.0,
      "completions/mean_length": 274.6875,
      "completions/mean_terminated_length": 203.4615478515625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9548480026423931,
      "epoch": 0.00727,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5470442175865173,
      "kl": 0.7236841060221195,
      "learning_rate": 1.3272651642503678e-05,
      "loss": 0.1474,
      "num_tokens": 17065622.0,
      "reward": 0.4723438024520874,
      "reward_std": 0.6569405198097229,
      "rewards/rollout_reward_func/mean": 0.4723438024520874,
      "rewards/rollout_reward_func/std": 0.6526515483856201,
      "sampling/importance_sampling_ratio/max": 1.1414462327957153,
      "sampling/importance_sampling_ratio/mean": 0.8740733861923218,
      "sampling/importance_sampling_ratio/min": 4.486999873763271e-19,
      "sampling/sampling_logp_difference/max": 11.952070236206055,
      "sampling/sampling_logp_difference/mean": 0.3007725775241852,
      "step": 727,
      "step_time": 8.726305865997347
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1713.0,
      "completions/max_terminated_length": 1683.0,
      "completions/mean_length": 670.6875,
      "completions/mean_terminated_length": 519.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5185446441173553,
      "epoch": 0.00728,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.1857069730758667,
      "kl": 0.6065621897578239,
      "learning_rate": 1.3222521674454382e-05,
      "loss": 0.0801,
      "num_tokens": 17091457.0,
      "reward": 0.9339969158172607,
      "reward_std": 0.1866850107908249,
      "rewards/rollout_reward_func/mean": 0.9339969158172607,
      "rewards/rollout_reward_func/std": 0.26401248574256897,
      "sampling/importance_sampling_ratio/max": 1.200834035873413,
      "sampling/importance_sampling_ratio/mean": 0.8874415159225464,
      "sampling/importance_sampling_ratio/min": 7.768480954004064e-13,
      "sampling/sampling_logp_difference/max": 13.792911529541016,
      "sampling/sampling_logp_difference/mean": 0.08284850418567657,
      "step": 728,
      "step_time": 12.622288633998323
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.5,
      "completions/max_length": 1567.0,
      "completions/max_terminated_length": 723.0,
      "completions/mean_length": 510.3125,
      "completions/mean_terminated_length": 433.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9089046493172646,
      "epoch": 0.00729,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0300158262252808,
      "kl": 0.7098236978054047,
      "learning_rate": 1.317256991543698e-05,
      "loss": -0.0122,
      "num_tokens": 17115128.0,
      "reward": 0.7396451830863953,
      "reward_std": 0.5693786144256592,
      "rewards/rollout_reward_func/mean": 0.7396451830863953,
      "rewards/rollout_reward_func/std": 0.5892823338508606,
      "sampling/importance_sampling_ratio/max": 1.330251932144165,
      "sampling/importance_sampling_ratio/mean": 0.8860480785369873,
      "sampling/importance_sampling_ratio/min": 2.4391209139200667e-19,
      "sampling/sampling_logp_difference/max": 23.852712631225586,
      "sampling/sampling_logp_difference/mean": 0.20512104034423828,
      "step": 729,
      "step_time": 11.56677891399886
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1869.0,
      "completions/max_terminated_length": 1869.0,
      "completions/mean_length": 790.4375,
      "completions/mean_terminated_length": 704.0833740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6976289600133896,
      "epoch": 0.0073,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6176397800445557,
      "kl": 0.9471012875437737,
      "learning_rate": 1.312279696079457e-05,
      "loss": 0.0569,
      "num_tokens": 17143047.0,
      "reward": 0.6242030262947083,
      "reward_std": 0.7229246497154236,
      "rewards/rollout_reward_func/mean": 0.6242030262947083,
      "rewards/rollout_reward_func/std": 0.7120653390884399,
      "sampling/importance_sampling_ratio/max": 1.5157206058502197,
      "sampling/importance_sampling_ratio/mean": 1.050324559211731,
      "sampling/importance_sampling_ratio/min": 0.8042353987693787,
      "sampling/sampling_logp_difference/max": 0.24314546585083008,
      "sampling/sampling_logp_difference/mean": 0.03125927224755287,
      "step": 730,
      "step_time": 12.78490624799997
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1920.0,
      "completions/max_terminated_length": 1541.0,
      "completions/mean_length": 651.9375,
      "completions/mean_terminated_length": 326.0909118652344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4630260542035103,
      "epoch": 0.00731,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7303574681282043,
      "kl": 0.8045604154467583,
      "learning_rate": 1.3073203403739188e-05,
      "loss": -0.0193,
      "num_tokens": 17168131.0,
      "reward": 0.738385021686554,
      "reward_std": 0.46354734897613525,
      "rewards/rollout_reward_func/mean": 0.738385021686554,
      "rewards/rollout_reward_func/std": 0.46966829895973206,
      "sampling/importance_sampling_ratio/max": 1.1401479244232178,
      "sampling/importance_sampling_ratio/mean": 0.9076488018035889,
      "sampling/importance_sampling_ratio/min": 3.3069316729417153e-10,
      "sampling/sampling_logp_difference/max": 11.252653121948242,
      "sampling/sampling_logp_difference/mean": 0.11183680593967438,
      "step": 731,
      "step_time": 12.341101227999388
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1644.0,
      "completions/max_terminated_length": 1644.0,
      "completions/mean_length": 486.875,
      "completions/mean_terminated_length": 479.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3528459798544645,
      "epoch": 0.00732,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9281958937644958,
      "kl": 0.7765117399394512,
      "learning_rate": 1.302378983534473e-05,
      "loss": -0.1358,
      "num_tokens": 17191129.0,
      "reward": 0.47802025079727173,
      "reward_std": 0.7559918165206909,
      "rewards/rollout_reward_func/mean": 0.47802025079727173,
      "rewards/rollout_reward_func/std": 0.7449539303779602,
      "sampling/importance_sampling_ratio/max": 1.4681038856506348,
      "sampling/importance_sampling_ratio/mean": 0.9294395446777344,
      "sampling/importance_sampling_ratio/min": 5.574653958567773e-39,
      "sampling/sampling_logp_difference/max": 29.669105529785156,
      "sampling/sampling_logp_difference/mean": 0.5341948866844177,
      "step": 732,
      "step_time": 11.09061357399878
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1674.0,
      "completions/max_terminated_length": 911.0,
      "completions/mean_length": 624.8125,
      "completions/mean_terminated_length": 439.0833435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2827068120241165,
      "epoch": 0.00733,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3800740242004395,
      "kl": 0.5796309653669596,
      "learning_rate": 1.2974556844539937e-05,
      "loss": 0.0772,
      "num_tokens": 17216099.0,
      "reward": 0.40293610095977783,
      "reward_std": 0.6704577207565308,
      "rewards/rollout_reward_func/mean": 0.40293610095977783,
      "rewards/rollout_reward_func/std": 0.6557809710502625,
      "sampling/importance_sampling_ratio/max": 1.4290149211883545,
      "sampling/importance_sampling_ratio/mean": 0.9813700914382935,
      "sampling/importance_sampling_ratio/min": 0.8558064699172974,
      "sampling/sampling_logp_difference/max": 0.2321643829345703,
      "sampling/sampling_logp_difference/mean": 0.026981856673955917,
      "step": 733,
      "step_time": 11.563322262003567
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1641.0,
      "completions/max_terminated_length": 1641.0,
      "completions/mean_length": 503.1875,
      "completions/mean_terminated_length": 479.13336181640625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5739212520420551,
      "epoch": 0.00734,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8295342922210693,
      "kl": 1.948024444282055,
      "learning_rate": 1.2925505018101369e-05,
      "loss": 0.0331,
      "num_tokens": 17238879.0,
      "reward": 0.48508942127227783,
      "reward_std": 0.7551009654998779,
      "rewards/rollout_reward_func/mean": 0.48508942127227783,
      "rewards/rollout_reward_func/std": 0.7418003082275391,
      "sampling/importance_sampling_ratio/max": 1.3433058261871338,
      "sampling/importance_sampling_ratio/mean": 0.8055706024169922,
      "sampling/importance_sampling_ratio/min": 1.1282068612672447e-09,
      "sampling/sampling_logp_difference/max": 21.492685317993164,
      "sampling/sampling_logp_difference/mean": 0.2886311113834381,
      "step": 734,
      "step_time": 10.330118177000259
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1813.0,
      "completions/max_terminated_length": 1755.0,
      "completions/mean_length": 527.0625,
      "completions/mean_terminated_length": 451.8461608886719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.168872538022697,
      "epoch": 0.00735,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0115301609039307,
      "kl": 0.7813679995015264,
      "learning_rate": 1.2876634940646377e-05,
      "loss": 0.1264,
      "num_tokens": 17261258.0,
      "reward": 0.6748666167259216,
      "reward_std": 0.574161171913147,
      "rewards/rollout_reward_func/mean": 0.6748666167259216,
      "rewards/rollout_reward_func/std": 0.6182008981704712,
      "sampling/importance_sampling_ratio/max": 1.3019212484359741,
      "sampling/importance_sampling_ratio/mean": 0.9208186268806458,
      "sampling/importance_sampling_ratio/min": 1.8064107528914702e-17,
      "sampling/sampling_logp_difference/max": 15.367431640625,
      "sampling/sampling_logp_difference/mean": 0.26089268922805786,
      "step": 735,
      "step_time": 10.695029417003752
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1765.0,
      "completions/max_terminated_length": 844.0,
      "completions/mean_length": 473.75,
      "completions/mean_terminated_length": 358.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7069266801699996,
      "epoch": 0.00736,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.8835708498954773,
      "kl": 0.6836840845644474,
      "learning_rate": 1.2827947194626187e-05,
      "loss": 0.1022,
      "num_tokens": 17283257.0,
      "reward": 0.740156888961792,
      "reward_std": 0.382536381483078,
      "rewards/rollout_reward_func/mean": 0.740156888961792,
      "rewards/rollout_reward_func/std": 0.5875173211097717,
      "sampling/importance_sampling_ratio/max": 1.0574625730514526,
      "sampling/importance_sampling_ratio/mean": 0.9406354427337646,
      "sampling/importance_sampling_ratio/min": 0.5679864287376404,
      "sampling/sampling_logp_difference/max": 0.22729015350341797,
      "sampling/sampling_logp_difference/mean": 0.029775314033031464,
      "step": 736,
      "step_time": 10.515512639998633
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1758.0,
      "completions/max_terminated_length": 1758.0,
      "completions/mean_length": 512.8125,
      "completions/mean_terminated_length": 512.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.372670072130859,
      "epoch": 0.00737,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8185736536979675,
      "kl": 0.6454538889229298,
      "learning_rate": 1.2779442360318923e-05,
      "loss": 0.0258,
      "num_tokens": 17305967.0,
      "reward": 0.3630608916282654,
      "reward_std": 0.7027163505554199,
      "rewards/rollout_reward_func/mean": 0.3630608916282654,
      "rewards/rollout_reward_func/std": 0.7953253388404846,
      "sampling/importance_sampling_ratio/max": 1.1545242071151733,
      "sampling/importance_sampling_ratio/mean": 0.8859954476356506,
      "sampling/importance_sampling_ratio/min": 3.4932785830665836e-12,
      "sampling/sampling_logp_difference/max": 19.71409797668457,
      "sampling/sampling_logp_difference/mean": 0.361183226108551,
      "step": 737,
      "step_time": 10.428036691995658
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1663.0,
      "completions/max_terminated_length": 1663.0,
      "completions/mean_length": 463.5,
      "completions/mean_terminated_length": 424.64288330078125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.071507633663714,
      "epoch": 0.00738,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.33071959018707275,
      "kl": 0.5005251131951809,
      "learning_rate": 1.273112101582269e-05,
      "loss": -0.0652,
      "num_tokens": 17327482.0,
      "reward": 0.7293957471847534,
      "reward_std": 0.47509658336639404,
      "rewards/rollout_reward_func/mean": 0.7293957471847534,
      "rewards/rollout_reward_func/std": 0.48446446657180786,
      "sampling/importance_sampling_ratio/max": 1.9239658117294312,
      "sampling/importance_sampling_ratio/mean": 1.033240795135498,
      "sampling/importance_sampling_ratio/min": 0.6341056823730469,
      "sampling/sampling_logp_difference/max": 0.18655729293823242,
      "sampling/sampling_logp_difference/mean": 0.02750220149755478,
      "step": 738,
      "step_time": 9.781790583998372
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2693.0,
      "completions/max_terminated_length": 1638.0,
      "completions/mean_length": 489.375,
      "completions/mean_terminated_length": 347.15386962890625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.9315629750490189,
      "epoch": 0.00739,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9887440204620361,
      "kl": 0.921125765889883,
      "learning_rate": 1.2682983737048727e-05,
      "loss": -0.1815,
      "num_tokens": 17350524.0,
      "reward": 0.48196661472320557,
      "reward_std": 0.7356183528900146,
      "rewards/rollout_reward_func/mean": 0.48196661472320557,
      "rewards/rollout_reward_func/std": 0.7396226525306702,
      "sampling/importance_sampling_ratio/max": 1.1352357864379883,
      "sampling/importance_sampling_ratio/mean": 0.8399649858474731,
      "sampling/importance_sampling_ratio/min": 4.154065566736639e-15,
      "sampling/sampling_logp_difference/max": 20.0938777923584,
      "sampling/sampling_logp_difference/mean": 0.22972136735916138,
      "step": 739,
      "step_time": 13.608685631001208
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1844.0,
      "completions/max_terminated_length": 1844.0,
      "completions/mean_length": 340.9375,
      "completions/mean_terminated_length": 310.6000061035156,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.052926362492144,
      "epoch": 0.0074,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8556272387504578,
      "kl": 0.6645176094025373,
      "learning_rate": 1.2635031097714486e-05,
      "loss": 0.1355,
      "num_tokens": 17370060.0,
      "reward": 0.734735369682312,
      "reward_std": 0.4914027452468872,
      "rewards/rollout_reward_func/mean": 0.734735369682312,
      "rewards/rollout_reward_func/std": 0.47525179386138916,
      "sampling/importance_sampling_ratio/max": 1.0577244758605957,
      "sampling/importance_sampling_ratio/mean": 0.9117563366889954,
      "sampling/importance_sampling_ratio/min": 2.097829243652427e-09,
      "sampling/sampling_logp_difference/max": 20.02821922302246,
      "sampling/sampling_logp_difference/mean": 0.20306676626205444,
      "step": 740,
      "step_time": 10.496169668003859
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1773.0,
      "completions/max_terminated_length": 1773.0,
      "completions/mean_length": 500.125,
      "completions/mean_terminated_length": 404.7857360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6767013743519783,
      "epoch": 0.00741,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9610945582389832,
      "kl": 0.3070337474346161,
      "learning_rate": 1.2587263669336813e-05,
      "loss": -0.173,
      "num_tokens": 17392013.0,
      "reward": 0.5547110438346863,
      "reward_std": 0.7415959239006042,
      "rewards/rollout_reward_func/mean": 0.5547110438346863,
      "rewards/rollout_reward_func/std": 0.7193043828010559,
      "sampling/importance_sampling_ratio/max": 1.7293601036071777,
      "sampling/importance_sampling_ratio/mean": 1.0488271713256836,
      "sampling/importance_sampling_ratio/min": 0.7163048386573792,
      "sampling/sampling_logp_difference/max": 0.22116947174072266,
      "sampling/sampling_logp_difference/mean": 0.03216445818543434,
      "step": 741,
      "step_time": 10.45166489499752
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1775.0,
      "completions/max_terminated_length": 1649.0,
      "completions/mean_length": 520.75,
      "completions/mean_terminated_length": 437.13336181640625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3348287204280496,
      "epoch": 0.00742,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7282703518867493,
      "kl": 0.5476732887327671,
      "learning_rate": 1.2539682021225167e-05,
      "loss": 0.084,
      "num_tokens": 17414994.0,
      "reward": 0.7303438186645508,
      "reward_std": 0.49965357780456543,
      "rewards/rollout_reward_func/mean": 0.7303438186645508,
      "rewards/rollout_reward_func/std": 0.48271888494491577,
      "sampling/importance_sampling_ratio/max": 1.2197695970535278,
      "sampling/importance_sampling_ratio/mean": 0.8541154861450195,
      "sampling/importance_sampling_ratio/min": 2.6318264465674404e-15,
      "sampling/sampling_logp_difference/max": 23.268247604370117,
      "sampling/sampling_logp_difference/mean": 0.3747257888317108,
      "step": 742,
      "step_time": 10.442595235997942
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1761.0,
      "completions/max_terminated_length": 862.0,
      "completions/mean_length": 653.125,
      "completions/mean_terminated_length": 375.3333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3994691092520952,
      "epoch": 0.00743,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2554022073745728,
      "kl": 0.4994579255580902,
      "learning_rate": 1.24922867204748e-05,
      "loss": 0.0362,
      "num_tokens": 17439675.0,
      "reward": 0.48485350608825684,
      "reward_std": 0.7600932717323303,
      "rewards/rollout_reward_func/mean": 0.48485350608825684,
      "rewards/rollout_reward_func/std": 0.7345492243766785,
      "sampling/importance_sampling_ratio/max": 1.0907167196273804,
      "sampling/importance_sampling_ratio/mean": 0.938186526298523,
      "sampling/importance_sampling_ratio/min": 8.219078154070303e-05,
      "sampling/sampling_logp_difference/max": 9.420609474182129,
      "sampling/sampling_logp_difference/mean": 0.06645502895116806,
      "step": 743,
      "step_time": 11.224063940999258
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1710.0,
      "completions/max_terminated_length": 1710.0,
      "completions/mean_length": 583.8125,
      "completions/mean_terminated_length": 571.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0856718234717846,
      "epoch": 0.00744,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.4446030855178833,
      "kl": 0.5523861926048994,
      "learning_rate": 1.2445078331959997e-05,
      "loss": -0.0422,
      "num_tokens": 17463926.0,
      "reward": 0.7999573945999146,
      "reward_std": 0.2761181592941284,
      "rewards/rollout_reward_func/mean": 0.7999573945999146,
      "rewards/rollout_reward_func/std": 0.4301184117794037,
      "sampling/importance_sampling_ratio/max": 1.1477067470550537,
      "sampling/importance_sampling_ratio/mean": 1.0019510984420776,
      "sampling/importance_sampling_ratio/min": 0.719074010848999,
      "sampling/sampling_logp_difference/max": 0.15160608291625977,
      "sampling/sampling_logp_difference/mean": 0.023981602862477303,
      "step": 744,
      "step_time": 10.226852859002975
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1830.0,
      "completions/max_terminated_length": 1830.0,
      "completions/mean_length": 568.0,
      "completions/mean_terminated_length": 567.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4264846490696073,
      "epoch": 0.00745,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4788230359554291,
      "kl": 0.43281921930611134,
      "learning_rate": 1.2398057418327372e-05,
      "loss": -0.061,
      "num_tokens": 17487996.0,
      "reward": 0.692672848701477,
      "reward_std": 0.7150239944458008,
      "rewards/rollout_reward_func/mean": 0.692672848701477,
      "rewards/rollout_reward_func/std": 0.6942877173423767,
      "sampling/importance_sampling_ratio/max": 1.0314947366714478,
      "sampling/importance_sampling_ratio/mean": 0.9112381935119629,
      "sampling/importance_sampling_ratio/min": 0.49092796444892883,
      "sampling/sampling_logp_difference/max": 0.19352221488952637,
      "sampling/sampling_logp_difference/mean": 0.0293723251670599,
      "step": 745,
      "step_time": 11.020930055001372
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 925.0,
      "completions/max_terminated_length": 854.0,
      "completions/mean_length": 444.625,
      "completions/mean_terminated_length": 376.7857360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.433227431960404,
      "epoch": 0.00746,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6223109364509583,
      "kl": 0.4400523453950882,
      "learning_rate": 1.2351224539989147e-05,
      "loss": 0.101,
      "num_tokens": 17509704.0,
      "reward": 0.2715453803539276,
      "reward_std": 0.6091394424438477,
      "rewards/rollout_reward_func/mean": 0.2715453803539276,
      "rewards/rollout_reward_func/std": 0.6211072206497192,
      "sampling/importance_sampling_ratio/max": 1.8517754077911377,
      "sampling/importance_sampling_ratio/mean": 0.9411547183990479,
      "sampling/importance_sampling_ratio/min": 1.112489093901825e-11,
      "sampling/sampling_logp_difference/max": 17.283639907836914,
      "sampling/sampling_logp_difference/mean": 0.17121385037899017,
      "step": 746,
      "step_time": 8.826772077001806
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 898.0,
      "completions/max_terminated_length": 898.0,
      "completions/mean_length": 231.875,
      "completions/mean_terminated_length": 231.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7005453575402498,
      "epoch": 0.00747,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.24883078038692474,
      "kl": 0.7954866699874401,
      "learning_rate": 1.2304580255116448e-05,
      "loss": 0.004,
      "num_tokens": 17527346.0,
      "reward": 0.47576555609703064,
      "reward_std": 0.699773371219635,
      "rewards/rollout_reward_func/mean": 0.47576555609703064,
      "rewards/rollout_reward_func/std": 0.7425278425216675,
      "sampling/importance_sampling_ratio/max": 1.1044284105300903,
      "sampling/importance_sampling_ratio/mean": 1.007746696472168,
      "sampling/importance_sampling_ratio/min": 0.9953226447105408,
      "sampling/sampling_logp_difference/max": 0.07866024971008301,
      "sampling/sampling_logp_difference/mean": 0.006233712192624807,
      "step": 747,
      "step_time": 7.365329735001069
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1678.0,
      "completions/max_terminated_length": 1678.0,
      "completions/mean_length": 550.9375,
      "completions/mean_terminated_length": 525.2000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3130786046385765,
      "epoch": 0.00748,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8183479905128479,
      "kl": 1.21758146956563,
      "learning_rate": 1.2258125119632705e-05,
      "loss": 0.0921,
      "num_tokens": 17551022.0,
      "reward": 0.35605254769325256,
      "reward_std": 0.7178950905799866,
      "rewards/rollout_reward_func/mean": 0.35605254769325256,
      "rewards/rollout_reward_func/std": 0.715105414390564,
      "sampling/importance_sampling_ratio/max": 1.226908564567566,
      "sampling/importance_sampling_ratio/mean": 0.9475939273834229,
      "sampling/importance_sampling_ratio/min": 3.827470784756315e-09,
      "sampling/sampling_logp_difference/max": 19.377355575561523,
      "sampling/sampling_logp_difference/mean": 0.21712824702262878,
      "step": 748,
      "step_time": 9.845192586000849
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 910.0,
      "completions/max_terminated_length": 910.0,
      "completions/mean_length": 474.1875,
      "completions/mean_terminated_length": 474.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9174310071393847,
      "epoch": 0.00749,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.19744867086410522,
      "kl": 0.5039901807904243,
      "learning_rate": 1.2211859687206987e-05,
      "loss": -0.006,
      "num_tokens": 17572988.0,
      "reward": 0.8032485842704773,
      "reward_std": 0.27180951833724976,
      "rewards/rollout_reward_func/mean": 0.8032485842704773,
      "rewards/rollout_reward_func/std": 0.42332255840301514,
      "sampling/importance_sampling_ratio/max": 1.071132779121399,
      "sampling/importance_sampling_ratio/mean": 0.9590617418289185,
      "sampling/importance_sampling_ratio/min": 1.4193701769471545e-09,
      "sampling/sampling_logp_difference/max": 20.02007293701172,
      "sampling/sampling_logp_difference/mean": 0.3502471446990967,
      "step": 749,
      "step_time": 7.659111125998606
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1823.0,
      "completions/max_terminated_length": 1816.0,
      "completions/mean_length": 591.375,
      "completions/mean_terminated_length": 450.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1982355881482363,
      "epoch": 0.0075,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2835159301757812,
      "kl": 0.6651489920914173,
      "learning_rate": 1.2165784509247401e-05,
      "loss": 0.1943,
      "num_tokens": 17596690.0,
      "reward": 0.3149935305118561,
      "reward_std": 0.8754026293754578,
      "rewards/rollout_reward_func/mean": 0.3149935305118561,
      "rewards/rollout_reward_func/std": 0.849023699760437,
      "sampling/importance_sampling_ratio/max": 1.905722975730896,
      "sampling/importance_sampling_ratio/mean": 0.8926113843917847,
      "sampling/importance_sampling_ratio/min": 3.2266555868940403e-18,
      "sampling/sampling_logp_difference/max": 21.16449737548828,
      "sampling/sampling_logp_difference/mean": 0.2772505581378937,
      "step": 750,
      "step_time": 11.255177262999496
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2687.0,
      "completions/max_terminated_length": 2687.0,
      "completions/mean_length": 907.4375,
      "completions/mean_terminated_length": 905.1428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2344273552298546,
      "epoch": 0.00751,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9568122625350952,
      "kl": 0.9456925317645073,
      "learning_rate": 1.2119900134894548e-05,
      "loss": 0.2053,
      "num_tokens": 17626053.0,
      "reward": 0.47586914896965027,
      "reward_std": 0.650198221206665,
      "rewards/rollout_reward_func/mean": 0.47586914896965027,
      "rewards/rollout_reward_func/std": 0.6454305052757263,
      "sampling/importance_sampling_ratio/max": 1.1783751249313354,
      "sampling/importance_sampling_ratio/mean": 0.9542602896690369,
      "sampling/importance_sampling_ratio/min": 0.7058330774307251,
      "sampling/sampling_logp_difference/max": 0.22318172454833984,
      "sampling/sampling_logp_difference/mean": 0.02799958921968937,
      "step": 751,
      "step_time": 13.411095282002861
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2601.0,
      "completions/max_terminated_length": 2601.0,
      "completions/mean_length": 346.875,
      "completions/mean_terminated_length": 346.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.94605971314013,
      "epoch": 0.00752,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.46977367997169495,
      "kl": 0.5245469771325588,
      "learning_rate": 1.2074207111014963e-05,
      "loss": 0.0208,
      "num_tokens": 17645670.0,
      "reward": 0.5410516262054443,
      "reward_std": 0.542356014251709,
      "rewards/rollout_reward_func/mean": 0.5410516262054443,
      "rewards/rollout_reward_func/std": 0.6480130553245544,
      "sampling/importance_sampling_ratio/max": 1.0574387311935425,
      "sampling/importance_sampling_ratio/mean": 0.9445570707321167,
      "sampling/importance_sampling_ratio/min": 1.2722375375417982e-09,
      "sampling/sampling_logp_difference/max": 19.52020263671875,
      "sampling/sampling_logp_difference/mean": 0.29536381363868713,
      "step": 752,
      "step_time": 11.75378493900098
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 995.0,
      "completions/max_terminated_length": 884.0,
      "completions/mean_length": 466.1875,
      "completions/mean_terminated_length": 407.0000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4586866348981857,
      "epoch": 0.00753,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9249142408370972,
      "kl": 0.6595425568521023,
      "learning_rate": 1.202870598219458e-05,
      "loss": 0.1062,
      "num_tokens": 17667976.0,
      "reward": 0.4097251892089844,
      "reward_std": 0.7499134540557861,
      "rewards/rollout_reward_func/mean": 0.4097251892089844,
      "rewards/rollout_reward_func/std": 0.7413457632064819,
      "sampling/importance_sampling_ratio/max": 1.2551409006118774,
      "sampling/importance_sampling_ratio/mean": 0.8820601105690002,
      "sampling/importance_sampling_ratio/min": 1.1845763259188402e-09,
      "sampling/sampling_logp_difference/max": 20.124723434448242,
      "sampling/sampling_logp_difference/mean": 0.24458499252796173,
      "step": 753,
      "step_time": 8.477802947998498
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 826.0,
      "completions/max_terminated_length": 826.0,
      "completions/mean_length": 173.375,
      "completions/mean_terminated_length": 173.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7214790694415569,
      "epoch": 0.00754,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5136967897415161,
      "kl": 1.7163413353264332,
      "learning_rate": 1.1983397290732278e-05,
      "loss": -0.0266,
      "num_tokens": 17684318.0,
      "reward": 0.7518415451049805,
      "reward_std": 0.7018980383872986,
      "rewards/rollout_reward_func/mean": 0.7518415451049805,
      "rewards/rollout_reward_func/std": 0.6781193017959595,
      "sampling/importance_sampling_ratio/max": 1.7689920663833618,
      "sampling/importance_sampling_ratio/mean": 1.0340709686279297,
      "sampling/importance_sampling_ratio/min": 0.8690214157104492,
      "sampling/sampling_logp_difference/max": 0.14054787158966064,
      "sampling/sampling_logp_difference/mean": 0.024354858323931694,
      "step": 754,
      "step_time": 7.233864550998987
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1804.0,
      "completions/max_terminated_length": 951.0,
      "completions/mean_length": 721.25,
      "completions/mean_terminated_length": 580.7142944335938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6483033746480942,
      "epoch": 0.00755,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1725960969924927,
      "kl": 0.6372070387005806,
      "learning_rate": 1.1938281576633395e-05,
      "loss": -0.0381,
      "num_tokens": 17711255.0,
      "reward": 0.3631216585636139,
      "reward_std": 0.8257102370262146,
      "rewards/rollout_reward_func/mean": 0.3631216585636139,
      "rewards/rollout_reward_func/std": 0.8011969923973083,
      "sampling/importance_sampling_ratio/max": 1.1890149116516113,
      "sampling/importance_sampling_ratio/mean": 0.8658469915390015,
      "sampling/importance_sampling_ratio/min": 0.534081220626831,
      "sampling/sampling_logp_difference/max": 0.21667003631591797,
      "sampling/sampling_logp_difference/mean": 0.031966742128133774,
      "step": 755,
      "step_time": 11.325240204998408
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 871.0,
      "completions/max_terminated_length": 871.0,
      "completions/mean_length": 385.3125,
      "completions/mean_terminated_length": 385.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0376734845340252,
      "epoch": 0.00756,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.19431179761886597,
      "kl": 0.7006364986300468,
      "learning_rate": 1.1893359377603276e-05,
      "loss": -0.0134,
      "num_tokens": 17732221.0,
      "reward": 0.7413300275802612,
      "reward_std": 0.5624690055847168,
      "rewards/rollout_reward_func/mean": 0.7413300275802612,
      "rewards/rollout_reward_func/std": 0.5827847719192505,
      "sampling/importance_sampling_ratio/max": 1.0682390928268433,
      "sampling/importance_sampling_ratio/mean": 0.915908932685852,
      "sampling/importance_sampling_ratio/min": 7.414613922840374e-10,
      "sampling/sampling_logp_difference/max": 20.520824432373047,
      "sampling/sampling_logp_difference/mean": 0.3735031485557556,
      "step": 756,
      "step_time": 7.180610282002817
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2609.0,
      "completions/max_terminated_length": 2609.0,
      "completions/mean_length": 909.6875,
      "completions/mean_terminated_length": 904.2667236328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4932313710451126,
      "epoch": 0.00757,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.681473970413208,
      "kl": 0.6423113793134689,
      "learning_rate": 1.1848631229040911e-05,
      "loss": 0.0119,
      "num_tokens": 17762568.0,
      "reward": 0.7293523550033569,
      "reward_std": 0.47520363330841064,
      "rewards/rollout_reward_func/mean": 0.7293523550033569,
      "rewards/rollout_reward_func/std": 0.4845912456512451,
      "sampling/importance_sampling_ratio/max": 1.0996289253234863,
      "sampling/importance_sampling_ratio/mean": 0.9663567543029785,
      "sampling/importance_sampling_ratio/min": 0.6512472033500671,
      "sampling/sampling_logp_difference/max": 0.24201345443725586,
      "sampling/sampling_logp_difference/mean": 0.03166501596570015,
      "step": 757,
      "step_time": 13.166974563000622
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1719.0,
      "completions/max_terminated_length": 1719.0,
      "completions/mean_length": 561.0625,
      "completions/mean_terminated_length": 561.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2475415458902717,
      "epoch": 0.00758,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0259679555892944,
      "kl": 1.605046870186925,
      "learning_rate": 1.1804097664032514e-05,
      "loss": 0.0058,
      "num_tokens": 17785996.0,
      "reward": 0.7412464618682861,
      "reward_std": 0.5970666408538818,
      "rewards/rollout_reward_func/mean": 0.7412464618682861,
      "rewards/rollout_reward_func/std": 0.5830405950546265,
      "sampling/importance_sampling_ratio/max": 1.2172211408615112,
      "sampling/importance_sampling_ratio/mean": 1.0296286344528198,
      "sampling/importance_sampling_ratio/min": 0.9001333713531494,
      "sampling/sampling_logp_difference/max": 0.43796634674072266,
      "sampling/sampling_logp_difference/mean": 0.027042042464017868,
      "step": 758,
      "step_time": 10.08297847299582
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 932.0,
      "completions/max_terminated_length": 932.0,
      "completions/mean_length": 534.75,
      "completions/mean_terminated_length": 510.0666809082031,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.069292152300477,
      "epoch": 0.00759,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5309575796127319,
      "kl": 0.5499103292822838,
      "learning_rate": 1.175975921334517e-05,
      "loss": 0.0773,
      "num_tokens": 17809252.0,
      "reward": 0.7441409826278687,
      "reward_std": 0.5578771829605103,
      "rewards/rollout_reward_func/mean": 0.7441409826278687,
      "rewards/rollout_reward_func/std": 0.5787772536277771,
      "sampling/importance_sampling_ratio/max": 1.0707719326019287,
      "sampling/importance_sampling_ratio/mean": 0.9349154233932495,
      "sampling/importance_sampling_ratio/min": 1.3502158592582081e-11,
      "sampling/sampling_logp_difference/max": 15.32494831085205,
      "sampling/sampling_logp_difference/mean": 0.16902156174182892,
      "step": 759,
      "step_time": 8.344077980000293
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2511.0,
      "completions/max_terminated_length": 2511.0,
      "completions/mean_length": 622.8125,
      "completions/mean_terminated_length": 533.1428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0459655299782753,
      "epoch": 0.0076,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.1790010929107666,
      "kl": 0.43973932694643736,
      "learning_rate": 1.171561640542054e-05,
      "loss": 0.0052,
      "num_tokens": 17833901.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/rollout_reward_func/mean": 1.0,
      "rewards/rollout_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.0905284881591797,
      "sampling/importance_sampling_ratio/mean": 0.955118715763092,
      "sampling/importance_sampling_ratio/min": 0.5079659223556519,
      "sampling/sampling_logp_difference/max": 0.2763094902038574,
      "sampling/sampling_logp_difference/mean": 0.02288873679935932,
      "step": 760,
      "step_time": 13.591258369999196
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1576.0,
      "completions/max_terminated_length": 1576.0,
      "completions/mean_length": 460.5625,
      "completions/mean_terminated_length": 460.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0788368582725525,
      "epoch": 0.00761,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.3379962146282196,
      "kl": 0.44239014480262995,
      "learning_rate": 1.1671669766368543e-05,
      "loss": -0.045,
      "num_tokens": 17856388.0,
      "reward": 0.7419393062591553,
      "reward_std": 0.38179007172584534,
      "rewards/rollout_reward_func/mean": 0.7419393062591553,
      "rewards/rollout_reward_func/std": 0.5857705473899841,
      "sampling/importance_sampling_ratio/max": 1.086158275604248,
      "sampling/importance_sampling_ratio/mean": 0.9441350698471069,
      "sampling/importance_sampling_ratio/min": 1.4271324609064725e-15,
      "sampling/sampling_logp_difference/max": 19.996288299560547,
      "sampling/sampling_logp_difference/mean": 0.3713934123516083,
      "step": 761,
      "step_time": 9.531842778000282
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 877.0,
      "completions/max_terminated_length": 860.0,
      "completions/mean_length": 376.3125,
      "completions/mean_terminated_length": 342.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3792107626795769,
      "epoch": 0.00762,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6009647846221924,
      "kl": 0.4590540947392583,
      "learning_rate": 1.1627919819961067e-05,
      "loss": -0.1244,
      "num_tokens": 17877572.0,
      "reward": 0.3507491648197174,
      "reward_std": 0.7280934453010559,
      "rewards/rollout_reward_func/mean": 0.3507491648197174,
      "rewards/rollout_reward_func/std": 0.7272289991378784,
      "sampling/importance_sampling_ratio/max": 1.0949815511703491,
      "sampling/importance_sampling_ratio/mean": 0.8695025444030762,
      "sampling/importance_sampling_ratio/min": 7.705835752477852e-13,
      "sampling/sampling_logp_difference/max": 19.539886474609375,
      "sampling/sampling_logp_difference/mean": 0.27713555097579956,
      "step": 762,
      "step_time": 7.783372008998413
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1733.0,
      "completions/max_terminated_length": 1733.0,
      "completions/mean_length": 601.5,
      "completions/mean_terminated_length": 600.3333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.241449773311615,
      "epoch": 0.00763,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0983632802963257,
      "kl": 0.6744500026106834,
      "learning_rate": 1.1584367087625765e-05,
      "loss": -0.1616,
      "num_tokens": 17901438.0,
      "reward": 0.4913095533847809,
      "reward_std": 0.6007736921310425,
      "rewards/rollout_reward_func/mean": 0.4913095533847809,
      "rewards/rollout_reward_func/std": 0.7257235646247864,
      "sampling/importance_sampling_ratio/max": 1.342429757118225,
      "sampling/importance_sampling_ratio/mean": 1.0083621740341187,
      "sampling/importance_sampling_ratio/min": 0.8092585206031799,
      "sampling/sampling_logp_difference/max": 0.1599712371826172,
      "sampling/sampling_logp_difference/mean": 0.026447080075740814,
      "step": 763,
      "step_time": 10.483578946001217
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1725.0,
      "completions/max_terminated_length": 1725.0,
      "completions/mean_length": 454.1875,
      "completions/mean_terminated_length": 454.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0648504048585892,
      "epoch": 0.00764,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0364596843719482,
      "kl": 0.5732341604307294,
      "learning_rate": 1.1541012088439823e-05,
      "loss": 0.066,
      "num_tokens": 17922556.0,
      "reward": 0.6201598644256592,
      "reward_std": 0.7255600690841675,
      "rewards/rollout_reward_func/mean": 0.6201598644256592,
      "rewards/rollout_reward_func/std": 0.7170162200927734,
      "sampling/importance_sampling_ratio/max": 1.3336695432662964,
      "sampling/importance_sampling_ratio/mean": 1.0366888046264648,
      "sampling/importance_sampling_ratio/min": 0.7806975245475769,
      "sampling/sampling_logp_difference/max": 0.19656658172607422,
      "sampling/sampling_logp_difference/mean": 0.024010509252548218,
      "step": 764,
      "step_time": 9.95501381199756
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 883.0,
      "completions/max_terminated_length": 883.0,
      "completions/mean_length": 465.8125,
      "completions/mean_terminated_length": 465.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4601557217538357,
      "epoch": 0.00765,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6929218769073486,
      "kl": 0.547133294865489,
      "learning_rate": 1.1497855339123753e-05,
      "loss": -0.0998,
      "num_tokens": 17944965.0,
      "reward": 0.536178708076477,
      "reward_std": 0.6678041815757751,
      "rewards/rollout_reward_func/mean": 0.536178708076477,
      "rewards/rollout_reward_func/std": 0.6550920605659485,
      "sampling/importance_sampling_ratio/max": 1.4717328548431396,
      "sampling/importance_sampling_ratio/mean": 1.047036051750183,
      "sampling/importance_sampling_ratio/min": 0.9220619797706604,
      "sampling/sampling_logp_difference/max": 0.15022170543670654,
      "sampling/sampling_logp_difference/mean": 0.022770844399929047,
      "step": 765,
      "step_time": 7.537215748001472
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1693.0,
      "completions/max_terminated_length": 1693.0,
      "completions/mean_length": 631.0,
      "completions/mean_terminated_length": 631.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0387753695249557,
      "epoch": 0.00766,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.36365771293640137,
      "kl": 0.4449761714786291,
      "learning_rate": 1.1454897354035272e-05,
      "loss": 0.0063,
      "num_tokens": 17969827.0,
      "reward": 0.875,
      "reward_std": 0.3535533845424652,
      "rewards/rollout_reward_func/mean": 0.875,
      "rewards/rollout_reward_func/std": 0.5,
      "sampling/importance_sampling_ratio/max": 1.0672754049301147,
      "sampling/importance_sampling_ratio/mean": 0.9928065538406372,
      "sampling/importance_sampling_ratio/min": 0.872882604598999,
      "sampling/sampling_logp_difference/max": 0.13175582885742188,
      "sampling/sampling_logp_difference/mean": 0.01641666702926159,
      "step": 766,
      "step_time": 10.278027699001541
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1710.0,
      "completions/max_terminated_length": 1710.0,
      "completions/mean_length": 482.5625,
      "completions/mean_terminated_length": 470.4667053222656,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.175311230123043,
      "epoch": 0.00767,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8083370923995972,
      "kl": 0.32899827696383,
      "learning_rate": 1.1412138645163142e-05,
      "loss": -0.0745,
      "num_tokens": 17991642.0,
      "reward": 0.6235032081604004,
      "reward_std": 0.7389266490936279,
      "rewards/rollout_reward_func/mean": 0.6235032081604004,
      "rewards/rollout_reward_func/std": 0.7142446637153625,
      "sampling/importance_sampling_ratio/max": 1.4838035106658936,
      "sampling/importance_sampling_ratio/mean": 1.0203726291656494,
      "sampling/importance_sampling_ratio/min": 0.7908853888511658,
      "sampling/sampling_logp_difference/max": 0.2537200450897217,
      "sampling/sampling_logp_difference/mean": 0.03492462635040283,
      "step": 767,
      "step_time": 9.983419091000542
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1687.0,
      "completions/max_terminated_length": 1687.0,
      "completions/mean_length": 391.4375,
      "completions/mean_terminated_length": 391.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4501814665272832,
      "epoch": 0.00768,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4538632929325104,
      "kl": 0.4714908958412707,
      "learning_rate": 1.136957972212109e-05,
      "loss": -0.0982,
      "num_tokens": 18012297.0,
      "reward": 0.5979105830192566,
      "reward_std": 0.6442286968231201,
      "rewards/rollout_reward_func/mean": 0.5979105830192566,
      "rewards/rollout_reward_func/std": 0.6482874751091003,
      "sampling/importance_sampling_ratio/max": 1.2632856369018555,
      "sampling/importance_sampling_ratio/mean": 1.0070483684539795,
      "sampling/importance_sampling_ratio/min": 0.7211505770683289,
      "sampling/sampling_logp_difference/max": 0.1986827850341797,
      "sampling/sampling_logp_difference/mean": 0.02486521378159523,
      "step": 768,
      "step_time": 9.69786530300189
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1668.0,
      "completions/max_terminated_length": 1668.0,
      "completions/mean_length": 579.5,
      "completions/mean_terminated_length": 579.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2188652902841568,
      "epoch": 0.00769,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5153340697288513,
      "kl": 0.46351793594658375,
      "learning_rate": 1.132722109214172e-05,
      "loss": -0.0271,
      "num_tokens": 18036786.0,
      "reward": 0.6211674213409424,
      "reward_std": 0.6337482333183289,
      "rewards/rollout_reward_func/mean": 0.6211674213409424,
      "rewards/rollout_reward_func/std": 0.7165546417236328,
      "sampling/importance_sampling_ratio/max": 1.079836368560791,
      "sampling/importance_sampling_ratio/mean": 0.988086998462677,
      "sampling/importance_sampling_ratio/min": 0.7794200778007507,
      "sampling/sampling_logp_difference/max": 0.1797013282775879,
      "sampling/sampling_logp_difference/mean": 0.027789676561951637,
      "step": 769,
      "step_time": 10.057829246998153
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1661.0,
      "completions/max_terminated_length": 1661.0,
      "completions/mean_length": 519.8125,
      "completions/mean_terminated_length": 519.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0896920189261436,
      "epoch": 0.0077,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7511692643165588,
      "kl": 0.6165959034115076,
      "learning_rate": 1.1285063260070454e-05,
      "loss": -0.0409,
      "num_tokens": 18059810.0,
      "reward": 0.7393866181373596,
      "reward_std": 0.5660543441772461,
      "rewards/rollout_reward_func/mean": 0.7393866181373596,
      "rewards/rollout_reward_func/std": 0.5897485017776489,
      "sampling/importance_sampling_ratio/max": 1.1987630128860474,
      "sampling/importance_sampling_ratio/mean": 1.0212581157684326,
      "sampling/importance_sampling_ratio/min": 0.8088760375976562,
      "sampling/sampling_logp_difference/max": 0.2016751766204834,
      "sampling/sampling_logp_difference/mean": 0.02580719254910946,
      "step": 770,
      "step_time": 9.902493616002175
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1700.0,
      "completions/max_terminated_length": 1700.0,
      "completions/mean_length": 617.375,
      "completions/mean_terminated_length": 617.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2341314414516091,
      "epoch": 0.00771,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.078872561454773,
      "kl": 0.4864620319567621,
      "learning_rate": 1.1243106728359554e-05,
      "loss": 0.0816,
      "num_tokens": 18084453.0,
      "reward": 0.746037483215332,
      "reward_std": 0.5858432650566101,
      "rewards/rollout_reward_func/mean": 0.746037483215332,
      "rewards/rollout_reward_func/std": 0.5719200372695923,
      "sampling/importance_sampling_ratio/max": 1.0918570756912231,
      "sampling/importance_sampling_ratio/mean": 0.8452653288841248,
      "sampling/importance_sampling_ratio/min": 9.600572290283105e-13,
      "sampling/sampling_logp_difference/max": 24.51651954650879,
      "sampling/sampling_logp_difference/mean": 0.410323828458786,
      "step": 771,
      "step_time": 10.321251612997003
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1527.0,
      "completions/max_terminated_length": 829.0,
      "completions/mean_length": 227.3125,
      "completions/mean_terminated_length": 140.6666717529297,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9253774359822273,
      "epoch": 0.00772,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6924529075622559,
      "kl": 0.7329768668860197,
      "learning_rate": 1.1201351997062115e-05,
      "loss": -0.1401,
      "num_tokens": 18102747.0,
      "reward": 0.5975286960601807,
      "reward_std": 0.5797138214111328,
      "rewards/rollout_reward_func/mean": 0.5975286960601807,
      "rewards/rollout_reward_func/std": 0.6472669839859009,
      "sampling/importance_sampling_ratio/max": 1.5817991495132446,
      "sampling/importance_sampling_ratio/mean": 0.8986973166465759,
      "sampling/importance_sampling_ratio/min": 1.514021463755455e-09,
      "sampling/sampling_logp_difference/max": 20.172626495361328,
      "sampling/sampling_logp_difference/mean": 0.4134727418422699,
      "step": 772,
      "step_time": 9.020000396998512
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1868.0,
      "completions/max_terminated_length": 1868.0,
      "completions/mean_length": 642.1875,
      "completions/mean_terminated_length": 642.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5680513381958008,
      "epoch": 0.00773,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9647583961486816,
      "kl": 0.8537516631186008,
      "learning_rate": 1.1159799563826072e-05,
      "loss": 0.1374,
      "num_tokens": 18127861.0,
      "reward": 0.5431058406829834,
      "reward_std": 0.6563330888748169,
      "rewards/rollout_reward_func/mean": 0.5431058406829834,
      "rewards/rollout_reward_func/std": 0.6444032192230225,
      "sampling/importance_sampling_ratio/max": 1.4112107753753662,
      "sampling/importance_sampling_ratio/mean": 0.9833142757415771,
      "sampling/importance_sampling_ratio/min": 0.6087625026702881,
      "sampling/sampling_logp_difference/max": 0.47165393829345703,
      "sampling/sampling_logp_difference/mean": 0.02824532799422741,
      "step": 773,
      "step_time": 11.265568363998682
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1825.0,
      "completions/max_terminated_length": 1825.0,
      "completions/mean_length": 758.0,
      "completions/mean_terminated_length": 737.0714721679688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5830428078770638,
      "epoch": 0.00774,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1141178607940674,
      "kl": 0.5774137564003468,
      "learning_rate": 1.1118449923888323e-05,
      "loss": -0.0178,
      "num_tokens": 18155490.0,
      "reward": 0.54345703125,
      "reward_std": 0.6561466455459595,
      "rewards/rollout_reward_func/mean": 0.54345703125,
      "rewards/rollout_reward_func/std": 0.640846848487854,
      "sampling/importance_sampling_ratio/max": 1.5611562728881836,
      "sampling/importance_sampling_ratio/mean": 1.019219160079956,
      "sampling/importance_sampling_ratio/min": 0.6163583397865295,
      "sampling/sampling_logp_difference/max": 0.17669034004211426,
      "sampling/sampling_logp_difference/mean": 0.02801942266523838,
      "step": 774,
      "step_time": 11.639104928999586
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1673.0,
      "completions/max_terminated_length": 1673.0,
      "completions/mean_length": 504.125,
      "completions/mean_terminated_length": 535.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3365510255098343,
      "epoch": 0.00775,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2500762641429901,
      "kl": 0.5879882294684649,
      "learning_rate": 1.1077303570068808e-05,
      "loss": -0.041,
      "num_tokens": 18178178.0,
      "reward": 0.5374444723129272,
      "reward_std": 0.5471700429916382,
      "rewards/rollout_reward_func/mean": 0.5374444723129272,
      "rewards/rollout_reward_func/std": 0.647605836391449,
      "sampling/importance_sampling_ratio/max": 1.1143274307250977,
      "sampling/importance_sampling_ratio/mean": 0.9412549734115601,
      "sampling/importance_sampling_ratio/min": 1.3498257416325044e-13,
      "sampling/sampling_logp_difference/max": 15.566362380981445,
      "sampling/sampling_logp_difference/mean": 0.2542265057563782,
      "step": 775,
      "step_time": 10.071138256998893
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1900.0,
      "completions/max_terminated_length": 1900.0,
      "completions/mean_length": 492.625,
      "completions/mean_terminated_length": 470.5333557128906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2100361976772547,
      "epoch": 0.00776,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.21411243081092834,
      "kl": 1.9855759907513857,
      "learning_rate": 1.1036360992764604e-05,
      "loss": -0.0204,
      "num_tokens": 18200988.0,
      "reward": 0.550023078918457,
      "reward_std": 0.638738214969635,
      "rewards/rollout_reward_func/mean": 0.550023078918457,
      "rewards/rollout_reward_func/std": 0.7349832057952881,
      "sampling/importance_sampling_ratio/max": 1.1098328828811646,
      "sampling/importance_sampling_ratio/mean": 0.8287668228149414,
      "sampling/importance_sampling_ratio/min": 9.012930175875738e-18,
      "sampling/sampling_logp_difference/max": 19.32984733581543,
      "sampling/sampling_logp_difference/mean": 0.36219245195388794,
      "step": 776,
      "step_time": 10.887349374001133
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1644.0,
      "completions/max_terminated_length": 1644.0,
      "completions/mean_length": 676.8125,
      "completions/mean_terminated_length": 665.2857666015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5140924379229546,
      "epoch": 0.00777,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 8.951486587524414,
      "kl": 6.305656157433987,
      "learning_rate": 1.0995622679944127e-05,
      "loss": 0.5018,
      "num_tokens": 18227079.0,
      "reward": 0.6039748191833496,
      "reward_std": 0.643397331237793,
      "rewards/rollout_reward_func/mean": 0.6039748191833496,
      "rewards/rollout_reward_func/std": 0.6425698399543762,
      "sampling/importance_sampling_ratio/max": 1.875885248184204,
      "sampling/importance_sampling_ratio/mean": 0.9379855990409851,
      "sampling/importance_sampling_ratio/min": 5.029616723106756e-10,
      "sampling/sampling_logp_difference/max": 10.993584632873535,
      "sampling/sampling_logp_difference/mean": 0.10140629857778549,
      "step": 777,
      "step_time": 11.189939857000354
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 879.0,
      "completions/max_terminated_length": 879.0,
      "completions/mean_length": 382.1875,
      "completions/mean_terminated_length": 382.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8766347877681255,
      "epoch": 0.00778,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.11639098823070526,
      "kl": 0.3997670477256179,
      "learning_rate": 1.095508911714129e-05,
      "loss": -0.0011,
      "num_tokens": 18247825.0,
      "reward": 0.3440532684326172,
      "reward_std": 0.6990350484848022,
      "rewards/rollout_reward_func/mean": 0.3440532684326172,
      "rewards/rollout_reward_func/std": 0.7305367588996887,
      "sampling/importance_sampling_ratio/max": 1.1562427282333374,
      "sampling/importance_sampling_ratio/mean": 1.021040678024292,
      "sampling/importance_sampling_ratio/min": 0.9338538646697998,
      "sampling/sampling_logp_difference/max": 0.10984015464782715,
      "sampling/sampling_logp_difference/mean": 0.01855529472231865,
      "step": 778,
      "step_time": 7.087990554002317
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1782.0,
      "completions/max_terminated_length": 1782.0,
      "completions/mean_length": 609.6875,
      "completions/mean_terminated_length": 608.4666748046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2483834773302078,
      "epoch": 0.00779,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9897423982620239,
      "kl": 0.6100182821974158,
      "learning_rate": 1.0914760787449712e-05,
      "loss": 0.0436,
      "num_tokens": 18273014.0,
      "reward": 0.6246968507766724,
      "reward_std": 0.7192336320877075,
      "rewards/rollout_reward_func/mean": 0.6246968507766724,
      "rewards/rollout_reward_func/std": 0.709377110004425,
      "sampling/importance_sampling_ratio/max": 1.2827117443084717,
      "sampling/importance_sampling_ratio/mean": 1.0055859088897705,
      "sampling/importance_sampling_ratio/min": 0.649367094039917,
      "sampling/sampling_logp_difference/max": 0.2630190849304199,
      "sampling/sampling_logp_difference/mean": 0.030202433466911316,
      "step": 779,
      "step_time": 10.757918397001049
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 961.0,
      "completions/max_terminated_length": 961.0,
      "completions/mean_length": 427.5,
      "completions/mean_terminated_length": 427.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9050486367195845,
      "epoch": 0.0078,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2900060713291168,
      "kl": 0.5987468138337135,
      "learning_rate": 1.0874638171516984e-05,
      "loss": 0.0134,
      "num_tokens": 18294282.0,
      "reward": 0.290595144033432,
      "reward_std": 0.7414313554763794,
      "rewards/rollout_reward_func/mean": 0.290595144033432,
      "rewards/rollout_reward_func/std": 0.7969918251037598,
      "sampling/importance_sampling_ratio/max": 1.074594497680664,
      "sampling/importance_sampling_ratio/mean": 0.9781032800674438,
      "sampling/importance_sampling_ratio/min": 0.7971331477165222,
      "sampling/sampling_logp_difference/max": 0.10892486572265625,
      "sampling/sampling_logp_difference/mean": 0.016184983775019646,
      "step": 780,
      "step_time": 8.24423339900386
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1758.0,
      "completions/max_terminated_length": 1758.0,
      "completions/mean_length": 452.75,
      "completions/mean_terminated_length": 452.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0673643983900547,
      "epoch": 0.00781,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4510493576526642,
      "kl": 0.6388772577047348,
      "learning_rate": 1.0834721747538928e-05,
      "loss": 0.036,
      "num_tokens": 18316400.0,
      "reward": 0.17789892852306366,
      "reward_std": 0.8486502170562744,
      "rewards/rollout_reward_func/mean": 0.17789892852306366,
      "rewards/rollout_reward_func/std": 0.8236864805221558,
      "sampling/importance_sampling_ratio/max": 1.0935420989990234,
      "sampling/importance_sampling_ratio/mean": 0.913424015045166,
      "sampling/importance_sampling_ratio/min": 3.240901591539114e-08,
      "sampling/sampling_logp_difference/max": 19.32783317565918,
      "sampling/sampling_logp_difference/mean": 0.26760992407798767,
      "step": 781,
      "step_time": 9.923354713002482
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 905.0,
      "completions/max_terminated_length": 905.0,
      "completions/mean_length": 341.75,
      "completions/mean_terminated_length": 341.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7991283601149917,
      "epoch": 0.00782,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3400386869907379,
      "kl": 0.6607560589909554,
      "learning_rate": 1.079501199125387e-05,
      "loss": -0.0054,
      "num_tokens": 18335792.0,
      "reward": 0.5403497219085693,
      "reward_std": 0.6570591330528259,
      "rewards/rollout_reward_func/mean": 0.5403497219085693,
      "rewards/rollout_reward_func/std": 0.6434723734855652,
      "sampling/importance_sampling_ratio/max": 1.099782109260559,
      "sampling/importance_sampling_ratio/mean": 0.9397333860397339,
      "sampling/importance_sampling_ratio/min": 1.1530884025390264e-09,
      "sampling/sampling_logp_difference/max": 20.578054428100586,
      "sampling/sampling_logp_difference/mean": 0.3304338753223419,
      "step": 782,
      "step_time": 7.215926331999071
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2541.0,
      "completions/max_terminated_length": 2541.0,
      "completions/mean_length": 823.5,
      "completions/mean_terminated_length": 761.800048828125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2743282318115234,
      "epoch": 0.00783,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.46485286951065063,
      "kl": 0.36274254508316517,
      "learning_rate": 1.0755509375937026e-05,
      "loss": -0.1259,
      "num_tokens": 18364303.0,
      "reward": 0.7928975224494934,
      "reward_std": 0.28588631749153137,
      "rewards/rollout_reward_func/mean": 0.7928975224494934,
      "rewards/rollout_reward_func/std": 0.44532617926597595,
      "sampling/importance_sampling_ratio/max": 1.0398144721984863,
      "sampling/importance_sampling_ratio/mean": 0.934585452079773,
      "sampling/importance_sampling_ratio/min": 0.6411113739013672,
      "sampling/sampling_logp_difference/max": 0.2279667854309082,
      "sampling/sampling_logp_difference/mean": 0.02997378446161747,
      "step": 783,
      "step_time": 13.427895172000717
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1623.0,
      "completions/max_terminated_length": 1623.0,
      "completions/mean_length": 436.75,
      "completions/mean_terminated_length": 436.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8659907300025225,
      "epoch": 0.00784,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3753396272659302,
      "kl": 0.832664180546999,
      "learning_rate": 1.0716214372394817e-05,
      "loss": -0.0254,
      "num_tokens": 18385746.0,
      "reward": 0.5996829867362976,
      "reward_std": 0.6432732343673706,
      "rewards/rollout_reward_func/mean": 0.5996829867362976,
      "rewards/rollout_reward_func/std": 0.646748423576355,
      "sampling/importance_sampling_ratio/max": 1.1182260513305664,
      "sampling/importance_sampling_ratio/mean": 1.0131070613861084,
      "sampling/importance_sampling_ratio/min": 0.9523528218269348,
      "sampling/sampling_logp_difference/max": 0.09661865234375,
      "sampling/sampling_logp_difference/mean": 0.01658991165459156,
      "step": 784,
      "step_time": 9.440319851999448
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 904.0,
      "completions/max_terminated_length": 904.0,
      "completions/mean_length": 442.8125,
      "completions/mean_terminated_length": 442.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9228840889409184,
      "epoch": 0.00785,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6460962891578674,
      "kl": 0.4981723763048649,
      "learning_rate": 1.0677127448959268e-05,
      "loss": -0.0007,
      "num_tokens": 18407390.0,
      "reward": 0.3928380012512207,
      "reward_std": 0.5678026676177979,
      "rewards/rollout_reward_func/mean": 0.3928380012512207,
      "rewards/rollout_reward_func/std": 0.5534406304359436,
      "sampling/importance_sampling_ratio/max": 1.309564471244812,
      "sampling/importance_sampling_ratio/mean": 1.0033607482910156,
      "sampling/importance_sampling_ratio/min": 0.7504433989524841,
      "sampling/sampling_logp_difference/max": 0.1423635482788086,
      "sampling/sampling_logp_difference/mean": 0.020574912428855896,
      "step": 785,
      "step_time": 7.836398951001684
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1786.0,
      "completions/max_terminated_length": 1738.0,
      "completions/mean_length": 620.875,
      "completions/mean_terminated_length": 543.2000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.08172214217484,
      "epoch": 0.00786,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6567062735557556,
      "kl": 0.900400921702385,
      "learning_rate": 1.0638249071482444e-05,
      "loss": -0.0724,
      "num_tokens": 18431861.0,
      "reward": 0.5441919565200806,
      "reward_std": 0.6513712406158447,
      "rewards/rollout_reward_func/mean": 0.5441919565200806,
      "rewards/rollout_reward_func/std": 0.6377096772193909,
      "sampling/importance_sampling_ratio/max": 1.1749457120895386,
      "sampling/importance_sampling_ratio/mean": 1.006902813911438,
      "sampling/importance_sampling_ratio/min": 0.6398711800575256,
      "sampling/sampling_logp_difference/max": 0.4001162052154541,
      "sampling/sampling_logp_difference/mean": 0.026360254734754562,
      "step": 786,
      "step_time": 11.040982605001773
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1728.0,
      "completions/max_terminated_length": 1728.0,
      "completions/mean_length": 575.6875,
      "completions/mean_terminated_length": 575.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0204183254390955,
      "epoch": 0.00787,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.384077250957489,
      "kl": 0.6521561183035374,
      "learning_rate": 1.0599579703330876e-05,
      "loss": -0.063,
      "num_tokens": 18456068.0,
      "reward": 0.4129524230957031,
      "reward_std": 0.7603150010108948,
      "rewards/rollout_reward_func/mean": 0.4129524230957031,
      "rewards/rollout_reward_func/std": 0.7379654049873352,
      "sampling/importance_sampling_ratio/max": 1.1004271507263184,
      "sampling/importance_sampling_ratio/mean": 0.9834367036819458,
      "sampling/importance_sampling_ratio/min": 0.8577218651771545,
      "sampling/sampling_logp_difference/max": 0.12458276748657227,
      "sampling/sampling_logp_difference/mean": 0.014630110934376717,
      "step": 787,
      "step_time": 9.917131021000387
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1760.0,
      "completions/max_terminated_length": 1760.0,
      "completions/mean_length": 553.3125,
      "completions/mean_terminated_length": 537.6666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.585887011140585,
      "epoch": 0.00788,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6507855653762817,
      "kl": 0.654875423759222,
      "learning_rate": 1.0561119805380036e-05,
      "loss": -0.0338,
      "num_tokens": 18480191.0,
      "reward": 0.16123749315738678,
      "reward_std": 0.7461072206497192,
      "rewards/rollout_reward_func/mean": 0.16123749315738678,
      "rewards/rollout_reward_func/std": 0.7343952059745789,
      "sampling/importance_sampling_ratio/max": 1.121432900428772,
      "sampling/importance_sampling_ratio/mean": 0.9797847270965576,
      "sampling/importance_sampling_ratio/min": 0.6212912201881409,
      "sampling/sampling_logp_difference/max": 0.22100090980529785,
      "sampling/sampling_logp_difference/mean": 0.028645431622862816,
      "step": 788,
      "step_time": 10.387487683001382
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1745.0,
      "completions/max_terminated_length": 1745.0,
      "completions/mean_length": 540.1875,
      "completions/mean_terminated_length": 484.69232177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.333840899169445,
      "epoch": 0.00789,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2878203392028809,
      "kl": 0.4396614246070385,
      "learning_rate": 1.052286983600887e-05,
      "loss": 0.1939,
      "num_tokens": 18503409.0,
      "reward": 0.6361831426620483,
      "reward_std": 0.7891682386398315,
      "rewards/rollout_reward_func/mean": 0.6361831426620483,
      "rewards/rollout_reward_func/std": 0.7824442982673645,
      "sampling/importance_sampling_ratio/max": 1.2960368394851685,
      "sampling/importance_sampling_ratio/mean": 1.0206698179244995,
      "sampling/importance_sampling_ratio/min": 0.7861971259117126,
      "sampling/sampling_logp_difference/max": 0.2002544403076172,
      "sampling/sampling_logp_difference/mean": 0.02232862450182438,
      "step": 789,
      "step_time": 10.716769833996295
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1695.0,
      "completions/max_terminated_length": 1695.0,
      "completions/mean_length": 565.4375,
      "completions/mean_terminated_length": 550.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0784967094659805,
      "epoch": 0.0079,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7718249559402466,
      "kl": 0.6057357247918844,
      "learning_rate": 1.0484830251094325e-05,
      "loss": -0.0306,
      "num_tokens": 18527181.0,
      "reward": 0.4174778461456299,
      "reward_std": 0.7431453466415405,
      "rewards/rollout_reward_func/mean": 0.4174778461456299,
      "rewards/rollout_reward_func/std": 0.7363526225090027,
      "sampling/importance_sampling_ratio/max": 1.9841102361679077,
      "sampling/importance_sampling_ratio/mean": 1.0148849487304688,
      "sampling/importance_sampling_ratio/min": 5.889702059391766e-10,
      "sampling/sampling_logp_difference/max": 21.035289764404297,
      "sampling/sampling_logp_difference/mean": 0.2071477621793747,
      "step": 790,
      "step_time": 9.93346297199787
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1653.0,
      "completions/max_terminated_length": 1653.0,
      "completions/mean_length": 483.125,
      "completions/mean_terminated_length": 513.2000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0313977003097534,
      "epoch": 0.00791,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.18182280659675598,
      "kl": 0.8306414913386106,
      "learning_rate": 1.0447001504005886e-05,
      "loss": -0.0113,
      "num_tokens": 18549222.0,
      "reward": 0.8604916334152222,
      "reward_std": 0.25832146406173706,
      "rewards/rollout_reward_func/mean": 0.8604916334152222,
      "rewards/rollout_reward_func/std": 0.3812122046947479,
      "sampling/importance_sampling_ratio/max": 1.448712944984436,
      "sampling/importance_sampling_ratio/mean": 1.0529201030731201,
      "sampling/importance_sampling_ratio/min": 0.8715413808822632,
      "sampling/sampling_logp_difference/max": 0.2350301742553711,
      "sampling/sampling_logp_difference/mean": 0.023046493530273438,
      "step": 791,
      "step_time": 10.12049878599646
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2664.0,
      "completions/max_terminated_length": 2664.0,
      "completions/mean_length": 557.1875,
      "completions/mean_terminated_length": 557.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8564425799995661,
      "epoch": 0.00792,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6016361713409424,
      "kl": 0.6657482590526342,
      "learning_rate": 1.0409384045600217e-05,
      "loss": 0.0548,
      "num_tokens": 18572651.0,
      "reward": 0.6248248219490051,
      "reward_std": 0.6328957676887512,
      "rewards/rollout_reward_func/mean": 0.6248248219490051,
      "rewards/rollout_reward_func/std": 0.7032572627067566,
      "sampling/importance_sampling_ratio/max": 1.1722145080566406,
      "sampling/importance_sampling_ratio/mean": 1.0185887813568115,
      "sampling/importance_sampling_ratio/min": 0.9424392580986023,
      "sampling/sampling_logp_difference/max": 0.36236095428466797,
      "sampling/sampling_logp_difference/mean": 0.027266057208180428,
      "step": 792,
      "step_time": 12.130519616002857
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1806.0,
      "completions/max_terminated_length": 1708.0,
      "completions/mean_length": 613.3125,
      "completions/mean_terminated_length": 443.91668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5379416961222887,
      "epoch": 0.00793,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.51056969165802,
      "kl": 0.6146212927997112,
      "learning_rate": 1.0371978324215758e-05,
      "loss": -0.036,
      "num_tokens": 18597014.0,
      "reward": 0.617790937423706,
      "reward_std": 0.7433221936225891,
      "rewards/rollout_reward_func/mean": 0.617790937423706,
      "rewards/rollout_reward_func/std": 0.7182427644729614,
      "sampling/importance_sampling_ratio/max": 1.4613673686981201,
      "sampling/importance_sampling_ratio/mean": 0.9111241698265076,
      "sampling/importance_sampling_ratio/min": 7.791502668875596e-19,
      "sampling/sampling_logp_difference/max": 23.959341049194336,
      "sampling/sampling_logp_difference/mean": 0.17783191800117493,
      "step": 793,
      "step_time": 11.578541545999542
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1744.0,
      "completions/max_terminated_length": 1744.0,
      "completions/mean_length": 406.5,
      "completions/mean_terminated_length": 406.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9375648330897093,
      "epoch": 0.00794,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.345596581697464,
      "kl": 0.6861499957740307,
      "learning_rate": 1.0334784785667393e-05,
      "loss": 0.0896,
      "num_tokens": 18618019.0,
      "reward": 0.67728590965271,
      "reward_std": 0.3775545060634613,
      "rewards/rollout_reward_func/mean": 0.67728590965271,
      "rewards/rollout_reward_func/std": 0.6141465306282043,
      "sampling/importance_sampling_ratio/max": 1.5775970220565796,
      "sampling/importance_sampling_ratio/mean": 0.9805713295936584,
      "sampling/importance_sampling_ratio/min": 8.457570199738029e-10,
      "sampling/sampling_logp_difference/max": 21.0222225189209,
      "sampling/sampling_logp_difference/mean": 0.16379141807556152,
      "step": 794,
      "step_time": 10.049497413998324
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1504.0,
      "completions/max_terminated_length": 1504.0,
      "completions/mean_length": 454.125,
      "completions/mean_terminated_length": 454.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1900217719376087,
      "epoch": 0.00795,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3054828345775604,
      "kl": 0.7439716327935457,
      "learning_rate": 1.0297803873241136e-05,
      "loss": -0.0251,
      "num_tokens": 18640469.0,
      "reward": 0.6615378260612488,
      "reward_std": 0.5307610034942627,
      "rewards/rollout_reward_func/mean": 0.6615378260612488,
      "rewards/rollout_reward_func/std": 0.5188701748847961,
      "sampling/importance_sampling_ratio/max": 1.1782723665237427,
      "sampling/importance_sampling_ratio/mean": 0.8835752010345459,
      "sampling/importance_sampling_ratio/min": 1.1827152382730844e-14,
      "sampling/sampling_logp_difference/max": 22.922910690307617,
      "sampling/sampling_logp_difference/mean": 0.47861799597740173,
      "step": 795,
      "step_time": 9.4346403010004
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1826.0,
      "completions/max_terminated_length": 1826.0,
      "completions/mean_length": 559.0,
      "completions/mean_terminated_length": 594.1333618164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3746932819485664,
      "epoch": 0.00796,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.48393118381500244,
      "kl": 0.7461394481360912,
      "learning_rate": 1.0261036027688845e-05,
      "loss": 0.0294,
      "num_tokens": 18664004.0,
      "reward": 0.7992275953292847,
      "reward_std": 0.2770940661430359,
      "rewards/rollout_reward_func/mean": 0.7992275953292847,
      "rewards/rollout_reward_func/std": 0.4316498935222626,
      "sampling/importance_sampling_ratio/max": 1.3035911321640015,
      "sampling/importance_sampling_ratio/mean": 0.9085742235183716,
      "sampling/importance_sampling_ratio/min": 3.9349867114735204e-11,
      "sampling/sampling_logp_difference/max": 22.770553588867188,
      "sampling/sampling_logp_difference/mean": 0.15652352571487427,
      "step": 796,
      "step_time": 10.70772068000042
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1687.0,
      "completions/max_terminated_length": 1687.0,
      "completions/mean_length": 495.6875,
      "completions/mean_terminated_length": 495.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1163543853908777,
      "epoch": 0.00797,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.032760500907898,
      "kl": 0.8480892330408096,
      "learning_rate": 1.0224481687222972e-05,
      "loss": -0.1759,
      "num_tokens": 18686574.0,
      "reward": 0.39841631054878235,
      "reward_std": 0.6730618476867676,
      "rewards/rollout_reward_func/mean": 0.39841631054878235,
      "rewards/rollout_reward_func/std": 0.6592654585838318,
      "sampling/importance_sampling_ratio/max": 1.2126784324645996,
      "sampling/importance_sampling_ratio/mean": 1.0183727741241455,
      "sampling/importance_sampling_ratio/min": 0.7327173948287964,
      "sampling/sampling_logp_difference/max": 0.22387659549713135,
      "sampling/sampling_logp_difference/mean": 0.024127475917339325,
      "step": 797,
      "step_time": 10.22481133400288
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2742.0,
      "completions/max_terminated_length": 913.0,
      "completions/mean_length": 425.4375,
      "completions/mean_terminated_length": 234.21429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.334334559738636,
      "epoch": 0.00798,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.172724962234497,
      "kl": 0.8280151374638081,
      "learning_rate": 1.0188141287511336e-05,
      "loss": 0.2451,
      "num_tokens": 18707754.0,
      "reward": 0.39060986042022705,
      "reward_std": 0.5301703214645386,
      "rewards/rollout_reward_func/mean": 0.39060986042022705,
      "rewards/rollout_reward_func/std": 0.5562459230422974,
      "sampling/importance_sampling_ratio/max": 1.22664475440979,
      "sampling/importance_sampling_ratio/mean": 0.9595726728439331,
      "sampling/importance_sampling_ratio/min": 0.52729731798172,
      "sampling/sampling_logp_difference/max": 0.5352630615234375,
      "sampling/sampling_logp_difference/mean": 0.032923340797424316,
      "step": 798,
      "step_time": 13.023616378997758
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1774.0,
      "completions/max_terminated_length": 1495.0,
      "completions/mean_length": 785.4375,
      "completions/mean_terminated_length": 719.5333862304688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3924375995993614,
      "epoch": 0.00799,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8883136510848999,
      "kl": 1.0203832425177097,
      "learning_rate": 1.0152015261671928e-05,
      "loss": -0.0963,
      "num_tokens": 18735912.0,
      "reward": 0.5361395478248596,
      "reward_std": 0.6338866949081421,
      "rewards/rollout_reward_func/mean": 0.5361395478248596,
      "rewards/rollout_reward_func/std": 0.65098637342453,
      "sampling/importance_sampling_ratio/max": 1.2885440587997437,
      "sampling/importance_sampling_ratio/mean": 0.9258767366409302,
      "sampling/importance_sampling_ratio/min": 2.933586465653053e-14,
      "sampling/sampling_logp_difference/max": 21.812103271484375,
      "sampling/sampling_logp_difference/mean": 0.16870269179344177,
      "step": 799,
      "step_time": 11.931749560004391
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1745.0,
      "completions/max_terminated_length": 1745.0,
      "completions/mean_length": 552.625,
      "completions/mean_terminated_length": 552.625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0840539131313562,
      "epoch": 0.008,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.8647434115409851,
      "kl": 0.9163031838834286,
      "learning_rate": 1.0116104040267764e-05,
      "loss": 0.0871,
      "num_tokens": 18759548.0,
      "reward": 0.5514715313911438,
      "reward_std": 0.4170602858066559,
      "rewards/rollout_reward_func/mean": 0.5514715313911438,
      "rewards/rollout_reward_func/std": 0.7343543767929077,
      "sampling/importance_sampling_ratio/max": 1.3332043886184692,
      "sampling/importance_sampling_ratio/mean": 0.902840793132782,
      "sampling/importance_sampling_ratio/min": 6.349403065541992e-06,
      "sampling/sampling_logp_difference/max": 11.658879280090332,
      "sampling/sampling_logp_difference/mean": 0.09741794317960739,
      "step": 800,
      "step_time": 10.337609752001299
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1788.0,
      "completions/max_terminated_length": 1788.0,
      "completions/mean_length": 776.9375,
      "completions/mean_terminated_length": 776.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0608799308538437,
      "epoch": 0.00801,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8929210901260376,
      "kl": 0.9312379024922848,
      "learning_rate": 1.0080408051301737e-05,
      "loss": 0.2762,
      "num_tokens": 18787128.0,
      "reward": 0.6849550008773804,
      "reward_std": 0.5422785878181458,
      "rewards/rollout_reward_func/mean": 0.6849550008773804,
      "rewards/rollout_reward_func/std": 0.5940957069396973,
      "sampling/importance_sampling_ratio/max": 1.286210298538208,
      "sampling/importance_sampling_ratio/mean": 0.996160626411438,
      "sampling/importance_sampling_ratio/min": 0.5684462189674377,
      "sampling/sampling_logp_difference/max": 0.17206192016601562,
      "sampling/sampling_logp_difference/mean": 0.022188302129507065,
      "step": 801,
      "step_time": 10.759810235998884
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 850.0,
      "completions/max_terminated_length": 850.0,
      "completions/mean_length": 366.875,
      "completions/mean_terminated_length": 366.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3419770952314138,
      "epoch": 0.00802,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7499772906303406,
      "kl": 0.759564958512783,
      "learning_rate": 1.0044927720211525e-05,
      "loss": 0.0016,
      "num_tokens": 18807246.0,
      "reward": 0.7394393086433411,
      "reward_std": 0.6028248071670532,
      "rewards/rollout_reward_func/mean": 0.7394393086433411,
      "rewards/rollout_reward_func/std": 0.5895853042602539,
      "sampling/importance_sampling_ratio/max": 1.3136202096939087,
      "sampling/importance_sampling_ratio/mean": 1.061023473739624,
      "sampling/importance_sampling_ratio/min": 0.9960272312164307,
      "sampling/sampling_logp_difference/max": 0.3390159606933594,
      "sampling/sampling_logp_difference/mean": 0.023645438253879547,
      "step": 802,
      "step_time": 7.7424213260001125
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1769.0,
      "completions/max_terminated_length": 1769.0,
      "completions/mean_length": 497.875,
      "completions/mean_terminated_length": 497.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8362635234370828,
      "epoch": 0.00803,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.5341147780418396,
      "kl": 0.601101697422564,
      "learning_rate": 1.0009663469864511e-05,
      "loss": -0.0689,
      "num_tokens": 18829315.0,
      "reward": 0.8007647395133972,
      "reward_std": 0.27534595131874084,
      "rewards/rollout_reward_func/mean": 0.8007647395133972,
      "rewards/rollout_reward_func/std": 0.4287925660610199,
      "sampling/importance_sampling_ratio/max": 1.0594313144683838,
      "sampling/importance_sampling_ratio/mean": 0.9927406907081604,
      "sampling/importance_sampling_ratio/min": 0.8524503707885742,
      "sampling/sampling_logp_difference/max": 0.20050978660583496,
      "sampling/sampling_logp_difference/mean": 0.021686537191271782,
      "step": 803,
      "step_time": 10.165524954994908
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1821.0,
      "completions/max_terminated_length": 1814.0,
      "completions/mean_length": 673.625,
      "completions/mean_terminated_length": 586.857177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3154522851109505,
      "epoch": 0.00804,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1040987968444824,
      "kl": 0.8714579660445452,
      "learning_rate": 9.974615720552763e-06,
      "loss": 0.1611,
      "num_tokens": 18855207.0,
      "reward": 0.5351758003234863,
      "reward_std": 0.559561014175415,
      "rewards/rollout_reward_func/mean": 0.5351758003234863,
      "rewards/rollout_reward_func/std": 0.5451896786689758,
      "sampling/importance_sampling_ratio/max": 1.5436676740646362,
      "sampling/importance_sampling_ratio/mean": 1.0130006074905396,
      "sampling/importance_sampling_ratio/min": 0.3887864947319031,
      "sampling/sampling_logp_difference/max": 0.2210068702697754,
      "sampling/sampling_logp_difference/mean": 0.02872476913034916,
      "step": 804,
      "step_time": 11.030571246999898
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1814.0,
      "completions/max_terminated_length": 956.0,
      "completions/mean_length": 531.375,
      "completions/mean_terminated_length": 473.14288330078125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.041836703196168,
      "epoch": 0.00805,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.49006494879722595,
      "kl": 0.8113394677639008,
      "learning_rate": 9.939784889987996e-06,
      "loss": 0.0365,
      "num_tokens": 18878294.0,
      "reward": 0.8674747943878174,
      "reward_std": 0.24544604122638702,
      "rewards/rollout_reward_func/mean": 0.8674747943878174,
      "rewards/rollout_reward_func/std": 0.36219996213912964,
      "sampling/importance_sampling_ratio/max": 1.1150754690170288,
      "sampling/importance_sampling_ratio/mean": 0.8919885158538818,
      "sampling/importance_sampling_ratio/min": 2.3928401974123497e-25,
      "sampling/sampling_logp_difference/max": 12.12380313873291,
      "sampling/sampling_logp_difference/mean": 0.31316688656806946,
      "step": 805,
      "step_time": 10.82163163199948
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1750.0,
      "completions/max_terminated_length": 1750.0,
      "completions/mean_length": 461.6875,
      "completions/mean_terminated_length": 520.7857666015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.199145445600152,
      "epoch": 0.00806,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9199261665344238,
      "kl": 0.7867299914360046,
      "learning_rate": 9.90517139329662e-06,
      "loss": 0.0998,
      "num_tokens": 18900235.0,
      "reward": 0.5915499329566956,
      "reward_std": 0.5641579627990723,
      "rewards/rollout_reward_func/mean": 0.5915499329566956,
      "rewards/rollout_reward_func/std": 0.5450581312179565,
      "sampling/importance_sampling_ratio/max": 1.467569351196289,
      "sampling/importance_sampling_ratio/mean": 0.8668947815895081,
      "sampling/importance_sampling_ratio/min": 3.755028163305403e-18,
      "sampling/sampling_logp_difference/max": 22.946992874145508,
      "sampling/sampling_logp_difference/mean": 0.33916187286376953,
      "step": 806,
      "step_time": 11.168252316998405
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2823.0,
      "completions/max_terminated_length": 937.0,
      "completions/mean_length": 730.9375,
      "completions/mean_terminated_length": 550.3077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4646447971463203,
      "epoch": 0.00807,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9735249280929565,
      "kl": 0.8349845968186855,
      "learning_rate": 9.870775643014778e-06,
      "loss": -0.268,
      "num_tokens": 18926912.0,
      "reward": 0.8015672564506531,
      "reward_std": 0.43219614028930664,
      "rewards/rollout_reward_func/mean": 0.8015672564506531,
      "rewards/rollout_reward_func/std": 0.4266863167285919,
      "sampling/importance_sampling_ratio/max": 1.5788898468017578,
      "sampling/importance_sampling_ratio/mean": 0.8292158246040344,
      "sampling/importance_sampling_ratio/min": 2.360158459440908e-18,
      "sampling/sampling_logp_difference/max": 22.513845443725586,
      "sampling/sampling_logp_difference/mean": 0.2512848377227783,
      "step": 807,
      "step_time": 14.927041187000214
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 911.0,
      "completions/max_terminated_length": 911.0,
      "completions/mean_length": 493.9375,
      "completions/mean_terminated_length": 475.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0723144467920065,
      "epoch": 0.00808,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8658473491668701,
      "kl": 0.4516723155975342,
      "learning_rate": 9.83659804908343e-06,
      "loss": -0.0252,
      "num_tokens": 18949436.0,
      "reward": 0.5258784890174866,
      "reward_std": 0.5329201817512512,
      "rewards/rollout_reward_func/mean": 0.5258784890174866,
      "rewards/rollout_reward_func/std": 0.5554165244102478,
      "sampling/importance_sampling_ratio/max": 1.822676181793213,
      "sampling/importance_sampling_ratio/mean": 1.0749199390411377,
      "sampling/importance_sampling_ratio/min": 0.4920676052570343,
      "sampling/sampling_logp_difference/max": 0.44754528999328613,
      "sampling/sampling_logp_difference/mean": 0.030049430206418037,
      "step": 808,
      "step_time": 8.09601287300029
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1820.0,
      "completions/max_terminated_length": 1820.0,
      "completions/mean_length": 637.9375,
      "completions/mean_terminated_length": 598.7692260742188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0589366778731346,
      "epoch": 0.00809,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7305939197540283,
      "kl": 0.7406642772257328,
      "learning_rate": 9.802639018843474e-06,
      "loss": -0.1568,
      "num_tokens": 18974886.0,
      "reward": 0.5379519462585449,
      "reward_std": 0.667236328125,
      "rewards/rollout_reward_func/mean": 0.5379519462585449,
      "rewards/rollout_reward_func/std": 0.6535242199897766,
      "sampling/importance_sampling_ratio/max": 1.219960331916809,
      "sampling/importance_sampling_ratio/mean": 0.9549218416213989,
      "sampling/importance_sampling_ratio/min": 0.47508400678634644,
      "sampling/sampling_logp_difference/max": 0.2778511047363281,
      "sampling/sampling_logp_difference/mean": 0.02513580024242401,
      "step": 809,
      "step_time": 11.385197961999438
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1600.0,
      "completions/max_terminated_length": 1600.0,
      "completions/mean_length": 536.6875,
      "completions/mean_terminated_length": 523.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1166133098304272,
      "epoch": 0.0081,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.6200278401374817,
      "kl": 0.8952498165890574,
      "learning_rate": 9.768898957030892e-06,
      "loss": 0.0235,
      "num_tokens": 18997983.0,
      "reward": 0.9325000047683716,
      "reward_std": 0.19091883301734924,
      "rewards/rollout_reward_func/mean": 0.9325000047683716,
      "rewards/rollout_reward_func/std": 0.27000001072883606,
      "sampling/importance_sampling_ratio/max": 1.141046404838562,
      "sampling/importance_sampling_ratio/mean": 0.9007001519203186,
      "sampling/importance_sampling_ratio/min": 5.903905275772559e-07,
      "sampling/sampling_logp_difference/max": 14.065266609191895,
      "sampling/sampling_logp_difference/mean": 0.10117751359939575,
      "step": 810,
      "step_time": 10.230438996000885
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1633.0,
      "completions/max_terminated_length": 1633.0,
      "completions/mean_length": 559.875,
      "completions/mean_terminated_length": 559.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1036676988005638,
      "epoch": 0.00811,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8022118210792542,
      "kl": 0.8497129008173943,
      "learning_rate": 9.735378265771905e-06,
      "loss": -0.0167,
      "num_tokens": 19021585.0,
      "reward": 0.7287209033966064,
      "reward_std": 0.5026803016662598,
      "rewards/rollout_reward_func/mean": 0.7287209033966064,
      "rewards/rollout_reward_func/std": 0.48568108677864075,
      "sampling/importance_sampling_ratio/max": 1.2999517917633057,
      "sampling/importance_sampling_ratio/mean": 1.0232300758361816,
      "sampling/importance_sampling_ratio/min": 0.8258807063102722,
      "sampling/sampling_logp_difference/max": 0.18195629119873047,
      "sampling/sampling_logp_difference/mean": 0.024819498881697655,
      "step": 811,
      "step_time": 10.113043157998618
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1681.0,
      "completions/max_terminated_length": 1681.0,
      "completions/mean_length": 431.625,
      "completions/mean_terminated_length": 405.4000244140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.064774263650179,
      "epoch": 0.00812,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5431745052337646,
      "kl": 0.450981667265296,
      "learning_rate": 9.702077344578207e-06,
      "loss": -0.1197,
      "num_tokens": 19043174.0,
      "reward": 0.4151538610458374,
      "reward_std": 0.6629163026809692,
      "rewards/rollout_reward_func/mean": 0.4151538610458374,
      "rewards/rollout_reward_func/std": 0.7347016930580139,
      "sampling/importance_sampling_ratio/max": 1.0576740503311157,
      "sampling/importance_sampling_ratio/mean": 0.9518200159072876,
      "sampling/importance_sampling_ratio/min": 0.6606349349021912,
      "sampling/sampling_logp_difference/max": 0.19762253761291504,
      "sampling/sampling_logp_difference/mean": 0.028942067176103592,
      "step": 812,
      "step_time": 10.590836293000393
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 924.0,
      "completions/max_terminated_length": 924.0,
      "completions/mean_length": 273.6875,
      "completions/mean_terminated_length": 289.8000183105469,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9137021452188492,
      "epoch": 0.00813,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.125824213027954,
      "kl": 0.8404285795986652,
      "learning_rate": 9.668996590342204e-06,
      "loss": 0.1212,
      "num_tokens": 19060833.0,
      "reward": 0.550174355506897,
      "reward_std": 0.7524987459182739,
      "rewards/rollout_reward_func/mean": 0.550174355506897,
      "rewards/rollout_reward_func/std": 0.7307266592979431,
      "sampling/importance_sampling_ratio/max": 1.34541916847229,
      "sampling/importance_sampling_ratio/mean": 0.9750956296920776,
      "sampling/importance_sampling_ratio/min": 0.6613489389419556,
      "sampling/sampling_logp_difference/max": 0.23618102073669434,
      "sampling/sampling_logp_difference/mean": 0.02888326346874237,
      "step": 813,
      "step_time": 7.479845257998022
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1820.0,
      "completions/max_terminated_length": 1820.0,
      "completions/mean_length": 585.9375,
      "completions/mean_terminated_length": 611.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.086360728368163,
      "epoch": 0.00814,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.30299049615859985,
      "kl": 1.017904531210661,
      "learning_rate": 9.636136397332247e-06,
      "loss": -0.0098,
      "num_tokens": 19084632.0,
      "reward": 0.8084148168563843,
      "reward_std": 0.37354955077171326,
      "rewards/rollout_reward_func/mean": 0.8084148168563843,
      "rewards/rollout_reward_func/std": 0.547380268573761,
      "sampling/importance_sampling_ratio/max": 1.1879652738571167,
      "sampling/importance_sampling_ratio/mean": 0.8927789926528931,
      "sampling/importance_sampling_ratio/min": 3.0121444227759753e-10,
      "sampling/sampling_logp_difference/max": 21.606277465820312,
      "sampling/sampling_logp_difference/mean": 0.222880020737648,
      "step": 814,
      "step_time": 10.619940137998128
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1753.0,
      "completions/max_terminated_length": 1753.0,
      "completions/mean_length": 715.375,
      "completions/mean_terminated_length": 677.6154174804688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3452595323324203,
      "epoch": 0.00815,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0344761610031128,
      "kl": 0.8576147593557835,
      "learning_rate": 9.60349715718798e-06,
      "loss": -0.1768,
      "num_tokens": 19111458.0,
      "reward": 0.687095046043396,
      "reward_std": 0.6036283373832703,
      "rewards/rollout_reward_func/mean": 0.687095046043396,
      "rewards/rollout_reward_func/std": 0.5950733423233032,
      "sampling/importance_sampling_ratio/max": 1.5080281496047974,
      "sampling/importance_sampling_ratio/mean": 0.957912802696228,
      "sampling/importance_sampling_ratio/min": 1.00651140379418e-27,
      "sampling/sampling_logp_difference/max": 14.115520477294922,
      "sampling/sampling_logp_difference/mean": 0.2512778639793396,
      "step": 815,
      "step_time": 11.475406899000518
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1753.0,
      "completions/max_terminated_length": 1753.0,
      "completions/mean_length": 531.3125,
      "completions/mean_terminated_length": 531.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0636624954640865,
      "epoch": 0.00816,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9443448781967163,
      "kl": 1.0696731247007847,
      "learning_rate": 9.57107925891565e-06,
      "loss": -0.0193,
      "num_tokens": 19133914.0,
      "reward": 0.6745638847351074,
      "reward_std": 0.5711893439292908,
      "rewards/rollout_reward_func/mean": 0.6745638847351074,
      "rewards/rollout_reward_func/std": 0.6137216091156006,
      "sampling/importance_sampling_ratio/max": 1.381074070930481,
      "sampling/importance_sampling_ratio/mean": 0.9826905727386475,
      "sampling/importance_sampling_ratio/min": 0.6111536026000977,
      "sampling/sampling_logp_difference/max": 0.26768970489501953,
      "sampling/sampling_logp_difference/mean": 0.02505345456302166,
      "step": 816,
      "step_time": 10.218649365000601
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 869.0,
      "completions/max_terminated_length": 869.0,
      "completions/mean_length": 473.125,
      "completions/mean_terminated_length": 473.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0282628200948238,
      "epoch": 0.00817,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6095431447029114,
      "kl": 1.2097649648785591,
      "learning_rate": 9.538883088883453e-06,
      "loss": 0.0116,
      "num_tokens": 19156332.0,
      "reward": 0.7292901277542114,
      "reward_std": 0.4712466895580292,
      "rewards/rollout_reward_func/mean": 0.7292901277542114,
      "rewards/rollout_reward_func/std": 0.48427656292915344,
      "sampling/importance_sampling_ratio/max": 1.413695216178894,
      "sampling/importance_sampling_ratio/mean": 0.9785386323928833,
      "sampling/importance_sampling_ratio/min": 6.815896779032023e-16,
      "sampling/sampling_logp_difference/max": 20.53183364868164,
      "sampling/sampling_logp_difference/mean": 0.2404121607542038,
      "step": 817,
      "step_time": 8.126033827000356
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1646.0,
      "completions/max_terminated_length": 1646.0,
      "completions/mean_length": 813.5625,
      "completions/mean_terminated_length": 813.4667358398438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6296167708933353,
      "epoch": 0.00818,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9984757900238037,
      "kl": 1.0780409909784794,
      "learning_rate": 9.506909030816966e-06,
      "loss": -0.0263,
      "num_tokens": 19184621.0,
      "reward": 0.6823630332946777,
      "reward_std": 0.5570809245109558,
      "rewards/rollout_reward_func/mean": 0.6823630332946777,
      "rewards/rollout_reward_func/std": 0.6000373959541321,
      "sampling/importance_sampling_ratio/max": 1.223429799079895,
      "sampling/importance_sampling_ratio/mean": 0.9315541982650757,
      "sampling/importance_sampling_ratio/min": 0.6541190147399902,
      "sampling/sampling_logp_difference/max": 0.2582693099975586,
      "sampling/sampling_logp_difference/mean": 0.0313548669219017,
      "step": 818,
      "step_time": 11.602596333998008
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2580.0,
      "completions/max_terminated_length": 2580.0,
      "completions/mean_length": 653.0,
      "completions/mean_terminated_length": 652.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1251396760344505,
      "epoch": 0.00819,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3050739765167236,
      "kl": 0.8827858082950115,
      "learning_rate": 9.475157465794559e-06,
      "loss": -0.1822,
      "num_tokens": 19210038.0,
      "reward": 0.4789104163646698,
      "reward_std": 0.6622974872589111,
      "rewards/rollout_reward_func/mean": 0.4789104163646698,
      "rewards/rollout_reward_func/std": 0.6463294625282288,
      "sampling/importance_sampling_ratio/max": 1.7162734270095825,
      "sampling/importance_sampling_ratio/mean": 0.9412834048271179,
      "sampling/importance_sampling_ratio/min": 6.434649257869296e-09,
      "sampling/sampling_logp_difference/max": 12.91025161743164,
      "sampling/sampling_logp_difference/mean": 0.10406047850847244,
      "step": 819,
      "step_time": 13.454904493002687
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 915.0,
      "completions/max_terminated_length": 915.0,
      "completions/mean_length": 390.5,
      "completions/mean_terminated_length": 370.3333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5351883172988892,
      "epoch": 0.0082,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6908355355262756,
      "kl": 0.834239911288023,
      "learning_rate": 9.443628772242827e-06,
      "loss": -0.0954,
      "num_tokens": 19230695.0,
      "reward": 0.42510488629341125,
      "reward_std": 0.8367767333984375,
      "rewards/rollout_reward_func/mean": 0.42510488629341125,
      "rewards/rollout_reward_func/std": 0.813878059387207,
      "sampling/importance_sampling_ratio/max": 1.1467736959457397,
      "sampling/importance_sampling_ratio/mean": 0.9803488254547119,
      "sampling/importance_sampling_ratio/min": 0.6546306610107422,
      "sampling/sampling_logp_difference/max": 0.2366471290588379,
      "sampling/sampling_logp_difference/mean": 0.03597789630293846,
      "step": 820,
      "step_time": 8.169084659000873
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 960.0,
      "completions/max_terminated_length": 960.0,
      "completions/mean_length": 373.8125,
      "completions/mean_terminated_length": 320.7692565917969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5964206214994192,
      "epoch": 0.00821,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0501189231872559,
      "kl": 0.816307570785284,
      "learning_rate": 9.412323325932121e-06,
      "loss": 0.0878,
      "num_tokens": 19250452.0,
      "reward": 0.6085258722305298,
      "reward_std": 0.6287703514099121,
      "rewards/rollout_reward_func/mean": 0.6085258722305298,
      "rewards/rollout_reward_func/std": 0.6301141381263733,
      "sampling/importance_sampling_ratio/max": 1.878570795059204,
      "sampling/importance_sampling_ratio/mean": 0.8508895635604858,
      "sampling/importance_sampling_ratio/min": 6.311179120303026e-24,
      "sampling/sampling_logp_difference/max": 20.288555145263672,
      "sampling/sampling_logp_difference/mean": 0.4080566465854645,
      "step": 821,
      "step_time": 9.169463145997724
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 920.0,
      "completions/max_terminated_length": 775.0,
      "completions/mean_length": 429.6875,
      "completions/mean_terminated_length": 328.69232177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5301375351846218,
      "epoch": 0.00822,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9929345846176147,
      "kl": 0.8466002121567726,
      "learning_rate": 9.381241499972052e-06,
      "loss": -0.0154,
      "num_tokens": 19271864.0,
      "reward": 0.6586923003196716,
      "reward_std": 0.5357671976089478,
      "rewards/rollout_reward_func/mean": 0.6586923003196716,
      "rewards/rollout_reward_func/std": 0.5230023264884949,
      "sampling/importance_sampling_ratio/max": 1.4073126316070557,
      "sampling/importance_sampling_ratio/mean": 0.9727127552032471,
      "sampling/importance_sampling_ratio/min": 0.6521593332290649,
      "sampling/sampling_logp_difference/max": 0.36335039138793945,
      "sampling/sampling_logp_difference/mean": 0.030383802950382233,
      "step": 822,
      "step_time": 8.5915530939983
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1810.0,
      "completions/max_terminated_length": 1751.0,
      "completions/mean_length": 729.1875,
      "completions/mean_terminated_length": 657.1333618164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7668047919869423,
      "epoch": 0.00823,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0589438676834106,
      "kl": 1.1768169924616814,
      "learning_rate": 9.350383664807024e-06,
      "loss": 0.145,
      "num_tokens": 19298460.0,
      "reward": 0.4990040063858032,
      "reward_std": 0.6691460609436035,
      "rewards/rollout_reward_func/mean": 0.4990040063858032,
      "rewards/rollout_reward_func/std": 0.7112597823143005,
      "sampling/importance_sampling_ratio/max": 1.1037815809249878,
      "sampling/importance_sampling_ratio/mean": 0.8248094320297241,
      "sampling/importance_sampling_ratio/min": 4.0957870233414295e-13,
      "sampling/sampling_logp_difference/max": 18.085391998291016,
      "sampling/sampling_logp_difference/mean": 0.15273456275463104,
      "step": 823,
      "step_time": 11.374424707999424
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1760.0,
      "completions/max_terminated_length": 1760.0,
      "completions/mean_length": 591.875,
      "completions/mean_terminated_length": 585.86669921875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7278697192668915,
      "epoch": 0.00824,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.5875260829925537,
      "kl": 0.8124999441206455,
      "learning_rate": 9.319750188211863e-06,
      "loss": -0.0031,
      "num_tokens": 19322520.0,
      "reward": 0.8626118898391724,
      "reward_std": 0.254454642534256,
      "rewards/rollout_reward_func/mean": 0.8626118898391724,
      "rewards/rollout_reward_func/std": 0.3754933178424835,
      "sampling/importance_sampling_ratio/max": 1.6555304527282715,
      "sampling/importance_sampling_ratio/mean": 0.9406616687774658,
      "sampling/importance_sampling_ratio/min": 3.5196107841888556e-15,
      "sampling/sampling_logp_difference/max": 14.035287857055664,
      "sampling/sampling_logp_difference/mean": 0.15046291053295135,
      "step": 824,
      "step_time": 11.28103633900173
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1645.0,
      "completions/max_terminated_length": 812.0,
      "completions/mean_length": 566.75,
      "completions/mean_terminated_length": 525.6428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4588134959340096,
      "epoch": 0.00825,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.55904221534729,
      "kl": 1.167261067777872,
      "learning_rate": 9.28934143528739e-06,
      "loss": -0.3106,
      "num_tokens": 19346666.0,
      "reward": 0.5328047275543213,
      "reward_std": 0.5625065565109253,
      "rewards/rollout_reward_func/mean": 0.5328047275543213,
      "rewards/rollout_reward_func/std": 0.5474912524223328,
      "sampling/importance_sampling_ratio/max": 1.6108300685882568,
      "sampling/importance_sampling_ratio/mean": 1.013755440711975,
      "sampling/importance_sampling_ratio/min": 1.1951637532736138e-16,
      "sampling/sampling_logp_difference/max": 10.763114929199219,
      "sampling/sampling_logp_difference/mean": 0.1695265918970108,
      "step": 825,
      "step_time": 11.489121474003696
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1784.0,
      "completions/max_terminated_length": 1665.0,
      "completions/mean_length": 679.25,
      "completions/mean_terminated_length": 591.2142944335938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.380052849650383,
      "epoch": 0.00826,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4801688194274902,
      "kl": 1.332910604774952,
      "learning_rate": 9.259157768456096e-06,
      "loss": 0.149,
      "num_tokens": 19372008.0,
      "reward": 0.5430288314819336,
      "reward_std": 0.622005045413971,
      "rewards/rollout_reward_func/mean": 0.5430288314819336,
      "rewards/rollout_reward_func/std": 0.6426093578338623,
      "sampling/importance_sampling_ratio/max": 1.3482846021652222,
      "sampling/importance_sampling_ratio/mean": 0.9946094751358032,
      "sampling/importance_sampling_ratio/min": 0.6365143656730652,
      "sampling/sampling_logp_difference/max": 0.2698502540588379,
      "sampling/sampling_logp_difference/mean": 0.030697882175445557,
      "step": 826,
      "step_time": 11.661463844000536
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 901.0,
      "completions/max_terminated_length": 885.0,
      "completions/mean_length": 503.5,
      "completions/mean_terminated_length": 431.18182373046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6534472033381462,
      "epoch": 0.00827,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.1564977169036865,
      "kl": 1.089133094996214,
      "learning_rate": 9.229199547457818e-06,
      "loss": 0.2325,
      "num_tokens": 19394803.0,
      "reward": 0.9325977563858032,
      "reward_std": 0.1906423419713974,
      "rewards/rollout_reward_func/mean": 0.9325977563858032,
      "rewards/rollout_reward_func/std": 0.2696090042591095,
      "sampling/importance_sampling_ratio/max": 1.697505235671997,
      "sampling/importance_sampling_ratio/mean": 0.9607182145118713,
      "sampling/importance_sampling_ratio/min": 2.145080891877637e-35,
      "sampling/sampling_logp_difference/max": 21.324905395507812,
      "sampling/sampling_logp_difference/mean": 0.2640230655670166,
      "step": 827,
      "step_time": 9.763424163998934
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2555.0,
      "completions/max_terminated_length": 1635.0,
      "completions/mean_length": 646.4375,
      "completions/mean_terminated_length": 429.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4569910895079374,
      "epoch": 0.00828,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.039236545562744,
      "kl": 0.9958012737333775,
      "learning_rate": 9.199467129345433e-06,
      "loss": 0.4194,
      "num_tokens": 19419629.0,
      "reward": 0.6815075874328613,
      "reward_std": 0.6111575365066528,
      "rewards/rollout_reward_func/mean": 0.6815075874328613,
      "rewards/rollout_reward_func/std": 0.6026984453201294,
      "sampling/importance_sampling_ratio/max": 1.309332251548767,
      "sampling/importance_sampling_ratio/mean": 1.0127289295196533,
      "sampling/importance_sampling_ratio/min": 0.6843210458755493,
      "sampling/sampling_logp_difference/max": 0.42806005477905273,
      "sampling/sampling_logp_difference/mean": 0.030694426968693733,
      "step": 828,
      "step_time": 13.713665061002757
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1816.0,
      "completions/max_terminated_length": 1720.0,
      "completions/mean_length": 570.1875,
      "completions/mean_terminated_length": 424.0000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4881047289818525,
      "epoch": 0.00829,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7003542184829712,
      "kl": 0.7544571086764336,
      "learning_rate": 9.169960868480637e-06,
      "loss": 0.5123,
      "num_tokens": 19443356.0,
      "reward": 0.38051170110702515,
      "reward_std": 0.8468402624130249,
      "rewards/rollout_reward_func/mean": 0.38051170110702515,
      "rewards/rollout_reward_func/std": 0.8676110506057739,
      "sampling/importance_sampling_ratio/max": 1.2118524312973022,
      "sampling/importance_sampling_ratio/mean": 0.8572760820388794,
      "sampling/importance_sampling_ratio/min": 3.04914322957206e-11,
      "sampling/sampling_logp_difference/max": 18.066123962402344,
      "sampling/sampling_logp_difference/mean": 0.13308291137218475,
      "step": 829,
      "step_time": 11.940435783000794
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1711.0,
      "completions/max_terminated_length": 918.0,
      "completions/mean_length": 643.375,
      "completions/mean_terminated_length": 540.923095703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3150411285459995,
      "epoch": 0.0083,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2813528776168823,
      "kl": 0.5340385101735592,
      "learning_rate": 9.140681116529694e-06,
      "loss": 0.1611,
      "num_tokens": 19468402.0,
      "reward": 0.606705904006958,
      "reward_std": 0.45818936824798584,
      "rewards/rollout_reward_func/mean": 0.606705904006958,
      "rewards/rollout_reward_func/std": 0.5248610377311707,
      "sampling/importance_sampling_ratio/max": 1.4137672185897827,
      "sampling/importance_sampling_ratio/mean": 0.985194206237793,
      "sampling/importance_sampling_ratio/min": 0.5519278049468994,
      "sampling/sampling_logp_difference/max": 0.31224751472473145,
      "sampling/sampling_logp_difference/mean": 0.028305964544415474,
      "step": 830,
      "step_time": 11.681810775997292
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1853.0,
      "completions/max_terminated_length": 1853.0,
      "completions/mean_length": 694.6875,
      "completions/mean_terminated_length": 650.5385131835938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5094962418079376,
      "epoch": 0.00831,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9624791145324707,
      "kl": 1.0659482181072235,
      "learning_rate": 9.111628222459245e-06,
      "loss": -0.1701,
      "num_tokens": 19494869.0,
      "reward": 0.6018824577331543,
      "reward_std": 0.6581567525863647,
      "rewards/rollout_reward_func/mean": 0.6018824577331543,
      "rewards/rollout_reward_func/std": 0.6421928405761719,
      "sampling/importance_sampling_ratio/max": 1.2821587324142456,
      "sampling/importance_sampling_ratio/mean": 0.8516614437103271,
      "sampling/importance_sampling_ratio/min": 1.7073431427330754e-12,
      "sampling/sampling_logp_difference/max": 19.930204391479492,
      "sampling/sampling_logp_difference/mean": 0.09408341348171234,
      "step": 831,
      "step_time": 12.267872003001685
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1668.0,
      "completions/max_terminated_length": 1610.0,
      "completions/mean_length": 611.6875,
      "completions/mean_terminated_length": 564.0769653320312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.664977416396141,
      "epoch": 0.00832,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3981138467788696,
      "kl": 1.3944795690476894,
      "learning_rate": 9.08280253253217e-06,
      "loss": -0.1673,
      "num_tokens": 19519877.0,
      "reward": 0.3489047586917877,
      "reward_std": 0.7523813843727112,
      "rewards/rollout_reward_func/mean": 0.3489047586917877,
      "rewards/rollout_reward_func/std": 0.7270240783691406,
      "sampling/importance_sampling_ratio/max": 1.4080395698547363,
      "sampling/importance_sampling_ratio/mean": 1.038267731666565,
      "sampling/importance_sampling_ratio/min": 0.7171053290367126,
      "sampling/sampling_logp_difference/max": 0.2115006446838379,
      "sampling/sampling_logp_difference/mean": 0.03192594647407532,
      "step": 832,
      "step_time": 11.86233180599811
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1938.0,
      "completions/max_terminated_length": 1938.0,
      "completions/mean_length": 708.875,
      "completions/mean_terminated_length": 691.7857666015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3941868394613266,
      "epoch": 0.00833,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.194218873977661,
      "kl": 0.4818591848015785,
      "learning_rate": 9.054204390303444e-06,
      "loss": 0.488,
      "num_tokens": 19545342.0,
      "reward": 0.4523615539073944,
      "reward_std": 0.8108676671981812,
      "rewards/rollout_reward_func/mean": 0.4523615539073944,
      "rewards/rollout_reward_func/std": 0.7900545597076416,
      "sampling/importance_sampling_ratio/max": 1.4962162971496582,
      "sampling/importance_sampling_ratio/mean": 0.9882126450538635,
      "sampling/importance_sampling_ratio/min": 4.542001540500107e-13,
      "sampling/sampling_logp_difference/max": 4.005954265594482,
      "sampling/sampling_logp_difference/mean": 0.12000299245119095,
      "step": 833,
      "step_time": 12.67064888099776
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1837.0,
      "completions/max_terminated_length": 1686.0,
      "completions/mean_length": 567.1875,
      "completions/mean_terminated_length": 500.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8314615935087204,
      "epoch": 0.00834,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8779469728469849,
      "kl": 0.9084946922957897,
      "learning_rate": 9.025834136616039e-06,
      "loss": -0.083,
      "num_tokens": 19569351.0,
      "reward": 0.7309854030609131,
      "reward_std": 0.49825090169906616,
      "rewards/rollout_reward_func/mean": 0.7309854030609131,
      "rewards/rollout_reward_func/std": 0.4813746511936188,
      "sampling/importance_sampling_ratio/max": 1.2836970090866089,
      "sampling/importance_sampling_ratio/mean": 0.7467170357704163,
      "sampling/importance_sampling_ratio/min": 3.936655528819644e-21,
      "sampling/sampling_logp_difference/max": 21.237323760986328,
      "sampling/sampling_logp_difference/mean": 0.26079699397087097,
      "step": 834,
      "step_time": 11.937724594998144
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1745.0,
      "completions/max_terminated_length": 1653.0,
      "completions/mean_length": 395.5625,
      "completions/mean_terminated_length": 299.0769348144531,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4481808468699455,
      "epoch": 0.00835,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5537081956863403,
      "kl": 0.6599366180598736,
      "learning_rate": 8.99769210959688e-06,
      "loss": 0.0663,
      "num_tokens": 19590181.0,
      "reward": 0.23363828659057617,
      "reward_std": 0.7884466648101807,
      "rewards/rollout_reward_func/mean": 0.23363828659057617,
      "rewards/rollout_reward_func/std": 0.7727001309394836,
      "sampling/importance_sampling_ratio/max": 1.3601166009902954,
      "sampling/importance_sampling_ratio/mean": 0.9171949028968811,
      "sampling/importance_sampling_ratio/min": 3.600579657359049e-05,
      "sampling/sampling_logp_difference/max": 4.965185165405273,
      "sampling/sampling_logp_difference/mean": 0.05462665483355522,
      "step": 835,
      "step_time": 11.578052087998003
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2518.0,
      "completions/max_terminated_length": 899.0,
      "completions/mean_length": 703.5,
      "completions/mean_terminated_length": 519.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7343329153954983,
      "epoch": 0.00836,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4744032621383667,
      "kl": 0.994485754519701,
      "learning_rate": 8.9697786446528e-06,
      "loss": -0.0248,
      "num_tokens": 19616401.0,
      "reward": 0.6905595660209656,
      "reward_std": 0.6409731507301331,
      "rewards/rollout_reward_func/mean": 0.6905595660209656,
      "rewards/rollout_reward_func/std": 0.6866020560264587,
      "sampling/importance_sampling_ratio/max": 2.59269118309021,
      "sampling/importance_sampling_ratio/mean": 1.0614553689956665,
      "sampling/importance_sampling_ratio/min": 0.5840990543365479,
      "sampling/sampling_logp_difference/max": 0.8644165992736816,
      "sampling/sampling_logp_difference/mean": 0.034912608563899994,
      "step": 836,
      "step_time": 13.648256481001226
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2738.0,
      "completions/max_terminated_length": 2738.0,
      "completions/mean_length": 916.6875,
      "completions/mean_terminated_length": 876.84619140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6768267527222633,
      "epoch": 0.00837,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.9078121185302734,
      "kl": 0.8948068208992481,
      "learning_rate": 8.942094074466541e-06,
      "loss": 0.6077,
      "num_tokens": 19646349.0,
      "reward": 0.8106357455253601,
      "reward_std": 0.4083813428878784,
      "rewards/rollout_reward_func/mean": 0.8106357455253601,
      "rewards/rollout_reward_func/std": 0.40789297223091125,
      "sampling/importance_sampling_ratio/max": 1.5912286043167114,
      "sampling/importance_sampling_ratio/mean": 0.9725443124771118,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.514399528503418,
      "sampling/sampling_logp_difference/mean": 0.029271146282553673,
      "step": 837,
      "step_time": 15.158612071001698
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1838.0,
      "completions/max_terminated_length": 1721.0,
      "completions/mean_length": 788.8125,
      "completions/mean_terminated_length": 687.8181762695312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4966382831335068,
      "epoch": 0.00838,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.5084989070892334,
      "kl": 1.1756614558398724,
      "learning_rate": 8.914638728992804e-06,
      "loss": 0.1448,
      "num_tokens": 19674021.0,
      "reward": 0.6862966418266296,
      "reward_std": 0.35817617177963257,
      "rewards/rollout_reward_func/mean": 0.6862966418266296,
      "rewards/rollout_reward_func/std": 0.5868947505950928,
      "sampling/importance_sampling_ratio/max": 1.640445590019226,
      "sampling/importance_sampling_ratio/mean": 0.9055992364883423,
      "sampling/importance_sampling_ratio/min": 1.481923091184445e-12,
      "sampling/sampling_logp_difference/max": 23.826921463012695,
      "sampling/sampling_logp_difference/mean": 0.11050599813461304,
      "step": 838,
      "step_time": 12.4268471280011
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1682.0,
      "completions/max_terminated_length": 1682.0,
      "completions/mean_length": 513.25,
      "completions/mean_terminated_length": 385.6153869628906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.343616545200348,
      "epoch": 0.00839,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8461141586303711,
      "kl": 1.0054912958294153,
      "learning_rate": 8.887412935454305e-06,
      "loss": 0.0284,
      "num_tokens": 19696503.0,
      "reward": 0.37417638301849365,
      "reward_std": 0.855149507522583,
      "rewards/rollout_reward_func/mean": 0.37417638301849365,
      "rewards/rollout_reward_func/std": 0.8699767589569092,
      "sampling/importance_sampling_ratio/max": 1.2897473573684692,
      "sampling/importance_sampling_ratio/mean": 0.8737258911132812,
      "sampling/importance_sampling_ratio/min": 7.296006680757028e-12,
      "sampling/sampling_logp_difference/max": 19.0920467376709,
      "sampling/sampling_logp_difference/mean": 0.12794698774814606,
      "step": 839,
      "step_time": 11.492352427998412
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1873.0,
      "completions/max_terminated_length": 805.0,
      "completions/mean_length": 581.9375,
      "completions/mean_terminated_length": 390.4545593261719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.794245295226574,
      "epoch": 0.0084,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.008689522743225,
      "kl": 0.46628272719681263,
      "learning_rate": 8.860417018337867e-06,
      "loss": -0.0867,
      "num_tokens": 19720796.0,
      "reward": 0.5409502387046814,
      "reward_std": 0.6638298034667969,
      "rewards/rollout_reward_func/mean": 0.5409502387046814,
      "rewards/rollout_reward_func/std": 0.6500688791275024,
      "sampling/importance_sampling_ratio/max": 1.3893746137619019,
      "sampling/importance_sampling_ratio/mean": 0.916649341583252,
      "sampling/importance_sampling_ratio/min": 2.1393077044416396e-16,
      "sampling/sampling_logp_difference/max": 15.725371360778809,
      "sampling/sampling_logp_difference/mean": 0.12579858303070068,
      "step": 840,
      "step_time": 12.48290561199974
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2470.0,
      "completions/max_terminated_length": 2470.0,
      "completions/mean_length": 595.3125,
      "completions/mean_terminated_length": 520.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6965140625834465,
      "epoch": 0.00841,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8562083840370178,
      "kl": 1.0028074942529202,
      "learning_rate": 8.833651299390577e-06,
      "loss": -0.0524,
      "num_tokens": 19744789.0,
      "reward": 0.5434600710868835,
      "reward_std": 0.6266568303108215,
      "rewards/rollout_reward_func/mean": 0.5434600710868835,
      "rewards/rollout_reward_func/std": 0.6487337350845337,
      "sampling/importance_sampling_ratio/max": 1.300250768661499,
      "sampling/importance_sampling_ratio/mean": 0.9672842025756836,
      "sampling/importance_sampling_ratio/min": 0.5986707806587219,
      "sampling/sampling_logp_difference/max": 0.3060324192047119,
      "sampling/sampling_logp_difference/mean": 0.03391437977552414,
      "step": 841,
      "step_time": 13.102914807997877
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1569.0,
      "completions/max_terminated_length": 703.0,
      "completions/mean_length": 365.8125,
      "completions/mean_terminated_length": 204.69232177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4248633719980717,
      "epoch": 0.00842,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.6080428957939148,
      "kl": 0.5857376400381327,
      "learning_rate": 8.807116097615934e-06,
      "loss": -0.1789,
      "num_tokens": 19764254.0,
      "reward": 0.8081870675086975,
      "reward_std": 0.37267574667930603,
      "rewards/rollout_reward_func/mean": 0.8081870675086975,
      "rewards/rollout_reward_func/std": 0.5463526248931885,
      "sampling/importance_sampling_ratio/max": 1.722246527671814,
      "sampling/importance_sampling_ratio/mean": 0.9362661838531494,
      "sampling/importance_sampling_ratio/min": 1.061198134066617e-15,
      "sampling/sampling_logp_difference/max": 12.578826904296875,
      "sampling/sampling_logp_difference/mean": 0.16376550495624542,
      "step": 842,
      "step_time": 10.419932542999959
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2682.0,
      "completions/max_terminated_length": 2682.0,
      "completions/mean_length": 930.25,
      "completions/mean_terminated_length": 888.3077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6619258429855108,
      "epoch": 0.00843,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1567827463150024,
      "kl": 0.773202994838357,
      "learning_rate": 8.780811729270039e-06,
      "loss": -0.0792,
      "num_tokens": 19793860.0,
      "reward": 0.4206119775772095,
      "reward_std": 0.7523462772369385,
      "rewards/rollout_reward_func/mean": 0.4206119775772095,
      "rewards/rollout_reward_func/std": 0.7298909425735474,
      "sampling/importance_sampling_ratio/max": 1.1743966341018677,
      "sampling/importance_sampling_ratio/mean": 0.9707918763160706,
      "sampling/importance_sampling_ratio/min": 0.7258999347686768,
      "sampling/sampling_logp_difference/max": 0.21558237075805664,
      "sampling/sampling_logp_difference/mean": 0.027988910675048828,
      "step": 843,
      "step_time": 14.27899786999842
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1713.0,
      "completions/max_terminated_length": 1713.0,
      "completions/mean_length": 621.875,
      "completions/mean_terminated_length": 569.4615478515625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5418082922697067,
      "epoch": 0.00844,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8666774034500122,
      "kl": 0.9294901825487614,
      "learning_rate": 8.75473850785785e-06,
      "loss": -0.0708,
      "num_tokens": 19819260.0,
      "reward": 0.5468859672546387,
      "reward_std": 0.6199377179145813,
      "rewards/rollout_reward_func/mean": 0.5468859672546387,
      "rewards/rollout_reward_func/std": 0.641813337802887,
      "sampling/importance_sampling_ratio/max": 2.5381054878234863,
      "sampling/importance_sampling_ratio/mean": 0.9931875467300415,
      "sampling/importance_sampling_ratio/min": 1.3282006884995483e-22,
      "sampling/sampling_logp_difference/max": 21.493534088134766,
      "sampling/sampling_logp_difference/mean": 0.24766294658184052,
      "step": 844,
      "step_time": 10.984246683998208
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1628.0,
      "completions/max_terminated_length": 1628.0,
      "completions/mean_length": 535.9375,
      "completions/mean_terminated_length": 574.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1912714540958405,
      "epoch": 0.00845,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.3755073547363281,
      "kl": 0.5333790984004736,
      "learning_rate": 8.728896744129431e-06,
      "loss": 0.0573,
      "num_tokens": 19842718.0,
      "reward": 0.6385596990585327,
      "reward_std": 0.49929893016815186,
      "rewards/rollout_reward_func/mean": 0.6385596990585327,
      "rewards/rollout_reward_func/std": 0.7776293158531189,
      "sampling/importance_sampling_ratio/max": 1.4382188320159912,
      "sampling/importance_sampling_ratio/mean": 0.8785102367401123,
      "sampling/importance_sampling_ratio/min": 2.853201696940794e-17,
      "sampling/sampling_logp_difference/max": 19.33411407470703,
      "sampling/sampling_logp_difference/mean": 0.23287230730056763,
      "step": 845,
      "step_time": 11.490396339000654
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1742.0,
      "completions/max_terminated_length": 1742.0,
      "completions/mean_length": 556.125,
      "completions/mean_terminated_length": 628.7142944335938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4220894165337086,
      "epoch": 0.00846,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.887571394443512,
      "kl": 0.8665335327386856,
      "learning_rate": 8.703286746076236e-06,
      "loss": -0.0819,
      "num_tokens": 19866255.0,
      "reward": 0.7358025312423706,
      "reward_std": 0.48972854018211365,
      "rewards/rollout_reward_func/mean": 0.7358025312423706,
      "rewards/rollout_reward_func/std": 0.4731985926628113,
      "sampling/importance_sampling_ratio/max": 1.2590749263763428,
      "sampling/importance_sampling_ratio/mean": 0.9341914057731628,
      "sampling/importance_sampling_ratio/min": 5.1194466443860165e-09,
      "sampling/sampling_logp_difference/max": 12.264307975769043,
      "sampling/sampling_logp_difference/mean": 0.11953256279230118,
      "step": 846,
      "step_time": 10.95450744899972
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1919.0,
      "completions/max_terminated_length": 1802.0,
      "completions/mean_length": 400.9375,
      "completions/mean_terminated_length": 364.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5588515996932983,
      "epoch": 0.00847,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.86642986536026,
      "kl": 0.7348360605537891,
      "learning_rate": 8.67790881892747e-06,
      "loss": 0.1818,
      "num_tokens": 19886923.0,
      "reward": 0.4856036305427551,
      "reward_std": 0.6066390872001648,
      "rewards/rollout_reward_func/mean": 0.4856036305427551,
      "rewards/rollout_reward_func/std": 0.7310357689857483,
      "sampling/importance_sampling_ratio/max": 1.860242247581482,
      "sampling/importance_sampling_ratio/mean": 0.9518278241157532,
      "sampling/importance_sampling_ratio/min": 6.5166964828578934e-21,
      "sampling/sampling_logp_difference/max": 13.589832305908203,
      "sampling/sampling_logp_difference/mean": 0.18746712803840637,
      "step": 847,
      "step_time": 11.4562061250017
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1799.0,
      "completions/max_terminated_length": 1799.0,
      "completions/mean_length": 826.8125,
      "completions/mean_terminated_length": 821.1333618164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6056239753961563,
      "epoch": 0.00848,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1376317739486694,
      "kl": 0.8339121714234352,
      "learning_rate": 8.65276326514643e-06,
      "loss": -0.0393,
      "num_tokens": 19914840.0,
      "reward": 0.5577309727668762,
      "reward_std": 0.6093219518661499,
      "rewards/rollout_reward_func/mean": 0.5577309727668762,
      "rewards/rollout_reward_func/std": 0.6330252289772034,
      "sampling/importance_sampling_ratio/max": 1.0903185606002808,
      "sampling/importance_sampling_ratio/mean": 0.8524914383888245,
      "sampling/importance_sampling_ratio/min": 5.307421724332073e-10,
      "sampling/sampling_logp_difference/max": 18.709674835205078,
      "sampling/sampling_logp_difference/mean": 0.13552984595298767,
      "step": 848,
      "step_time": 11.546565594002459
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 959.0,
      "completions/max_terminated_length": 826.0,
      "completions/mean_length": 420.5625,
      "completions/mean_terminated_length": 409.8571472167969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0567972287535667,
      "epoch": 0.00849,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.198185920715332,
      "kl": 0.6392921367660165,
      "learning_rate": 8.627850384426892e-06,
      "loss": 0.1205,
      "num_tokens": 19935642.0,
      "reward": 0.7965946197509766,
      "reward_std": 0.44257768988609314,
      "rewards/rollout_reward_func/mean": 0.7965946197509766,
      "rewards/rollout_reward_func/std": 0.4373229146003723,
      "sampling/importance_sampling_ratio/max": 1.5170950889587402,
      "sampling/importance_sampling_ratio/mean": 1.0384242534637451,
      "sampling/importance_sampling_ratio/min": 0.5717442035675049,
      "sampling/sampling_logp_difference/max": 0.2306065559387207,
      "sampling/sampling_logp_difference/mean": 0.02571997232735157,
      "step": 849,
      "step_time": 8.7694756769979
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1698.0,
      "completions/max_terminated_length": 1571.0,
      "completions/mean_length": 671.5625,
      "completions/mean_terminated_length": 574.3077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6147702522575855,
      "epoch": 0.0085,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.7526480555534363,
      "kl": 0.7538327034562826,
      "learning_rate": 8.603170473689556e-06,
      "loss": -0.0618,
      "num_tokens": 19961161.0,
      "reward": 0.742318332195282,
      "reward_std": 0.38092145323753357,
      "rewards/rollout_reward_func/mean": 0.742318332195282,
      "rewards/rollout_reward_func/std": 0.5845356583595276,
      "sampling/importance_sampling_ratio/max": 1.741798758506775,
      "sampling/importance_sampling_ratio/mean": 1.0214293003082275,
      "sampling/importance_sampling_ratio/min": 0.6479222774505615,
      "sampling/sampling_logp_difference/max": 0.17653179168701172,
      "sampling/sampling_logp_difference/mean": 0.028669750317931175,
      "step": 850,
      "step_time": 11.040839593999408
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2547.0,
      "completions/max_terminated_length": 1665.0,
      "completions/mean_length": 862.625,
      "completions/mean_terminated_length": 640.923095703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5978053659200668,
      "epoch": 0.00851,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6283629536628723,
      "kl": 0.6347989067435265,
      "learning_rate": 8.57872382707851e-06,
      "loss": -0.1392,
      "num_tokens": 19990379.0,
      "reward": 0.549738883972168,
      "reward_std": 0.6441460251808167,
      "rewards/rollout_reward_func/mean": 0.549738883972168,
      "rewards/rollout_reward_func/std": 0.632943332195282,
      "sampling/importance_sampling_ratio/max": 1.3055534362792969,
      "sampling/importance_sampling_ratio/mean": 0.7613422870635986,
      "sampling/importance_sampling_ratio/min": 9.3658226305476e-17,
      "sampling/sampling_logp_difference/max": 19.130117416381836,
      "sampling/sampling_logp_difference/mean": 0.2234911173582077,
      "step": 851,
      "step_time": 14.869379875997765
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1736.0,
      "completions/max_terminated_length": 1736.0,
      "completions/mean_length": 675.25,
      "completions/mean_terminated_length": 663.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.587817221879959,
      "epoch": 0.00852,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9943965077400208,
      "kl": 0.7236828785389662,
      "learning_rate": 8.554510735957703e-06,
      "loss": -0.0107,
      "num_tokens": 20015814.0,
      "reward": 0.6274935007095337,
      "reward_std": 0.6897695660591125,
      "rewards/rollout_reward_func/mean": 0.6274935007095337,
      "rewards/rollout_reward_func/std": 0.7003945708274841,
      "sampling/importance_sampling_ratio/max": 1.4658186435699463,
      "sampling/importance_sampling_ratio/mean": 1.0040793418884277,
      "sampling/importance_sampling_ratio/min": 0.6262918710708618,
      "sampling/sampling_logp_difference/max": 0.21318650245666504,
      "sampling/sampling_logp_difference/mean": 0.028212320059537888,
      "step": 852,
      "step_time": 11.562111783996443
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.375,
      "completions/max_length": 1594.0,
      "completions/max_terminated_length": 1594.0,
      "completions/mean_length": 672.125,
      "completions/mean_terminated_length": 757.9000244140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.89031283557415,
      "epoch": 0.00853,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.873642086982727,
      "kl": 0.6163831502199173,
      "learning_rate": 8.530531488907497e-06,
      "loss": 0.1761,
      "num_tokens": 20041664.0,
      "reward": 0.544397234916687,
      "reward_std": 0.6564151048660278,
      "rewards/rollout_reward_func/mean": 0.544397234916687,
      "rewards/rollout_reward_func/std": 0.6445695757865906,
      "sampling/importance_sampling_ratio/max": 1.4975697994232178,
      "sampling/importance_sampling_ratio/mean": 1.0877764225006104,
      "sampling/importance_sampling_ratio/min": 0.5981258153915405,
      "sampling/sampling_logp_difference/max": 0.32051849365234375,
      "sampling/sampling_logp_difference/mean": 0.03295322507619858,
      "step": 853,
      "step_time": 11.51585849500043
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1891.0,
      "completions/max_terminated_length": 1514.0,
      "completions/mean_length": 595.3125,
      "completions/mean_terminated_length": 333.91668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1851290743798018,
      "epoch": 0.00854,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6594514846801758,
      "kl": 0.6568558383733034,
      "learning_rate": 8.50678637172121e-06,
      "loss": 0.4265,
      "num_tokens": 20064706.0,
      "reward": 0.6829565167427063,
      "reward_std": 0.5638391971588135,
      "rewards/rollout_reward_func/mean": 0.6829565167427063,
      "rewards/rollout_reward_func/std": 0.597036600112915,
      "sampling/importance_sampling_ratio/max": 1.317177414894104,
      "sampling/importance_sampling_ratio/mean": 1.0139837265014648,
      "sampling/importance_sampling_ratio/min": 0.8732389807701111,
      "sampling/sampling_logp_difference/max": 0.2485790252685547,
      "sampling/sampling_logp_difference/mean": 0.022637341171503067,
      "step": 854,
      "step_time": 11.540881980999984
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1777.0,
      "completions/max_terminated_length": 1616.0,
      "completions/mean_length": 581.0,
      "completions/mean_terminated_length": 333.54547119140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3499940428882837,
      "epoch": 0.00855,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3857532739639282,
      "kl": 0.4499694611877203,
      "learning_rate": 8.483275667401712e-06,
      "loss": -0.1868,
      "num_tokens": 20088128.0,
      "reward": 0.663384199142456,
      "reward_std": 0.5259945392608643,
      "rewards/rollout_reward_func/mean": 0.663384199142456,
      "rewards/rollout_reward_func/std": 0.5160158276557922,
      "sampling/importance_sampling_ratio/max": 1.5781711339950562,
      "sampling/importance_sampling_ratio/mean": 1.0322387218475342,
      "sampling/importance_sampling_ratio/min": 0.4895573854446411,
      "sampling/sampling_logp_difference/max": 0.16893243789672852,
      "sampling/sampling_logp_difference/mean": 0.027261829003691673,
      "step": 855,
      "step_time": 11.527388483000323
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2615.0,
      "completions/max_terminated_length": 2615.0,
      "completions/mean_length": 605.25,
      "completions/mean_terminated_length": 583.6428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3380172066390514,
      "epoch": 0.00856,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8050965070724487,
      "kl": 0.9280556160956621,
      "learning_rate": 8.459999656158066e-06,
      "loss": 0.0987,
      "num_tokens": 20112669.0,
      "reward": 0.5728265047073364,
      "reward_std": 0.6759073138237,
      "rewards/rollout_reward_func/mean": 0.5728265047073364,
      "rewards/rollout_reward_func/std": 0.7966110110282898,
      "sampling/importance_sampling_ratio/max": 1.1119288206100464,
      "sampling/importance_sampling_ratio/mean": 0.9603935480117798,
      "sampling/importance_sampling_ratio/min": 0.7017307877540588,
      "sampling/sampling_logp_difference/max": 0.28988218307495117,
      "sampling/sampling_logp_difference/mean": 0.026009375229477882,
      "step": 856,
      "step_time": 13.018082385000525
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1813.0,
      "completions/max_terminated_length": 1813.0,
      "completions/mean_length": 553.5,
      "completions/mean_terminated_length": 586.1333618164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2999430745840073,
      "epoch": 0.00857,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7815220952033997,
      "kl": 0.710914371535182,
      "learning_rate": 8.436958615402168e-06,
      "loss": -0.08,
      "num_tokens": 20136710.0,
      "reward": 0.2562781274318695,
      "reward_std": 0.9035817384719849,
      "rewards/rollout_reward_func/mean": 0.2562781274318695,
      "rewards/rollout_reward_func/std": 0.911335289478302,
      "sampling/importance_sampling_ratio/max": 1.5248175859451294,
      "sampling/importance_sampling_ratio/mean": 0.9765069484710693,
      "sampling/importance_sampling_ratio/min": 4.861466340599918e-09,
      "sampling/sampling_logp_difference/max": 14.353463172912598,
      "sampling/sampling_logp_difference/mean": 0.10320278257131577,
      "step": 857,
      "step_time": 11.256288738999501
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1778.0,
      "completions/max_terminated_length": 1600.0,
      "completions/mean_length": 652.375,
      "completions/mean_terminated_length": 570.0833740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.375842172652483,
      "epoch": 0.00858,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5724060535430908,
      "kl": 0.6190143898129463,
      "learning_rate": 8.414152819745456e-06,
      "loss": 0.245,
      "num_tokens": 20162085.0,
      "reward": 0.43366920948028564,
      "reward_std": 0.8310080766677856,
      "rewards/rollout_reward_func/mean": 0.43366920948028564,
      "rewards/rollout_reward_func/std": 0.8081969022750854,
      "sampling/importance_sampling_ratio/max": 1.4629466533660889,
      "sampling/importance_sampling_ratio/mean": 1.0269651412963867,
      "sampling/importance_sampling_ratio/min": 0.5214112997055054,
      "sampling/sampling_logp_difference/max": 0.21766376495361328,
      "sampling/sampling_logp_difference/mean": 0.030175719410181046,
      "step": 858,
      "step_time": 11.760869570000068
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1878.0,
      "completions/max_terminated_length": 1718.0,
      "completions/mean_length": 660.625,
      "completions/mean_terminated_length": 537.84619140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.492962770164013,
      "epoch": 0.00859,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.538702964782715,
      "kl": 1.2380655985325575,
      "learning_rate": 8.391582540995634e-06,
      "loss": -0.1453,
      "num_tokens": 20187913.0,
      "reward": 0.6254206299781799,
      "reward_std": 0.7325132489204407,
      "rewards/rollout_reward_func/mean": 0.6254206299781799,
      "rewards/rollout_reward_func/std": 0.7079835534095764,
      "sampling/importance_sampling_ratio/max": 1.0390255451202393,
      "sampling/importance_sampling_ratio/mean": 0.8642021417617798,
      "sampling/importance_sampling_ratio/min": 2.401305283089572e-12,
      "sampling/sampling_logp_difference/max": 4.715145111083984,
      "sampling/sampling_logp_difference/mean": 0.1304188221693039,
      "step": 859,
      "step_time": 12.394305724999867
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1779.0,
      "completions/max_terminated_length": 1779.0,
      "completions/mean_length": 692.875,
      "completions/mean_terminated_length": 683.5000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.637023851275444,
      "epoch": 0.0086,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.46400588750839233,
      "kl": 0.8243474513292313,
      "learning_rate": 8.369248048153426e-06,
      "loss": -0.1142,
      "num_tokens": 20214077.0,
      "reward": 0.8619884848594666,
      "reward_std": 0.2555566728115082,
      "rewards/rollout_reward_func/mean": 0.8619884848594666,
      "rewards/rollout_reward_func/std": 0.3771306872367859,
      "sampling/importance_sampling_ratio/max": 1.4582828283309937,
      "sampling/importance_sampling_ratio/mean": 0.961353063583374,
      "sampling/importance_sampling_ratio/min": 0.535504162311554,
      "sampling/sampling_logp_difference/max": 0.36265063285827637,
      "sampling/sampling_logp_difference/mean": 0.02828431874513626,
      "step": 860,
      "step_time": 11.59096393300024
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.4375,
      "completions/max_length": 986.0,
      "completions/max_terminated_length": 834.0,
      "completions/mean_length": 316.125,
      "completions/mean_terminated_length": 334.5555725097656,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5804153867065907,
      "epoch": 0.00861,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0168676376342773,
      "kl": 0.5524692758917809,
      "learning_rate": 8.347149607409375e-06,
      "loss": 0.4106,
      "num_tokens": 20233711.0,
      "reward": 0.4855349659919739,
      "reward_std": 0.6063748598098755,
      "rewards/rollout_reward_func/mean": 0.4855349659919739,
      "rewards/rollout_reward_func/std": 0.7377108335494995,
      "sampling/importance_sampling_ratio/max": 1.7198963165283203,
      "sampling/importance_sampling_ratio/mean": 0.9181568026542664,
      "sampling/importance_sampling_ratio/min": 1.028254078659626e-23,
      "sampling/sampling_logp_difference/max": 25.154605865478516,
      "sampling/sampling_logp_difference/mean": 0.25244954228401184,
      "step": 861,
      "step_time": 9.745336346999466
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 858.0,
      "completions/max_terminated_length": 833.0,
      "completions/mean_length": 418.5,
      "completions/mean_terminated_length": 414.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.7160881832242012,
      "epoch": 0.00862,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4226198196411133,
      "kl": 0.7954097464680672,
      "learning_rate": 8.325287482140678e-06,
      "loss": 0.0201,
      "num_tokens": 20254939.0,
      "reward": 0.7550102472305298,
      "reward_std": 0.6929355263710022,
      "rewards/rollout_reward_func/mean": 0.7550102472305298,
      "rewards/rollout_reward_func/std": 0.669450581073761,
      "sampling/importance_sampling_ratio/max": 1.4824931621551514,
      "sampling/importance_sampling_ratio/mean": 0.9735046029090881,
      "sampling/importance_sampling_ratio/min": 1.8523210309345783e-12,
      "sampling/sampling_logp_difference/max": 21.202926635742188,
      "sampling/sampling_logp_difference/mean": 0.14043988287448883,
      "step": 862,
      "step_time": 8.459676982000019
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1634.0,
      "completions/max_terminated_length": 1634.0,
      "completions/mean_length": 500.1875,
      "completions/mean_terminated_length": 477.0000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3943701796233654,
      "epoch": 0.00863,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6988595724105835,
      "kl": 0.5270493477582932,
      "learning_rate": 8.303661932908027e-06,
      "loss": -0.2362,
      "num_tokens": 20277694.0,
      "reward": 0.6196505427360535,
      "reward_std": 0.7043532133102417,
      "rewards/rollout_reward_func/mean": 0.6196505427360535,
      "rewards/rollout_reward_func/std": 0.7169739007949829,
      "sampling/importance_sampling_ratio/max": 1.3301119804382324,
      "sampling/importance_sampling_ratio/mean": 1.0376079082489014,
      "sampling/importance_sampling_ratio/min": 0.8183373212814331,
      "sampling/sampling_logp_difference/max": 0.17167234420776367,
      "sampling/sampling_logp_difference/mean": 0.028911402449011803,
      "step": 863,
      "step_time": 10.576060584002335
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1640.0,
      "completions/max_terminated_length": 1640.0,
      "completions/mean_length": 519.6875,
      "completions/mean_terminated_length": 447.3333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.532177161425352,
      "epoch": 0.00864,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0171775817871094,
      "kl": 0.9230003990232944,
      "learning_rate": 8.28227321745252e-06,
      "loss": -0.0451,
      "num_tokens": 20300346.0,
      "reward": 0.7399643659591675,
      "reward_std": 0.5723190307617188,
      "rewards/rollout_reward_func/mean": 0.7399643659591675,
      "rewards/rollout_reward_func/std": 0.5918923020362854,
      "sampling/importance_sampling_ratio/max": 1.071752667427063,
      "sampling/importance_sampling_ratio/mean": 0.960952639579773,
      "sampling/importance_sampling_ratio/min": 0.6709620952606201,
      "sampling/sampling_logp_difference/max": 0.2935771942138672,
      "sampling/sampling_logp_difference/mean": 0.02820460870862007,
      "step": 864,
      "step_time": 10.566467523001847
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1848.0,
      "completions/max_terminated_length": 1567.0,
      "completions/mean_length": 702.875,
      "completions/mean_terminated_length": 501.3333435058594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4484433680772781,
      "epoch": 0.00865,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7992624640464783,
      "kl": 0.7109470553696156,
      "learning_rate": 8.261121590692589e-06,
      "loss": -0.0857,
      "num_tokens": 20326340.0,
      "reward": 0.5625596046447754,
      "reward_std": 0.8305506110191345,
      "rewards/rollout_reward_func/mean": 0.5625596046447754,
      "rewards/rollout_reward_func/std": 0.8077432513237,
      "sampling/importance_sampling_ratio/max": 1.387725591659546,
      "sampling/importance_sampling_ratio/mean": 0.9366302490234375,
      "sampling/importance_sampling_ratio/min": 0.36314693093299866,
      "sampling/sampling_logp_difference/max": 0.3036642074584961,
      "sampling/sampling_logp_difference/mean": 0.031516507267951965,
      "step": 865,
      "step_time": 11.462626068994723
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1786.0,
      "completions/max_terminated_length": 897.0,
      "completions/mean_length": 451.75,
      "completions/mean_terminated_length": 310.8461608886719,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4549365378916264,
      "epoch": 0.00866,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8810009360313416,
      "kl": 0.5866758339107037,
      "learning_rate": 8.240207304720952e-06,
      "loss": 0.0114,
      "num_tokens": 20348050.0,
      "reward": 0.427706778049469,
      "reward_std": 0.8124872446060181,
      "rewards/rollout_reward_func/mean": 0.427706778049469,
      "rewards/rollout_reward_func/std": 0.8088839054107666,
      "sampling/importance_sampling_ratio/max": 1.2646095752716064,
      "sampling/importance_sampling_ratio/mean": 0.9437618851661682,
      "sampling/importance_sampling_ratio/min": 9.954144453150349e-11,
      "sampling/sampling_logp_difference/max": 21.652442932128906,
      "sampling/sampling_logp_difference/mean": 0.17391659319400787,
      "step": 866,
      "step_time": 11.270964882998669
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1673.0,
      "completions/max_terminated_length": 1673.0,
      "completions/mean_length": 718.875,
      "completions/mean_terminated_length": 702.0833740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6829408183693886,
      "epoch": 0.00867,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.460478663444519,
      "kl": 0.5484035182744265,
      "learning_rate": 8.21953060880161e-06,
      "loss": 0.1209,
      "num_tokens": 20375134.0,
      "reward": 0.6345479488372803,
      "reward_std": 0.79779052734375,
      "rewards/rollout_reward_func/mean": 0.6345479488372803,
      "rewards/rollout_reward_func/std": 0.785805881023407,
      "sampling/importance_sampling_ratio/max": 1.6394373178482056,
      "sampling/importance_sampling_ratio/mean": 0.9429079294204712,
      "sampling/importance_sampling_ratio/min": 2.684520040929783e-06,
      "sampling/sampling_logp_difference/max": 13.380247116088867,
      "sampling/sampling_logp_difference/mean": 0.06520868092775345,
      "step": 867,
      "step_time": 11.832775195001886
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1847.0,
      "completions/max_terminated_length": 1847.0,
      "completions/mean_length": 777.375,
      "completions/mean_terminated_length": 719.1333618164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5732631906867027,
      "epoch": 0.00868,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7712260484695435,
      "kl": 0.6357038896530867,
      "learning_rate": 8.199091749366899e-06,
      "loss": -0.1145,
      "num_tokens": 20402425.0,
      "reward": 0.6165534853935242,
      "reward_std": 0.6365007758140564,
      "rewards/rollout_reward_func/mean": 0.6165534853935242,
      "rewards/rollout_reward_func/std": 0.6236023306846619,
      "sampling/importance_sampling_ratio/max": 1.4995654821395874,
      "sampling/importance_sampling_ratio/mean": 0.9516746997833252,
      "sampling/importance_sampling_ratio/min": 0.40293988585472107,
      "sampling/sampling_logp_difference/max": 0.1952519416809082,
      "sampling/sampling_logp_difference/mean": 0.032004453241825104,
      "step": 868,
      "step_time": 11.767692498002361
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1809.0,
      "completions/max_terminated_length": 1809.0,
      "completions/mean_length": 608.0,
      "completions/mean_terminated_length": 636.857177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.716252975165844,
      "epoch": 0.00869,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9498699307441711,
      "kl": 0.6328095942735672,
      "learning_rate": 8.178890970014508e-06,
      "loss": -0.2174,
      "num_tokens": 20426864.0,
      "reward": 0.48921462893486023,
      "reward_std": 0.7382587790489197,
      "rewards/rollout_reward_func/mean": 0.48921462893486023,
      "rewards/rollout_reward_func/std": 0.7261632680892944,
      "sampling/importance_sampling_ratio/max": 1.2383610010147095,
      "sampling/importance_sampling_ratio/mean": 0.8901652097702026,
      "sampling/importance_sampling_ratio/min": 6.534243903334042e-14,
      "sampling/sampling_logp_difference/max": 8.03129768371582,
      "sampling/sampling_logp_difference/mean": 0.14414212107658386,
      "step": 869,
      "step_time": 11.56240649099891
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1638.0,
      "completions/max_terminated_length": 1638.0,
      "completions/mean_length": 628.75,
      "completions/mean_terminated_length": 629.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.305404158309102,
      "epoch": 0.0087,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.33756670355796814,
      "kl": 0.46743301674723625,
      "learning_rate": 8.158928511504619e-06,
      "loss": -0.0629,
      "num_tokens": 20452072.0,
      "reward": 0.9339969158172607,
      "reward_std": 0.1866850107908249,
      "rewards/rollout_reward_func/mean": 0.9339969158172607,
      "rewards/rollout_reward_func/std": 0.26401248574256897,
      "sampling/importance_sampling_ratio/max": 1.5042190551757812,
      "sampling/importance_sampling_ratio/mean": 0.9548539519309998,
      "sampling/importance_sampling_ratio/min": 4.480743253683123e-25,
      "sampling/sampling_logp_difference/max": 16.155794143676758,
      "sampling/sampling_logp_difference/mean": 0.2226213663816452,
      "step": 870,
      "step_time": 11.185077878000811
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1702.0,
      "completions/max_terminated_length": 1702.0,
      "completions/mean_length": 762.3125,
      "completions/mean_terminated_length": 695.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5795362293720245,
      "epoch": 0.00871,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 3.5433173179626465,
      "kl": 0.5320579279214144,
      "learning_rate": 8.139204611757018e-06,
      "loss": 0.5718,
      "num_tokens": 20479073.0,
      "reward": 0.9325000047683716,
      "reward_std": 0.19091883301734924,
      "rewards/rollout_reward_func/mean": 0.9325000047683716,
      "rewards/rollout_reward_func/std": 0.27000001072883606,
      "sampling/importance_sampling_ratio/max": 2.0700676441192627,
      "sampling/importance_sampling_ratio/mean": 1.0458123683929443,
      "sampling/importance_sampling_ratio/min": 0.7694677710533142,
      "sampling/sampling_logp_difference/max": 0.2197880744934082,
      "sampling/sampling_logp_difference/mean": 0.028165217489004135,
      "step": 871,
      "step_time": 11.566083742000046
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1682.0,
      "completions/max_terminated_length": 1682.0,
      "completions/mean_length": 495.625,
      "completions/mean_terminated_length": 422.6153869628906,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.593256676569581,
      "epoch": 0.00872,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.516950249671936,
      "kl": 0.5292001627385616,
      "learning_rate": 8.119719505848254e-06,
      "loss": 0.0613,
      "num_tokens": 20501367.0,
      "reward": 0.6072980761528015,
      "reward_std": 0.6280356645584106,
      "rewards/rollout_reward_func/mean": 0.6072980761528015,
      "rewards/rollout_reward_func/std": 0.6383813619613647,
      "sampling/importance_sampling_ratio/max": 1.5601048469543457,
      "sampling/importance_sampling_ratio/mean": 0.9701641798019409,
      "sampling/importance_sampling_ratio/min": 4.372587625312008e-08,
      "sampling/sampling_logp_difference/max": 12.588584899902344,
      "sampling/sampling_logp_difference/mean": 0.08689281344413757,
      "step": 872,
      "step_time": 11.12732017000053
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 1790.0,
      "completions/max_terminated_length": 1776.0,
      "completions/mean_length": 754.375,
      "completions/mean_terminated_length": 638.1666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3244204595685005,
      "epoch": 0.00873,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.336706280708313,
      "kl": 0.5829499885439873,
      "learning_rate": 8.100473426008855e-06,
      "loss": 0.3377,
      "num_tokens": 20528131.0,
      "reward": 0.37071558833122253,
      "reward_std": 0.8137291669845581,
      "rewards/rollout_reward_func/mean": 0.37071558833122253,
      "rewards/rollout_reward_func/std": 0.799475908279419,
      "sampling/importance_sampling_ratio/max": 1.5791144371032715,
      "sampling/importance_sampling_ratio/mean": 1.00725519657135,
      "sampling/importance_sampling_ratio/min": 0.7311570644378662,
      "sampling/sampling_logp_difference/max": 0.26241302490234375,
      "sampling/sampling_logp_difference/mean": 0.02878488041460514,
      "step": 873,
      "step_time": 12.388991318997796
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 949.0,
      "completions/max_terminated_length": 949.0,
      "completions/mean_length": 490.1875,
      "completions/mean_terminated_length": 490.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.294161831960082,
      "epoch": 0.00874,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.934031069278717,
      "kl": 0.674106739461422,
      "learning_rate": 8.081466601620542e-06,
      "loss": -0.0419,
      "num_tokens": 20550461.0,
      "reward": 0.5973422527313232,
      "reward_std": 0.5332619547843933,
      "rewards/rollout_reward_func/mean": 0.5973422527313232,
      "rewards/rollout_reward_func/std": 0.5371720194816589,
      "sampling/importance_sampling_ratio/max": 1.4790107011795044,
      "sampling/importance_sampling_ratio/mean": 0.9683039784431458,
      "sampling/importance_sampling_ratio/min": 9.884627796111797e-16,
      "sampling/sampling_logp_difference/max": 17.783985137939453,
      "sampling/sampling_logp_difference/mean": 0.19819864630699158,
      "step": 874,
      "step_time": 8.811566386000777
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 995.0,
      "completions/max_terminated_length": 970.0,
      "completions/mean_length": 579.0,
      "completions/mean_terminated_length": 551.2667236328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.922658510506153,
      "epoch": 0.00875,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9020832777023315,
      "kl": 0.6629271544516087,
      "learning_rate": 8.062699259213504e-06,
      "loss": 0.1646,
      "num_tokens": 20574928.0,
      "reward": 0.5466711521148682,
      "reward_std": 0.6534305214881897,
      "rewards/rollout_reward_func/mean": 0.5466711521148682,
      "rewards/rollout_reward_func/std": 0.6401097774505615,
      "sampling/importance_sampling_ratio/max": 1.173270344734192,
      "sampling/importance_sampling_ratio/mean": 0.9317067265510559,
      "sampling/importance_sampling_ratio/min": 2.7132601178835036e-13,
      "sampling/sampling_logp_difference/max": 12.76082992553711,
      "sampling/sampling_logp_difference/mean": 0.15377794206142426,
      "step": 875,
      "step_time": 9.029639915999724
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1725.0,
      "completions/max_terminated_length": 848.0,
      "completions/mean_length": 444.375,
      "completions/mean_terminated_length": 234.38462829589844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1269741170108318,
      "epoch": 0.00876,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6054916977882385,
      "kl": 0.48462827503681183,
      "learning_rate": 8.044171622463702e-06,
      "loss": -0.1111,
      "num_tokens": 20595722.0,
      "reward": 0.5577684044837952,
      "reward_std": 0.7503025531768799,
      "rewards/rollout_reward_func/mean": 0.5577684044837952,
      "rewards/rollout_reward_func/std": 0.7275595664978027,
      "sampling/importance_sampling_ratio/max": 1.3461798429489136,
      "sampling/importance_sampling_ratio/mean": 0.9932927489280701,
      "sampling/importance_sampling_ratio/min": 0.607151448726654,
      "sampling/sampling_logp_difference/max": 0.2766590118408203,
      "sampling/sampling_logp_difference/mean": 0.032893549650907516,
      "step": 876,
      "step_time": 10.510236685002383
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2654.0,
      "completions/max_terminated_length": 920.0,
      "completions/mean_length": 683.75,
      "completions/mean_terminated_length": 396.66668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.8582657985389233,
      "epoch": 0.00877,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1846977472305298,
      "kl": 0.6025748625397682,
      "learning_rate": 8.0258839121902e-06,
      "loss": -0.1643,
      "num_tokens": 20621257.0,
      "reward": 0.24873721599578857,
      "reward_std": 0.8577569127082825,
      "rewards/rollout_reward_func/mean": 0.24873721599578857,
      "rewards/rollout_reward_func/std": 0.8411441445350647,
      "sampling/importance_sampling_ratio/max": 1.4264299869537354,
      "sampling/importance_sampling_ratio/mean": 0.8798441886901855,
      "sampling/importance_sampling_ratio/min": 4.2839990973564683e-16,
      "sampling/sampling_logp_difference/max": 19.75971031188965,
      "sampling/sampling_logp_difference/mean": 0.17021381855010986,
      "step": 877,
      "step_time": 14.701492886002598
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1634.0,
      "completions/max_terminated_length": 1634.0,
      "completions/mean_length": 621.875,
      "completions/mean_terminated_length": 621.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1086784899234772,
      "epoch": 0.00878,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5171053409576416,
      "kl": 0.8469741763547063,
      "learning_rate": 8.007836346352521e-06,
      "loss": -0.0396,
      "num_tokens": 20646188.0,
      "reward": 0.5939199924468994,
      "reward_std": 0.5606219172477722,
      "rewards/rollout_reward_func/mean": 0.5939199924468994,
      "rewards/rollout_reward_func/std": 0.5416219830513,
      "sampling/importance_sampling_ratio/max": 1.5220807790756226,
      "sampling/importance_sampling_ratio/mean": 1.0334352254867554,
      "sampling/importance_sampling_ratio/min": 0.6731024384498596,
      "sampling/sampling_logp_difference/max": 0.2771270275115967,
      "sampling/sampling_logp_difference/mean": 0.02703181654214859,
      "step": 878,
      "step_time": 9.886477818998173
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1669.0,
      "completions/max_terminated_length": 1669.0,
      "completions/mean_length": 639.0625,
      "completions/mean_terminated_length": 634.5333862304688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.793570600450039,
      "epoch": 0.00879,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9696449637413025,
      "kl": 1.3178325407207012,
      "learning_rate": 7.990029140048067e-06,
      "loss": -0.0412,
      "num_tokens": 20671148.0,
      "reward": 0.6276829242706299,
      "reward_std": 0.629447877407074,
      "rewards/rollout_reward_func/mean": 0.6276829242706299,
      "rewards/rollout_reward_func/std": 0.7057676911354065,
      "sampling/importance_sampling_ratio/max": 1.6147441864013672,
      "sampling/importance_sampling_ratio/mean": 0.974979043006897,
      "sampling/importance_sampling_ratio/min": 6.8158451972666345e-12,
      "sampling/sampling_logp_difference/max": 26.430315017700195,
      "sampling/sampling_logp_difference/mean": 0.21375468373298645,
      "step": 879,
      "step_time": 10.997017424999285
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1909.0,
      "completions/max_terminated_length": 834.0,
      "completions/mean_length": 302.4375,
      "completions/mean_terminated_length": 167.92857360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6791924145072699,
      "epoch": 0.0088,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.5403629541397095,
      "kl": 0.45001867040991783,
      "learning_rate": 7.972462505509554e-06,
      "loss": 0.2727,
      "num_tokens": 20688789.0,
      "reward": 0.6055557727813721,
      "reward_std": 0.35813477635383606,
      "rewards/rollout_reward_func/mean": 0.6055557727813721,
      "rewards/rollout_reward_func/std": 0.6366932392120361,
      "sampling/importance_sampling_ratio/max": 1.543851375579834,
      "sampling/importance_sampling_ratio/mean": 0.9876994490623474,
      "sampling/importance_sampling_ratio/min": 9.259017996754437e-10,
      "sampling/sampling_logp_difference/max": 20.766813278198242,
      "sampling/sampling_logp_difference/mean": 0.12878945469856262,
      "step": 880,
      "step_time": 11.019704422000359
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1807.0,
      "completions/max_terminated_length": 1807.0,
      "completions/mean_length": 564.8125,
      "completions/mean_terminated_length": 529.0714721679688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.147148385643959,
      "epoch": 0.00881,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.131545066833496,
      "kl": 0.5382124595344067,
      "learning_rate": 7.955136652102465e-06,
      "loss": 0.1118,
      "num_tokens": 20712105.0,
      "reward": 0.6072295904159546,
      "reward_std": 0.6346068382263184,
      "rewards/rollout_reward_func/mean": 0.6072295904159546,
      "rewards/rollout_reward_func/std": 0.6388509273529053,
      "sampling/importance_sampling_ratio/max": 1.2644541263580322,
      "sampling/importance_sampling_ratio/mean": 0.982638418674469,
      "sampling/importance_sampling_ratio/min": 0.5137205719947815,
      "sampling/sampling_logp_difference/max": 0.14012908935546875,
      "sampling/sampling_logp_difference/mean": 0.025454984977841377,
      "step": 881,
      "step_time": 10.703276163998453
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1855.0,
      "completions/max_terminated_length": 1720.0,
      "completions/mean_length": 606.625,
      "completions/mean_terminated_length": 542.7692260742188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4622820541262627,
      "epoch": 0.00882,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9022000432014465,
      "kl": 0.8186977505683899,
      "learning_rate": 7.938051786322573e-06,
      "loss": 0.181,
      "num_tokens": 20736973.0,
      "reward": 0.3493487238883972,
      "reward_std": 0.6926413774490356,
      "rewards/rollout_reward_func/mean": 0.3493487238883972,
      "rewards/rollout_reward_func/std": 0.7325334548950195,
      "sampling/importance_sampling_ratio/max": 1.272384762763977,
      "sampling/importance_sampling_ratio/mean": 0.9605676531791687,
      "sampling/importance_sampling_ratio/min": 0.6927120685577393,
      "sampling/sampling_logp_difference/max": 0.2397756576538086,
      "sampling/sampling_logp_difference/mean": 0.02628670446574688,
      "step": 882,
      "step_time": 11.710070709999854
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1796.0,
      "completions/max_terminated_length": 1796.0,
      "completions/mean_length": 450.625,
      "completions/mean_terminated_length": 431.4000244140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.365532759577036,
      "epoch": 0.00883,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.41091716289520264,
      "kl": 0.8314289562404156,
      "learning_rate": 7.92120811179348e-06,
      "loss": -0.0315,
      "num_tokens": 20758679.0,
      "reward": 0.5427137613296509,
      "reward_std": 0.6561460494995117,
      "rewards/rollout_reward_func/mean": 0.5427137613296509,
      "rewards/rollout_reward_func/std": 0.6440098881721497,
      "sampling/importance_sampling_ratio/max": 1.2311145067214966,
      "sampling/importance_sampling_ratio/mean": 0.8522403240203857,
      "sampling/importance_sampling_ratio/min": 1.0348461670917075e-10,
      "sampling/sampling_logp_difference/max": 22.480558395385742,
      "sampling/sampling_logp_difference/mean": 0.282776802778244,
      "step": 883,
      "step_time": 10.457846403998701
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2513.0,
      "completions/max_terminated_length": 2513.0,
      "completions/mean_length": 745.1875,
      "completions/mean_terminated_length": 720.1538696289062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.721795916557312,
      "epoch": 0.00884,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0163999795913696,
      "kl": 0.8014495149254799,
      "learning_rate": 7.90460582926417e-06,
      "loss": -0.1695,
      "num_tokens": 20785638.0,
      "reward": 0.5304260849952698,
      "reward_std": 0.6389857530593872,
      "rewards/rollout_reward_func/mean": 0.5304260849952698,
      "rewards/rollout_reward_func/std": 0.6609392166137695,
      "sampling/importance_sampling_ratio/max": 1.587839961051941,
      "sampling/importance_sampling_ratio/mean": 1.0441172122955322,
      "sampling/importance_sampling_ratio/min": 0.64992356300354,
      "sampling/sampling_logp_difference/max": 0.3224518299102783,
      "sampling/sampling_logp_difference/mean": 0.029431842267513275,
      "step": 884,
      "step_time": 14.253267656002208
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 2718.0,
      "completions/max_terminated_length": 2718.0,
      "completions/mean_length": 808.25,
      "completions/mean_terminated_length": 714.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.469788022339344,
      "epoch": 0.00885,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8183573484420776,
      "kl": 0.5638963356614113,
      "learning_rate": 7.888245136606638e-06,
      "loss": -0.0744,
      "num_tokens": 20813034.0,
      "reward": 0.48574918508529663,
      "reward_std": 0.6890072822570801,
      "rewards/rollout_reward_func/mean": 0.48574918508529663,
      "rewards/rollout_reward_func/std": 0.728642463684082,
      "sampling/importance_sampling_ratio/max": 1.4011123180389404,
      "sampling/importance_sampling_ratio/mean": 0.860373854637146,
      "sampling/importance_sampling_ratio/min": 3.0392853354486646e-14,
      "sampling/sampling_logp_difference/max": 18.79837417602539,
      "sampling/sampling_logp_difference/mean": 0.09360989183187485,
      "step": 885,
      "step_time": 15.52104178099944
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1808.0,
      "completions/max_terminated_length": 838.0,
      "completions/mean_length": 425.9375,
      "completions/mean_terminated_length": 227.72727966308594,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5657994076609612,
      "epoch": 0.00886,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.075132966041565,
      "kl": 0.664998609572649,
      "learning_rate": 7.872126228813521e-06,
      "loss": -0.0667,
      "num_tokens": 20833876.0,
      "reward": 0.40332669019699097,
      "reward_std": 0.6686867475509644,
      "rewards/rollout_reward_func/mean": 0.40332669019699097,
      "rewards/rollout_reward_func/std": 0.6529890894889832,
      "sampling/importance_sampling_ratio/max": 1.525459885597229,
      "sampling/importance_sampling_ratio/mean": 0.927977979183197,
      "sampling/importance_sampling_ratio/min": 1.994525686256442e-16,
      "sampling/sampling_logp_difference/max": 18.43327522277832,
      "sampling/sampling_logp_difference/mean": 0.15537595748901367,
      "step": 886,
      "step_time": 11.419101400997533
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1823.0,
      "completions/max_terminated_length": 1823.0,
      "completions/mean_length": 569.1875,
      "completions/mean_terminated_length": 645.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2116938531398773,
      "epoch": 0.00887,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8437082767486572,
      "kl": 0.6463228994980454,
      "learning_rate": 7.856249297995781e-06,
      "loss": 0.1372,
      "num_tokens": 20857817.0,
      "reward": 0.2549014687538147,
      "reward_std": 0.817358136177063,
      "rewards/rollout_reward_func/mean": 0.2549014687538147,
      "rewards/rollout_reward_func/std": 0.8297445774078369,
      "sampling/importance_sampling_ratio/max": 1.4011951684951782,
      "sampling/importance_sampling_ratio/mean": 0.9172854423522949,
      "sampling/importance_sampling_ratio/min": 9.86034809358216e-10,
      "sampling/sampling_logp_difference/max": 20.859975814819336,
      "sampling/sampling_logp_difference/mean": 0.15532410144805908,
      "step": 887,
      "step_time": 10.64267695199851
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1743.0,
      "completions/max_terminated_length": 970.0,
      "completions/mean_length": 397.1875,
      "completions/mean_terminated_length": 307.4666748046875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0191563218832016,
      "epoch": 0.00888,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9001659750938416,
      "kl": 0.3077131463214755,
      "learning_rate": 7.840614533380407e-06,
      "loss": 0.1388,
      "num_tokens": 20877905.0,
      "reward": 0.6724181771278381,
      "reward_std": 0.6219671368598938,
      "rewards/rollout_reward_func/mean": 0.6724181771278381,
      "rewards/rollout_reward_func/std": 0.6114208698272705,
      "sampling/importance_sampling_ratio/max": 1.0734118223190308,
      "sampling/importance_sampling_ratio/mean": 0.9716770648956299,
      "sampling/importance_sampling_ratio/min": 0.7508154511451721,
      "sampling/sampling_logp_difference/max": 0.15123987197875977,
      "sampling/sampling_logp_difference/mean": 0.02280103974044323,
      "step": 888,
      "step_time": 9.973963502003244
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1725.0,
      "completions/max_terminated_length": 1725.0,
      "completions/mean_length": 576.625,
      "completions/mean_terminated_length": 591.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6211242526769638,
      "epoch": 0.00889,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5097296833992004,
      "kl": 0.6542508285492659,
      "learning_rate": 7.825222121308165e-06,
      "loss": -0.1105,
      "num_tokens": 20902343.0,
      "reward": 0.6031678318977356,
      "reward_std": 0.5720747113227844,
      "rewards/rollout_reward_func/mean": 0.6031678318977356,
      "rewards/rollout_reward_func/std": 0.6364299654960632,
      "sampling/importance_sampling_ratio/max": 1.318142294883728,
      "sampling/importance_sampling_ratio/mean": 0.9422881603240967,
      "sampling/importance_sampling_ratio/min": 7.009434807754567e-13,
      "sampling/sampling_logp_difference/max": 7.515300750732422,
      "sampling/sampling_logp_difference/mean": 0.17381206154823303,
      "step": 889,
      "step_time": 10.858224349000011
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1621.0,
      "completions/max_terminated_length": 1621.0,
      "completions/mean_length": 569.5,
      "completions/mean_terminated_length": 569.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.206977304071188,
      "epoch": 0.0089,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2844575643539429,
      "kl": 0.9624159522354603,
      "learning_rate": 7.810072245231378e-06,
      "loss": 0.1312,
      "num_tokens": 20925812.0,
      "reward": 0.6748967170715332,
      "reward_std": 0.5698680877685547,
      "rewards/rollout_reward_func/mean": 0.6748967170715332,
      "rewards/rollout_reward_func/std": 0.6123073697090149,
      "sampling/importance_sampling_ratio/max": 1.1972264051437378,
      "sampling/importance_sampling_ratio/mean": 0.9819202423095703,
      "sampling/importance_sampling_ratio/min": 0.7178293466567993,
      "sampling/sampling_logp_difference/max": 0.18522977828979492,
      "sampling/sampling_logp_difference/mean": 0.027439866214990616,
      "step": 890,
      "step_time": 10.12577328000225
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2741.0,
      "completions/max_terminated_length": 1734.0,
      "completions/mean_length": 699.0625,
      "completions/mean_terminated_length": 562.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.930400412529707,
      "epoch": 0.00891,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0224251747131348,
      "kl": 0.3827819302678108,
      "learning_rate": 7.79516508571173e-06,
      "loss": 0.5402,
      "num_tokens": 20951863.0,
      "reward": 0.42415499687194824,
      "reward_std": 0.7142229080200195,
      "rewards/rollout_reward_func/mean": 0.42415499687194824,
      "rewards/rollout_reward_func/std": 0.7220098376274109,
      "sampling/importance_sampling_ratio/max": 1.2409158945083618,
      "sampling/importance_sampling_ratio/mean": 1.0189790725708008,
      "sampling/importance_sampling_ratio/min": 0.8730770945549011,
      "sampling/sampling_logp_difference/max": 0.22806739807128906,
      "sampling/sampling_logp_difference/mean": 0.03132522106170654,
      "step": 891,
      "step_time": 13.749488969002414
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1640.0,
      "completions/max_terminated_length": 1640.0,
      "completions/mean_length": 560.4375,
      "completions/mean_terminated_length": 546.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0346073806285858,
      "epoch": 0.00892,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4949568808078766,
      "kl": 0.7398681044578552,
      "learning_rate": 7.780500820418133e-06,
      "loss": -0.1476,
      "num_tokens": 20975434.0,
      "reward": 0.6586809158325195,
      "reward_std": 0.48808738589286804,
      "rewards/rollout_reward_func/mean": 0.6586809158325195,
      "rewards/rollout_reward_func/std": 0.5231407284736633,
      "sampling/importance_sampling_ratio/max": 1.092974066734314,
      "sampling/importance_sampling_ratio/mean": 0.9681469202041626,
      "sampling/importance_sampling_ratio/min": 0.7239977121353149,
      "sampling/sampling_logp_difference/max": 0.20400047302246094,
      "sampling/sampling_logp_difference/mean": 0.02400052361190319,
      "step": 892,
      "step_time": 10.885314254002878
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2553.0,
      "completions/max_terminated_length": 2553.0,
      "completions/mean_length": 783.5,
      "completions/mean_terminated_length": 783.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0983014479279518,
      "epoch": 0.00893,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5916489362716675,
      "kl": 0.6931402124464512,
      "learning_rate": 7.766079624124588e-06,
      "loss": 0.0304,
      "num_tokens": 21002628.0,
      "reward": 0.7294528484344482,
      "reward_std": 0.5011008381843567,
      "rewards/rollout_reward_func/mean": 0.7294528484344482,
      "rewards/rollout_reward_func/std": 0.48415589332580566,
      "sampling/importance_sampling_ratio/max": 1.2062876224517822,
      "sampling/importance_sampling_ratio/mean": 0.9976500272750854,
      "sampling/importance_sampling_ratio/min": 0.761707067489624,
      "sampling/sampling_logp_difference/max": 0.20887231826782227,
      "sampling/sampling_logp_difference/mean": 0.027715733274817467,
      "step": 893,
      "step_time": 12.673456579999765
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1682.0,
      "completions/max_terminated_length": 1682.0,
      "completions/mean_length": 540.5,
      "completions/mean_terminated_length": 546.6923217773438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.385519064962864,
      "epoch": 0.00894,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1519970893859863,
      "kl": 0.9397054016590118,
      "learning_rate": 7.75190166870812e-06,
      "loss": 0.2186,
      "num_tokens": 21025389.0,
      "reward": 0.47772902250289917,
      "reward_std": 0.6120089292526245,
      "rewards/rollout_reward_func/mean": 0.47772902250289917,
      "rewards/rollout_reward_func/std": 0.6523107290267944,
      "sampling/importance_sampling_ratio/max": 1.515763521194458,
      "sampling/importance_sampling_ratio/mean": 0.9944348335266113,
      "sampling/importance_sampling_ratio/min": 0.5991809368133545,
      "sampling/sampling_logp_difference/max": 0.20254743099212646,
      "sampling/sampling_logp_difference/mean": 0.031139777973294258,
      "step": 894,
      "step_time": 10.577616595999643
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1700.0,
      "completions/max_terminated_length": 1700.0,
      "completions/mean_length": 536.6875,
      "completions/mean_terminated_length": 470.66668701171875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8474718313664198,
      "epoch": 0.00895,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.6779471039772034,
      "kl": 0.33879080321639776,
      "learning_rate": 7.737967123146714e-06,
      "loss": -0.1411,
      "num_tokens": 21048097.0,
      "reward": 0.8129424452781677,
      "reward_std": 0.36376577615737915,
      "rewards/rollout_reward_func/mean": 0.8129424452781677,
      "rewards/rollout_reward_func/std": 0.5332269072532654,
      "sampling/importance_sampling_ratio/max": 2.0459420680999756,
      "sampling/importance_sampling_ratio/mean": 1.0526803731918335,
      "sampling/importance_sampling_ratio/min": 0.8011847138404846,
      "sampling/sampling_logp_difference/max": 0.16142654418945312,
      "sampling/sampling_logp_difference/mean": 0.02706306241452694,
      "step": 895,
      "step_time": 10.098001406000549
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1811.0,
      "completions/max_terminated_length": 1791.0,
      "completions/mean_length": 635.375,
      "completions/mean_terminated_length": 557.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8927110061049461,
      "epoch": 0.00896,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9759595394134521,
      "kl": 1.7634342648088932,
      "learning_rate": 7.724276153517313e-06,
      "loss": 0.1072,
      "num_tokens": 21072952.0,
      "reward": 0.6152157187461853,
      "reward_std": 0.6176170110702515,
      "rewards/rollout_reward_func/mean": 0.6152157187461853,
      "rewards/rollout_reward_func/std": 0.6218934059143066,
      "sampling/importance_sampling_ratio/max": 1.068014144897461,
      "sampling/importance_sampling_ratio/mean": 0.9748421311378479,
      "sampling/importance_sampling_ratio/min": 0.6557506918907166,
      "sampling/sampling_logp_difference/max": 0.3492274284362793,
      "sampling/sampling_logp_difference/mean": 0.02069617621600628,
      "step": 896,
      "step_time": 10.339501369997379
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1565.0,
      "completions/max_terminated_length": 1565.0,
      "completions/mean_length": 365.8125,
      "completions/mean_terminated_length": 365.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7447046190500259,
      "epoch": 0.00897,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.26732975244522095,
      "kl": 0.40643124654889107,
      "learning_rate": 7.710828922993833e-06,
      "loss": -0.0227,
      "num_tokens": 21092962.0,
      "reward": 0.49431777000427246,
      "reward_std": 0.6780946850776672,
      "rewards/rollout_reward_func/mean": 0.49431777000427246,
      "rewards/rollout_reward_func/std": 0.8133965730667114,
      "sampling/importance_sampling_ratio/max": 1.1393474340438843,
      "sampling/importance_sampling_ratio/mean": 1.002786636352539,
      "sampling/importance_sampling_ratio/min": 0.6920955181121826,
      "sampling/sampling_logp_difference/max": 0.12295222282409668,
      "sampling/sampling_logp_difference/mean": 0.024784576147794724,
      "step": 897,
      "step_time": 9.14841874400372
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1835.0,
      "completions/max_terminated_length": 1835.0,
      "completions/mean_length": 513.6875,
      "completions/mean_terminated_length": 437.7692565917969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.6111144050955772,
      "epoch": 0.00898,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7333419322967529,
      "kl": 0.36466221883893013,
      "learning_rate": 7.697625591845221e-06,
      "loss": 0.0262,
      "num_tokens": 21115636.0,
      "reward": 0.4898563027381897,
      "reward_std": 0.7341861724853516,
      "rewards/rollout_reward_func/mean": 0.4898563027381897,
      "rewards/rollout_reward_func/std": 0.7260259985923767,
      "sampling/importance_sampling_ratio/max": 1.1896928548812866,
      "sampling/importance_sampling_ratio/mean": 0.941190242767334,
      "sampling/importance_sampling_ratio/min": 1.9325998579233783e-09,
      "sampling/sampling_logp_difference/max": 14.83749771118164,
      "sampling/sampling_logp_difference/mean": 0.11291645467281342,
      "step": 898,
      "step_time": 10.997788931003015
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2510.0,
      "completions/max_terminated_length": 2510.0,
      "completions/mean_length": 631.875,
      "completions/mean_terminated_length": 618.2667236328125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7942952401936054,
      "epoch": 0.00899,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8226004838943481,
      "kl": 0.3383679762482643,
      "learning_rate": 7.684666317433535e-06,
      "loss": -0.0703,
      "num_tokens": 21140223.0,
      "reward": 0.5491703152656555,
      "reward_std": 0.6388058662414551,
      "rewards/rollout_reward_func/mean": 0.5491703152656555,
      "rewards/rollout_reward_func/std": 0.7354444265365601,
      "sampling/importance_sampling_ratio/max": 1.5765507221221924,
      "sampling/importance_sampling_ratio/mean": 1.0428099632263184,
      "sampling/importance_sampling_ratio/min": 0.9098498225212097,
      "sampling/sampling_logp_difference/max": 0.1980304718017578,
      "sampling/sampling_logp_difference/mean": 0.020594816654920578,
      "step": 899,
      "step_time": 13.13632427900302
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2685.0,
      "completions/max_terminated_length": 1532.0,
      "completions/mean_length": 738.5,
      "completions/mean_terminated_length": 608.7333374023438,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0487808659672737,
      "epoch": 0.009,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8894680142402649,
      "kl": 0.6339483857154846,
      "learning_rate": 7.671951254212094e-06,
      "loss": 0.0457,
      "num_tokens": 21166917.0,
      "reward": 0.6792424917221069,
      "reward_std": 0.6108062267303467,
      "rewards/rollout_reward_func/mean": 0.6792424917221069,
      "rewards/rollout_reward_func/std": 0.6021819710731506,
      "sampling/importance_sampling_ratio/max": 1.3375359773635864,
      "sampling/importance_sampling_ratio/mean": 0.9837954044342041,
      "sampling/importance_sampling_ratio/min": 0.5622478127479553,
      "sampling/sampling_logp_difference/max": 0.24944067001342773,
      "sampling/sampling_logp_difference/mean": 0.025279920548200607,
      "step": 900,
      "step_time": 13.404942203000246
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1855.0,
      "completions/max_terminated_length": 1855.0,
      "completions/mean_length": 536.0625,
      "completions/mean_terminated_length": 536.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8331081196665764,
      "epoch": 0.00901,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5032199025154114,
      "kl": 0.42342324927449226,
      "learning_rate": 7.659480553723602e-06,
      "loss": 0.008,
      "num_tokens": 21190048.0,
      "reward": 0.8019207715988159,
      "reward_std": 0.42891186475753784,
      "rewards/rollout_reward_func/mean": 0.8019207715988159,
      "rewards/rollout_reward_func/std": 0.426250696182251,
      "sampling/importance_sampling_ratio/max": 1.54166841506958,
      "sampling/importance_sampling_ratio/mean": 1.0303772687911987,
      "sampling/importance_sampling_ratio/min": 0.915360152721405,
      "sampling/sampling_logp_difference/max": 0.22404098510742188,
      "sampling/sampling_logp_difference/mean": 0.01804255321621895,
      "step": 901,
      "step_time": 10.10704620999968
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1655.0,
      "completions/max_terminated_length": 1655.0,
      "completions/mean_length": 685.375,
      "completions/mean_terminated_length": 680.1333618164062,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2804638892412186,
      "epoch": 0.00902,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6163058280944824,
      "kl": 0.6758141405880451,
      "learning_rate": 7.64725436459837e-06,
      "loss": -0.0887,
      "num_tokens": 21215865.0,
      "reward": 0.5522110462188721,
      "reward_std": 0.7031264305114746,
      "rewards/rollout_reward_func/mean": 0.5522110462188721,
      "rewards/rollout_reward_func/std": 0.7277012467384338,
      "sampling/importance_sampling_ratio/max": 1.1641026735305786,
      "sampling/importance_sampling_ratio/mean": 0.9916483759880066,
      "sampling/importance_sampling_ratio/min": 0.8805674910545349,
      "sampling/sampling_logp_difference/max": 0.12718766927719116,
      "sampling/sampling_logp_difference/mean": 0.02393229492008686,
      "step": 902,
      "step_time": 10.31488685900149
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2733.0,
      "completions/max_terminated_length": 2733.0,
      "completions/mean_length": 823.0,
      "completions/mean_terminated_length": 681.0714721679688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0154969058930874,
      "epoch": 0.00903,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2128349542617798,
      "kl": 0.34380718041211367,
      "learning_rate": 7.635272832552527e-06,
      "loss": 0.0982,
      "num_tokens": 21243710.0,
      "reward": 0.7330141067504883,
      "reward_std": 0.4692023992538452,
      "rewards/rollout_reward_func/mean": 0.7330141067504883,
      "rewards/rollout_reward_func/std": 0.4778425395488739,
      "sampling/importance_sampling_ratio/max": 1.1760233640670776,
      "sampling/importance_sampling_ratio/mean": 0.9518567323684692,
      "sampling/importance_sampling_ratio/min": 0.5812722444534302,
      "sampling/sampling_logp_difference/max": 0.30291748046875,
      "sampling/sampling_logp_difference/mean": 0.027780747041106224,
      "step": 903,
      "step_time": 14.134035882001626
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2692.0,
      "completions/max_terminated_length": 2692.0,
      "completions/mean_length": 754.375,
      "completions/mean_terminated_length": 802.5333862304688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.266778964549303,
      "epoch": 0.00904,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6960416436195374,
      "kl": 0.6547396071255207,
      "learning_rate": 7.623536100386304e-06,
      "loss": -0.1049,
      "num_tokens": 21270585.0,
      "reward": 0.6113023161888123,
      "reward_std": 0.6450860500335693,
      "rewards/rollout_reward_func/mean": 0.6113023161888123,
      "rewards/rollout_reward_func/std": 0.630847156047821,
      "sampling/importance_sampling_ratio/max": 1.1484172344207764,
      "sampling/importance_sampling_ratio/mean": 0.9811941981315613,
      "sampling/importance_sampling_ratio/min": 0.8199650645256042,
      "sampling/sampling_logp_difference/max": 0.16441631317138672,
      "sampling/sampling_logp_difference/mean": 0.028866268694400787,
      "step": 904,
      "step_time": 13.289869776999694
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 929.0,
      "completions/max_terminated_length": 817.0,
      "completions/mean_length": 377.75,
      "completions/mean_terminated_length": 341.0000305175781,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6593099581077695,
      "epoch": 0.00905,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8097413778305054,
      "kl": 0.36023109778761864,
      "learning_rate": 7.612044307982303e-06,
      "loss": 0.1777,
      "num_tokens": 21290368.0,
      "reward": 0.6144157648086548,
      "reward_std": 0.6402077674865723,
      "rewards/rollout_reward_func/mean": 0.6144157648086548,
      "rewards/rollout_reward_func/std": 0.6251063346862793,
      "sampling/importance_sampling_ratio/max": 1.0503044128417969,
      "sampling/importance_sampling_ratio/mean": 0.9988479614257812,
      "sampling/importance_sampling_ratio/min": 0.899393618106842,
      "sampling/sampling_logp_difference/max": 0.12119221687316895,
      "sampling/sampling_logp_difference/mean": 0.016978489235043526,
      "step": 905,
      "step_time": 7.65948420499808
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1764.0,
      "completions/max_terminated_length": 1764.0,
      "completions/mean_length": 435.25,
      "completions/mean_terminated_length": 435.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6395179815590382,
      "epoch": 0.00906,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.27002251148223877,
      "kl": 0.2774491189047694,
      "learning_rate": 7.600797592303854e-06,
      "loss": 0.0383,
      "num_tokens": 21311157.0,
      "reward": 0.559807538986206,
      "reward_std": 0.7345445156097412,
      "rewards/rollout_reward_func/mean": 0.559807538986206,
      "rewards/rollout_reward_func/std": 0.7131476402282715,
      "sampling/importance_sampling_ratio/max": 1.0938831567764282,
      "sampling/importance_sampling_ratio/mean": 1.0006980895996094,
      "sampling/importance_sampling_ratio/min": 0.925350546836853,
      "sampling/sampling_logp_difference/max": 0.1249433159828186,
      "sampling/sampling_logp_difference/mean": 0.00972763728350401,
      "step": 906,
      "step_time": 10.104257541999687
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1708.0,
      "completions/max_terminated_length": 1708.0,
      "completions/mean_length": 598.0625,
      "completions/mean_terminated_length": 503.21429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9141396787017584,
      "epoch": 0.00907,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05513852834701538,
      "kl": 0.57020035199821,
      "learning_rate": 7.589796087393373e-06,
      "loss": 0.0048,
      "num_tokens": 21335082.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/rollout_reward_func/mean": 1.0,
      "rewards/rollout_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2179646492004395,
      "sampling/importance_sampling_ratio/mean": 1.0014066696166992,
      "sampling/importance_sampling_ratio/min": 0.9194760918617249,
      "sampling/sampling_logp_difference/max": 0.19546198844909668,
      "sampling/sampling_logp_difference/mean": 0.024612030014395714,
      "step": 907,
      "step_time": 10.497125726002196
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1715.0,
      "completions/max_terminated_length": 1645.0,
      "completions/mean_length": 787.875,
      "completions/mean_terminated_length": 775.6428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0623344704508781,
      "epoch": 0.00908,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8550467491149902,
      "kl": 0.5217671804130077,
      "learning_rate": 7.5790399243707634e-06,
      "loss": 0.0986,
      "num_tokens": 21362557.0,
      "reward": 0.6031180620193481,
      "reward_std": 0.5773729681968689,
      "rewards/rollout_reward_func/mean": 0.6031180620193481,
      "rewards/rollout_reward_func/std": 0.643212080001831,
      "sampling/importance_sampling_ratio/max": 1.2933756113052368,
      "sampling/importance_sampling_ratio/mean": 1.010263204574585,
      "sampling/importance_sampling_ratio/min": 0.8713012933731079,
      "sampling/sampling_logp_difference/max": 0.289780855178833,
      "sampling/sampling_logp_difference/mean": 0.023806758224964142,
      "step": 908,
      "step_time": 10.856283142002212
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1788.0,
      "completions/max_terminated_length": 1788.0,
      "completions/mean_length": 574.4375,
      "completions/mean_terminated_length": 574.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8248041793704033,
      "epoch": 0.00909,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.0680042505264282,
      "kl": 10.065505442209542,
      "learning_rate": 7.568529231431858e-06,
      "loss": 0.0293,
      "num_tokens": 21386622.0,
      "reward": 0.9351791143417358,
      "reward_std": 0.1833411604166031,
      "rewards/rollout_reward_func/mean": 0.9351791143417358,
      "rewards/rollout_reward_func/std": 0.25928354263305664,
      "sampling/importance_sampling_ratio/max": 1.0475608110427856,
      "sampling/importance_sampling_ratio/mean": 0.9806429743766785,
      "sampling/importance_sampling_ratio/min": 0.7996190786361694,
      "sampling/sampling_logp_difference/max": 0.11143046617507935,
      "sampling/sampling_logp_difference/mean": 0.014862350188195705,
      "step": 909,
      "step_time": 9.932957754996096
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2772.0,
      "completions/max_terminated_length": 1711.0,
      "completions/mean_length": 669.75,
      "completions/mean_terminated_length": 564.9285888671875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0929285734891891,
      "epoch": 0.0091,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7003353238105774,
      "kl": 0.5134495496749878,
      "learning_rate": 7.558264133846891e-06,
      "loss": 0.1126,
      "num_tokens": 21412145.0,
      "reward": 0.335368812084198,
      "reward_std": 0.9366194009780884,
      "rewards/rollout_reward_func/mean": 0.335368812084198,
      "rewards/rollout_reward_func/std": 0.9063203930854797,
      "sampling/importance_sampling_ratio/max": 1.0817559957504272,
      "sampling/importance_sampling_ratio/mean": 0.9200998544692993,
      "sampling/importance_sampling_ratio/min": 8.696077608463483e-23,
      "sampling/sampling_logp_difference/max": 20.244184494018555,
      "sampling/sampling_logp_difference/mean": 0.40539321303367615,
      "step": 910,
      "step_time": 13.14108785100143
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1568.0,
      "completions/max_terminated_length": 1568.0,
      "completions/mean_length": 443.9375,
      "completions/mean_terminated_length": 443.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5971345640718937,
      "epoch": 0.00911,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4052808880805969,
      "kl": 0.4047260135412216,
      "learning_rate": 7.548244753958995e-06,
      "loss": -0.0507,
      "num_tokens": 21433446.0,
      "reward": 0.6610804796218872,
      "reward_std": 0.5322914123535156,
      "rewards/rollout_reward_func/mean": 0.6610804796218872,
      "rewards/rollout_reward_func/std": 0.5193619728088379,
      "sampling/importance_sampling_ratio/max": 1.2635427713394165,
      "sampling/importance_sampling_ratio/mean": 1.0117714405059814,
      "sampling/importance_sampling_ratio/min": 0.9800905585289001,
      "sampling/sampling_logp_difference/max": 0.1156005859375,
      "sampling/sampling_logp_difference/mean": 0.012533750385046005,
      "step": 911,
      "step_time": 9.370437974001106
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2546.0,
      "completions/max_terminated_length": 2546.0,
      "completions/mean_length": 906.4375,
      "completions/mean_terminated_length": 844.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8468065205961466,
      "epoch": 0.00912,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8210791945457458,
      "kl": 1.2743759732693434,
      "learning_rate": 7.538471211182758e-06,
      "loss": 0.017,
      "num_tokens": 21462455.0,
      "reward": 0.6779497861862183,
      "reward_std": 0.6251046657562256,
      "rewards/rollout_reward_func/mean": 0.6779497861862183,
      "rewards/rollout_reward_func/std": 0.6134231686592102,
      "sampling/importance_sampling_ratio/max": 1.2545032501220703,
      "sampling/importance_sampling_ratio/mean": 0.9758158326148987,
      "sampling/importance_sampling_ratio/min": 0.6222091317176819,
      "sampling/sampling_logp_difference/max": 0.3523130416870117,
      "sampling/sampling_logp_difference/mean": 0.03040887601673603,
      "step": 912,
      "step_time": 13.45637492899732
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1838.0,
      "completions/max_terminated_length": 1838.0,
      "completions/mean_length": 785.25,
      "completions/mean_terminated_length": 734.2000122070312,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9632822647690773,
      "epoch": 0.00913,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3958889842033386,
      "kl": 0.6757143493741751,
      "learning_rate": 7.528943622002787e-06,
      "loss": -0.0465,
      "num_tokens": 21489835.0,
      "reward": 0.6853433847427368,
      "reward_std": 0.6457865834236145,
      "rewards/rollout_reward_func/mean": 0.6853433847427368,
      "rewards/rollout_reward_func/std": 0.7064494490623474,
      "sampling/importance_sampling_ratio/max": 1.0217704772949219,
      "sampling/importance_sampling_ratio/mean": 0.8210113048553467,
      "sampling/importance_sampling_ratio/min": 5.755709277244225e-12,
      "sampling/sampling_logp_difference/max": 21.0534725189209,
      "sampling/sampling_logp_difference/mean": 0.2711680233478546,
      "step": 913,
      "step_time": 11.148073878001014
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1527.0,
      "completions/max_terminated_length": 1527.0,
      "completions/mean_length": 321.5,
      "completions/mean_terminated_length": 321.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6037962809205055,
      "epoch": 0.00914,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.1739882379770279,
      "kl": 0.6759455371648073,
      "learning_rate": 7.519662099972328e-06,
      "loss": 0.0051,
      "num_tokens": 21508394.0,
      "reward": 0.8076955080032349,
      "reward_std": 0.376568466424942,
      "rewards/rollout_reward_func/mean": 0.8076955080032349,
      "rewards/rollout_reward_func/std": 0.5514951348304749,
      "sampling/importance_sampling_ratio/max": 1.0968670845031738,
      "sampling/importance_sampling_ratio/mean": 0.98610520362854,
      "sampling/importance_sampling_ratio/min": 0.8102249503135681,
      "sampling/sampling_logp_difference/max": 0.13266444206237793,
      "sampling/sampling_logp_difference/mean": 0.018566107377409935,
      "step": 914,
      "step_time": 8.736856137997165
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1724.0,
      "completions/max_terminated_length": 1724.0,
      "completions/mean_length": 673.8125,
      "completions/mean_terminated_length": 667.3333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1473029125481844,
      "epoch": 0.00915,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5289828181266785,
      "kl": 1.5756232012063265,
      "learning_rate": 7.5106267557119095e-06,
      "loss": -0.1363,
      "num_tokens": 21533573.0,
      "reward": 0.6887268424034119,
      "reward_std": 0.7147108316421509,
      "rewards/rollout_reward_func/mean": 0.6887268424034119,
      "rewards/rollout_reward_func/std": 0.6939460635185242,
      "sampling/importance_sampling_ratio/max": 1.9137521982192993,
      "sampling/importance_sampling_ratio/mean": 1.027882695198059,
      "sampling/importance_sampling_ratio/min": 0.6374307870864868,
      "sampling/sampling_logp_difference/max": 0.18635988235473633,
      "sampling/sampling_logp_difference/mean": 0.027002299204468727,
      "step": 915,
      "step_time": 10.586535669001023
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 952.0,
      "completions/max_terminated_length": 952.0,
      "completions/mean_length": 292.5625,
      "completions/mean_terminated_length": 253.20001220703125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8850036077201366,
      "epoch": 0.00916,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3857382833957672,
      "kl": 0.4309754427522421,
      "learning_rate": 7.501837696908025e-06,
      "loss": -0.0128,
      "num_tokens": 21552132.0,
      "reward": 0.5644669532775879,
      "reward_std": 0.7024494409561157,
      "rewards/rollout_reward_func/mean": 0.5644669532775879,
      "rewards/rollout_reward_func/std": 0.8011266589164734,
      "sampling/importance_sampling_ratio/max": 1.1797492504119873,
      "sampling/importance_sampling_ratio/mean": 1.0145529508590698,
      "sampling/importance_sampling_ratio/min": 0.9792538285255432,
      "sampling/sampling_logp_difference/max": 0.13399505615234375,
      "sampling/sampling_logp_difference/mean": 0.019814807921648026,
      "step": 916,
      "step_time": 7.647461837001174
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 4512.0,
      "completions/max_terminated_length": 4512.0,
      "completions/mean_length": 906.4375,
      "completions/mean_terminated_length": 922.0000610351562,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1957883536815643,
      "epoch": 0.00917,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7054538726806641,
      "kl": 0.48830094933509827,
      "learning_rate": 7.493295028311847e-06,
      "loss": -0.2382,
      "num_tokens": 21581519.0,
      "reward": 0.5560638904571533,
      "reward_std": 0.7256908416748047,
      "rewards/rollout_reward_func/mean": 0.5560638904571533,
      "rewards/rollout_reward_func/std": 0.7191237211227417,
      "sampling/importance_sampling_ratio/max": 1.198398232460022,
      "sampling/importance_sampling_ratio/mean": 0.994527280330658,
      "sampling/importance_sampling_ratio/min": 0.7407657504081726,
      "sampling/sampling_logp_difference/max": 0.20366859436035156,
      "sampling/sampling_logp_difference/mean": 0.02718549780547619,
      "step": 917,
      "step_time": 18.682448868998108
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1614.0,
      "completions/max_terminated_length": 905.0,
      "completions/mean_length": 379.625,
      "completions/mean_terminated_length": 258.7857360839844,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9576259655877948,
      "epoch": 0.00918,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0585336685180664,
      "kl": 0.7372055435553193,
      "learning_rate": 7.4849988517379864e-06,
      "loss": -0.0262,
      "num_tokens": 21601350.0,
      "reward": 0.4997327923774719,
      "reward_std": 0.8269989490509033,
      "rewards/rollout_reward_func/mean": 0.4997327923774719,
      "rewards/rollout_reward_func/std": 0.8029912114143372,
      "sampling/importance_sampling_ratio/max": 1.419249415397644,
      "sampling/importance_sampling_ratio/mean": 0.9604176878929138,
      "sampling/importance_sampling_ratio/min": 8.176670603619873e-13,
      "sampling/sampling_logp_difference/max": 19.340530395507812,
      "sampling/sampling_logp_difference/mean": 0.14869196712970734,
      "step": 918,
      "step_time": 10.659414175997881
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1708.0,
      "completions/max_terminated_length": 1708.0,
      "completions/mean_length": 586.6875,
      "completions/mean_terminated_length": 586.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4561714865267277,
      "epoch": 0.00919,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9275510311126709,
      "kl": 1.2585165202617645,
      "learning_rate": 7.476949266063266e-06,
      "loss": 0.0269,
      "num_tokens": 21625597.0,
      "reward": 0.440158486366272,
      "reward_std": 0.8788151741027832,
      "rewards/rollout_reward_func/mean": 0.440158486366272,
      "rewards/rollout_reward_func/std": 0.8829710483551025,
      "sampling/importance_sampling_ratio/max": 1.36922025680542,
      "sampling/importance_sampling_ratio/mean": 1.0277163982391357,
      "sampling/importance_sampling_ratio/min": 0.7417441010475159,
      "sampling/sampling_logp_difference/max": 0.19813776016235352,
      "sampling/sampling_logp_difference/mean": 0.02928227372467518,
      "step": 919,
      "step_time": 10.300592616000358
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.25,
      "completions/max_length": 2545.0,
      "completions/max_terminated_length": 2545.0,
      "completions/mean_length": 778.8125,
      "completions/mean_terminated_length": 665.6666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4501533173024654,
      "epoch": 0.0092,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9381486177444458,
      "kl": 0.498706191778183,
      "learning_rate": 7.4691463672255525e-06,
      "loss": 0.3647,
      "num_tokens": 21653270.0,
      "reward": 0.36258313059806824,
      "reward_std": 0.8164314031600952,
      "rewards/rollout_reward_func/mean": 0.36258313059806824,
      "rewards/rollout_reward_func/std": 0.8018656373023987,
      "sampling/importance_sampling_ratio/max": 1.9161971807479858,
      "sampling/importance_sampling_ratio/mean": 0.9534551501274109,
      "sampling/importance_sampling_ratio/min": 1.753973096709627e-27,
      "sampling/sampling_logp_difference/max": 18.165767669677734,
      "sampling/sampling_logp_difference/mean": 0.2794097363948822,
      "step": 920,
      "step_time": 14.131353471002512
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1648.0,
      "completions/max_terminated_length": 1648.0,
      "completions/mean_length": 630.6875,
      "completions/mean_terminated_length": 654.1428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9474610686302185,
      "epoch": 0.00921,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.56325763463974,
      "kl": 0.33852662704885006,
      "learning_rate": 7.461590248222611e-06,
      "loss": 0.0833,
      "num_tokens": 21677960.0,
      "reward": 0.7582535743713379,
      "reward_std": 0.4476691782474518,
      "rewards/rollout_reward_func/mean": 0.7582535743713379,
      "rewards/rollout_reward_func/std": 0.6606298685073853,
      "sampling/importance_sampling_ratio/max": 1.1404532194137573,
      "sampling/importance_sampling_ratio/mean": 1.0167580842971802,
      "sampling/importance_sampling_ratio/min": 0.8776476979255676,
      "sampling/sampling_logp_difference/max": 0.32285594940185547,
      "sampling/sampling_logp_difference/mean": 0.02678714692592621,
      "step": 921,
      "step_time": 10.174510041999383
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.3125,
      "completions/max_length": 1641.0,
      "completions/max_terminated_length": 799.0,
      "completions/mean_length": 573.9375,
      "completions/mean_terminated_length": 461.6363830566406,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5762027725577354,
      "epoch": 0.00922,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0231633186340332,
      "kl": 0.48799662478268147,
      "learning_rate": 7.454280999110993e-06,
      "loss": -0.2254,
      "num_tokens": 21702184.0,
      "reward": 0.36501461267471313,
      "reward_std": 0.819319486618042,
      "rewards/rollout_reward_func/mean": 0.36501461267471313,
      "rewards/rollout_reward_func/std": 0.802863359451294,
      "sampling/importance_sampling_ratio/max": 2.3272345066070557,
      "sampling/importance_sampling_ratio/mean": 1.0599923133850098,
      "sampling/importance_sampling_ratio/min": 1.924074011228072e-09,
      "sampling/sampling_logp_difference/max": 23.84836769104004,
      "sampling/sampling_logp_difference/mean": 0.12632055580615997,
      "step": 922,
      "step_time": 10.880198688002565
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1663.0,
      "completions/max_terminated_length": 1663.0,
      "completions/mean_length": 593.375,
      "completions/mean_terminated_length": 621.857177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2243758365511894,
      "epoch": 0.00923,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.7541025876998901,
      "kl": 0.7608963958919048,
      "learning_rate": 7.447218707004966e-06,
      "loss": 0.0695,
      "num_tokens": 21726467.0,
      "reward": 0.8644884824752808,
      "reward_std": 0.25103044509887695,
      "rewards/rollout_reward_func/mean": 0.8644884824752808,
      "rewards/rollout_reward_func/std": 0.37042948603630066,
      "sampling/importance_sampling_ratio/max": 1.115951657295227,
      "sampling/importance_sampling_ratio/mean": 0.9323912858963013,
      "sampling/importance_sampling_ratio/min": 3.5870521827239535e-18,
      "sampling/sampling_logp_difference/max": 19.46834945678711,
      "sampling/sampling_logp_difference/mean": 0.25364577770233154,
      "step": 923,
      "step_time": 10.767865566998807
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2713.0,
      "completions/max_terminated_length": 2713.0,
      "completions/mean_length": 563.875,
      "completions/mean_terminated_length": 540.3333740234375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0779541647061706,
      "epoch": 0.00924,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1293989419937134,
      "kl": 0.4025811795145273,
      "learning_rate": 7.440403456075477e-06,
      "loss": 0.1887,
      "num_tokens": 21749925.0,
      "reward": 0.69163978099823,
      "reward_std": 0.7217557430267334,
      "rewards/rollout_reward_func/mean": 0.69163978099823,
      "rewards/rollout_reward_func/std": 0.7000256776809692,
      "sampling/importance_sampling_ratio/max": 1.1607335805892944,
      "sampling/importance_sampling_ratio/mean": 0.9869091510772705,
      "sampling/importance_sampling_ratio/min": 0.49302518367767334,
      "sampling/sampling_logp_difference/max": 0.21123361587524414,
      "sampling/sampling_logp_difference/mean": 0.024501094594597816,
      "step": 924,
      "step_time": 13.13042078699982
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1729.0,
      "completions/max_terminated_length": 1729.0,
      "completions/mean_length": 660.9375,
      "completions/mean_terminated_length": 640.857177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1226611100137234,
      "epoch": 0.00925,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5156862139701843,
      "kl": 0.6851179236546159,
      "learning_rate": 7.433835327549145e-06,
      "loss": -0.0086,
      "num_tokens": 21775361.0,
      "reward": 0.6115769147872925,
      "reward_std": 0.5643194317817688,
      "rewards/rollout_reward_func/mean": 0.6115769147872925,
      "rewards/rollout_reward_func/std": 0.6259238123893738,
      "sampling/importance_sampling_ratio/max": 1.121882677078247,
      "sampling/importance_sampling_ratio/mean": 0.9424519538879395,
      "sampling/importance_sampling_ratio/min": 2.627967177204482e-09,
      "sampling/sampling_logp_difference/max": 20.458229064941406,
      "sampling/sampling_logp_difference/mean": 0.14450575411319733,
      "step": 925,
      "step_time": 10.993475043000217
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2567.0,
      "completions/max_terminated_length": 2567.0,
      "completions/mean_length": 736.0,
      "completions/mean_terminated_length": 590.6428833007812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9215019103139639,
      "epoch": 0.00926,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.31145110726356506,
      "kl": 0.4159590173512697,
      "learning_rate": 7.427514399707298e-06,
      "loss": -0.0881,
      "num_tokens": 21802005.0,
      "reward": 0.7297779321670532,
      "reward_std": 0.47415828704833984,
      "rewards/rollout_reward_func/mean": 0.7297779321670532,
      "rewards/rollout_reward_func/std": 0.48406365513801575,
      "sampling/importance_sampling_ratio/max": 1.1005873680114746,
      "sampling/importance_sampling_ratio/mean": 0.9677538871765137,
      "sampling/importance_sampling_ratio/min": 0.763094425201416,
      "sampling/sampling_logp_difference/max": 0.2339935302734375,
      "sampling/sampling_logp_difference/mean": 0.021205667406320572,
      "step": 926,
      "step_time": 12.961881677003475
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1815.0,
      "completions/max_terminated_length": 1815.0,
      "completions/mean_length": 880.5,
      "completions/mean_terminated_length": 880.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.360878974199295,
      "epoch": 0.00927,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5894932150840759,
      "kl": 0.7931026481091976,
      "learning_rate": 7.421440747885031e-06,
      "loss": -0.0017,
      "num_tokens": 21831765.0,
      "reward": 0.48163801431655884,
      "reward_std": 0.6439712047576904,
      "rewards/rollout_reward_func/mean": 0.48163801431655884,
      "rewards/rollout_reward_func/std": 0.6378288865089417,
      "sampling/importance_sampling_ratio/max": 1.246580719947815,
      "sampling/importance_sampling_ratio/mean": 0.9620636105537415,
      "sampling/importance_sampling_ratio/min": 0.6710092425346375,
      "sampling/sampling_logp_difference/max": 0.3129310607910156,
      "sampling/sampling_logp_difference/mean": 0.024504264816641808,
      "step": 927,
      "step_time": 11.517711353002596
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 4451.0,
      "completions/max_terminated_length": 2451.0,
      "completions/mean_length": 1017.4375,
      "completions/mean_terminated_length": 783.7142944335938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4174047484993935,
      "epoch": 0.00928,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8378195762634277,
      "kl": 0.44171001948416233,
      "learning_rate": 7.415614444470322e-06,
      "loss": 0.1745,
      "num_tokens": 21863495.0,
      "reward": 0.3586850166320801,
      "reward_std": 0.7309129238128662,
      "rewards/rollout_reward_func/mean": 0.3586850166320801,
      "rewards/rollout_reward_func/std": 0.7186984419822693,
      "sampling/importance_sampling_ratio/max": 1.2173033952713013,
      "sampling/importance_sampling_ratio/mean": 0.9171372652053833,
      "sampling/importance_sampling_ratio/min": 1.4540553205932838e-09,
      "sampling/sampling_logp_difference/max": 20.35857391357422,
      "sampling/sampling_logp_difference/mean": 0.11823946237564087,
      "step": 928,
      "step_time": 18.927746672001376
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1767.0,
      "completions/max_terminated_length": 1767.0,
      "completions/mean_length": 597.5625,
      "completions/mean_terminated_length": 493.23077392578125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3686841549351811,
      "epoch": 0.00929,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8347067832946777,
      "kl": 0.7293758243322372,
      "learning_rate": 7.41003555890315e-06,
      "loss": 0.0363,
      "num_tokens": 21887181.0,
      "reward": 0.5554860234260559,
      "reward_std": 0.7175382375717163,
      "rewards/rollout_reward_func/mean": 0.5554860234260559,
      "rewards/rollout_reward_func/std": 0.7233709692955017,
      "sampling/importance_sampling_ratio/max": 1.2702950239181519,
      "sampling/importance_sampling_ratio/mean": 0.9194525480270386,
      "sampling/importance_sampling_ratio/min": 6.597641988292935e-09,
      "sampling/sampling_logp_difference/max": 19.067825317382812,
      "sampling/sampling_logp_difference/mean": 0.12523064017295837,
      "step": 929,
      "step_time": 10.850483162001183
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2495.0,
      "completions/max_terminated_length": 2495.0,
      "completions/mean_length": 940.3125,
      "completions/mean_terminated_length": 940.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2467999905347824,
      "epoch": 0.0093,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7135283946990967,
      "kl": 1.613026924431324,
      "learning_rate": 7.404704157674698e-06,
      "loss": -0.0238,
      "num_tokens": 21917160.0,
      "reward": 0.7534105777740479,
      "reward_std": 0.5461990237236023,
      "rewards/rollout_reward_func/mean": 0.7534105777740479,
      "rewards/rollout_reward_func/std": 0.5696120858192444,
      "sampling/importance_sampling_ratio/max": 1.3275927305221558,
      "sampling/importance_sampling_ratio/mean": 0.9463579654693604,
      "sampling/importance_sampling_ratio/min": 1.5325456181880526e-10,
      "sampling/sampling_logp_difference/max": 20.303138732910156,
      "sampling/sampling_logp_difference/mean": 0.17929981648921967,
      "step": 930,
      "step_time": 13.030415182001889
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 971.0,
      "completions/max_terminated_length": 971.0,
      "completions/mean_length": 291.4375,
      "completions/mean_terminated_length": 291.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.011501308530569,
      "epoch": 0.00931,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7269600033760071,
      "kl": 0.35910916700959206,
      "learning_rate": 7.399620304326519e-06,
      "loss": -0.0265,
      "num_tokens": 21935710.0,
      "reward": 0.6029990911483765,
      "reward_std": 0.6371184587478638,
      "rewards/rollout_reward_func/mean": 0.6029990911483765,
      "rewards/rollout_reward_func/std": 0.6409536004066467,
      "sampling/importance_sampling_ratio/max": 1.428518295288086,
      "sampling/importance_sampling_ratio/mean": 1.0477371215820312,
      "sampling/importance_sampling_ratio/min": 0.9860415458679199,
      "sampling/sampling_logp_difference/max": 0.31217169761657715,
      "sampling/sampling_logp_difference/mean": 0.025960223749279976,
      "step": 931,
      "step_time": 7.831203928000832
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2513.0,
      "completions/max_terminated_length": 2513.0,
      "completions/mean_length": 815.9375,
      "completions/mean_terminated_length": 682.7857666015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.4875488840043545,
      "epoch": 0.00932,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8612878918647766,
      "kl": 0.5463398490101099,
      "learning_rate": 7.394784059449813e-06,
      "loss": 0.1129,
      "num_tokens": 21963928.0,
      "reward": 0.6006250381469727,
      "reward_std": 0.533503532409668,
      "rewards/rollout_reward_func/mean": 0.6006250381469727,
      "rewards/rollout_reward_func/std": 0.5331810116767883,
      "sampling/importance_sampling_ratio/max": 1.3346319198608398,
      "sampling/importance_sampling_ratio/mean": 0.9812444448471069,
      "sampling/importance_sampling_ratio/min": 0.4736875593662262,
      "sampling/sampling_logp_difference/max": 0.15204215049743652,
      "sampling/sampling_logp_difference/mean": 0.024497125297784805,
      "step": 932,
      "step_time": 13.453231855997728
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1678.0,
      "completions/max_terminated_length": 1657.0,
      "completions/mean_length": 660.0,
      "completions/mean_terminated_length": 524.7857666015625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.3184725791215897,
      "epoch": 0.00933,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.42664217948913574,
      "kl": 0.720109143294394,
      "learning_rate": 7.390195480684698e-06,
      "loss": -0.0189,
      "num_tokens": 21989191.0,
      "reward": 0.7345342040061951,
      "reward_std": 0.2841508686542511,
      "rewards/rollout_reward_func/mean": 0.7345342040061951,
      "rewards/rollout_reward_func/std": 0.4752768874168396,
      "sampling/importance_sampling_ratio/max": 1.247147798538208,
      "sampling/importance_sampling_ratio/mean": 0.9460393190383911,
      "sampling/importance_sampling_ratio/min": 2.1331059141260766e-09,
      "sampling/sampling_logp_difference/max": 19.941804885864258,
      "sampling/sampling_logp_difference/mean": 0.1821393072605133,
      "step": 933,
      "step_time": 10.225670503994479
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1749.0,
      "completions/max_terminated_length": 1743.0,
      "completions/mean_length": 525.9375,
      "completions/mean_terminated_length": 444.4000244140625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1991154849529266,
      "epoch": 0.00934,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2691125869750977,
      "kl": 0.42402342706918716,
      "learning_rate": 7.385854622719507e-06,
      "loss": -0.2552,
      "num_tokens": 22012305.0,
      "reward": 0.5505916476249695,
      "reward_std": 0.7064019441604614,
      "rewards/rollout_reward_func/mean": 0.5505916476249695,
      "rewards/rollout_reward_func/std": 0.7297753691673279,
      "sampling/importance_sampling_ratio/max": 2.0745084285736084,
      "sampling/importance_sampling_ratio/mean": 1.0241830348968506,
      "sampling/importance_sampling_ratio/min": 0.5395413637161255,
      "sampling/sampling_logp_difference/max": 0.17245268821716309,
      "sampling/sampling_logp_difference/mean": 0.028339963406324387,
      "step": 934,
      "step_time": 10.508139764002408
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2621.0,
      "completions/max_terminated_length": 2621.0,
      "completions/mean_length": 682.5625,
      "completions/mean_terminated_length": 687.0667114257812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2216514013707638,
      "epoch": 0.00935,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.059267282485962,
      "kl": 0.6724894940853119,
      "learning_rate": 7.3817615372901485e-06,
      "loss": 0.0928,
      "num_tokens": 22037975.0,
      "reward": 0.6821403503417969,
      "reward_std": 0.6025813817977905,
      "rewards/rollout_reward_func/mean": 0.6821403503417969,
      "rewards/rollout_reward_func/std": 0.5925062894821167,
      "sampling/importance_sampling_ratio/max": 1.249484658241272,
      "sampling/importance_sampling_ratio/mean": 1.0269086360931396,
      "sampling/importance_sampling_ratio/min": 0.8592435121536255,
      "sampling/sampling_logp_difference/max": 0.19988059997558594,
      "sampling/sampling_logp_difference/mean": 0.02258341945707798,
      "step": 935,
      "step_time": 12.71358796000277
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2609.0,
      "completions/max_terminated_length": 2609.0,
      "completions/mean_length": 552.5625,
      "completions/mean_terminated_length": 552.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7531117033213377,
      "epoch": 0.00936,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.27935758233070374,
      "kl": 0.9695424139499664,
      "learning_rate": 7.377916273179499e-06,
      "loss": 0.033,
      "num_tokens": 22061168.0,
      "reward": 0.47674301266670227,
      "reward_std": 0.6659535765647888,
      "rewards/rollout_reward_func/mean": 0.47674301266670227,
      "rewards/rollout_reward_func/std": 0.6522634625434875,
      "sampling/importance_sampling_ratio/max": 1.100854754447937,
      "sampling/importance_sampling_ratio/mean": 0.9293348789215088,
      "sampling/importance_sampling_ratio/min": 3.140082638353192e-11,
      "sampling/sampling_logp_difference/max": 24.127408981323242,
      "sampling/sampling_logp_difference/mean": 0.34962764382362366,
      "step": 936,
      "step_time": 12.03118005899887
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1622.0,
      "completions/max_terminated_length": 1622.0,
      "completions/mean_length": 566.6875,
      "completions/mean_terminated_length": 550.6666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.191943185403943,
      "epoch": 0.00937,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.10401836782693863,
      "kl": 0.4724997617304325,
      "learning_rate": 7.3743188762167965e-06,
      "loss": 0.0073,
      "num_tokens": 22084596.0,
      "reward": 0.932991623878479,
      "reward_std": 0.18952836096286774,
      "rewards/rollout_reward_func/mean": 0.932991623878479,
      "rewards/rollout_reward_func/std": 0.26803359389305115,
      "sampling/importance_sampling_ratio/max": 1.8026518821716309,
      "sampling/importance_sampling_ratio/mean": 1.0627552270889282,
      "sampling/importance_sampling_ratio/min": 0.6956905126571655,
      "sampling/sampling_logp_difference/max": 0.1355518102645874,
      "sampling/sampling_logp_difference/mean": 0.02446575090289116,
      "step": 937,
      "step_time": 10.107435746998817
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 3608.0,
      "completions/max_terminated_length": 1734.0,
      "completions/mean_length": 695.3125,
      "completions/mean_terminated_length": 477.21429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1752154491841793,
      "epoch": 0.00938,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5469532608985901,
      "kl": 0.6869704630225897,
      "learning_rate": 7.370969389277123e-06,
      "loss": -0.0507,
      "num_tokens": 22110915.0,
      "reward": 0.3138445317745209,
      "reward_std": 0.7903960943222046,
      "rewards/rollout_reward_func/mean": 0.3138445317745209,
      "rewards/rollout_reward_func/std": 0.7866000533103943,
      "sampling/importance_sampling_ratio/max": 1.1757632493972778,
      "sampling/importance_sampling_ratio/mean": 0.911155104637146,
      "sampling/importance_sampling_ratio/min": 4.465717956558292e-14,
      "sampling/sampling_logp_difference/max": 9.136783599853516,
      "sampling/sampling_logp_difference/mean": 0.1481875479221344,
      "step": 938,
      "step_time": 16.41168698099864
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1709.0,
      "completions/max_terminated_length": 1709.0,
      "completions/mean_length": 720.0625,
      "completions/mean_terminated_length": 720.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0182040557265282,
      "epoch": 0.00939,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.27784213423728943,
      "kl": 0.6447312906384468,
      "learning_rate": 7.3678678522808635e-06,
      "loss": -0.0052,
      "num_tokens": 22138014.0,
      "reward": 0.5431214570999146,
      "reward_std": 0.6553395986557007,
      "rewards/rollout_reward_func/mean": 0.5431214570999146,
      "rewards/rollout_reward_func/std": 0.6418570876121521,
      "sampling/importance_sampling_ratio/max": 1.2361091375350952,
      "sampling/importance_sampling_ratio/mean": 0.9879679679870605,
      "sampling/importance_sampling_ratio/min": 0.8301875591278076,
      "sampling/sampling_logp_difference/max": 0.18427610397338867,
      "sampling/sampling_logp_difference/mean": 0.024791153147816658,
      "step": 939,
      "step_time": 10.49327785999958
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1784.0,
      "completions/max_terminated_length": 1784.0,
      "completions/mean_length": 763.9375,
      "completions/mean_terminated_length": 810.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2880686223506927,
      "epoch": 0.0094,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6450246572494507,
      "kl": 0.8672529999166727,
      "learning_rate": 7.365014302193262e-06,
      "loss": -0.0354,
      "num_tokens": 22165350.0,
      "reward": 0.3765874207019806,
      "reward_std": 0.9027256965637207,
      "rewards/rollout_reward_func/mean": 0.3765874207019806,
      "rewards/rollout_reward_func/std": 0.8721551299095154,
      "sampling/importance_sampling_ratio/max": 1.2561724185943604,
      "sampling/importance_sampling_ratio/mean": 0.8746622204780579,
      "sampling/importance_sampling_ratio/min": 2.3831940110588534e-13,
      "sampling/sampling_logp_difference/max": 11.573331832885742,
      "sampling/sampling_logp_difference/mean": 0.14953207969665527,
      "step": 940,
      "step_time": 11.353113205001137
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 2626.0,
      "completions/max_terminated_length": 2626.0,
      "completions/mean_length": 552.4375,
      "completions/mean_terminated_length": 511.71429443359375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9311306402087212,
      "epoch": 0.00941,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.588034987449646,
      "kl": 0.3703209077939391,
      "learning_rate": 7.362408773023957e-06,
      "loss": 0.086,
      "num_tokens": 22187977.0,
      "reward": 0.8128387928009033,
      "reward_std": 0.3646600842475891,
      "rewards/rollout_reward_func/mean": 0.8128387928009033,
      "rewards/rollout_reward_func/std": 0.5344045758247375,
      "sampling/importance_sampling_ratio/max": 2.523348808288574,
      "sampling/importance_sampling_ratio/mean": 1.075166940689087,
      "sampling/importance_sampling_ratio/min": 0.7499494552612305,
      "sampling/sampling_logp_difference/max": 0.2484269142150879,
      "sampling/sampling_logp_difference/mean": 0.021085303276777267,
      "step": 941,
      "step_time": 12.739953676998994
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 2468.0,
      "completions/max_terminated_length": 2468.0,
      "completions/mean_length": 579.5625,
      "completions/mean_terminated_length": 616.0667114257812,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0227010995149612,
      "epoch": 0.00942,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4521440267562866,
      "kl": 3.2357364613562822,
      "learning_rate": 7.360051295826582e-06,
      "loss": -0.0448,
      "num_tokens": 22211570.0,
      "reward": 0.4956478476524353,
      "reward_std": 0.8264386653900146,
      "rewards/rollout_reward_func/mean": 0.4956478476524353,
      "rewards/rollout_reward_func/std": 0.8121087551116943,
      "sampling/importance_sampling_ratio/max": 1.133670687675476,
      "sampling/importance_sampling_ratio/mean": 0.9619743824005127,
      "sampling/importance_sampling_ratio/min": 0.6059390902519226,
      "sampling/sampling_logp_difference/max": 0.20134878158569336,
      "sampling/sampling_logp_difference/mean": 0.024113085120916367,
      "step": 942,
      "step_time": 12.01624893799817
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1683.0,
      "completions/max_terminated_length": 1683.0,
      "completions/mean_length": 537.6875,
      "completions/mean_terminated_length": 523.6666870117188,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2072043642401695,
      "epoch": 0.00943,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5466581583023071,
      "kl": 0.4850291069597006,
      "learning_rate": 7.3579418986984045e-06,
      "loss": -0.0508,
      "num_tokens": 22234596.0,
      "reward": 0.4333139955997467,
      "reward_std": 0.7302460670471191,
      "rewards/rollout_reward_func/mean": 0.4333139955997467,
      "rewards/rollout_reward_func/std": 0.8023877143859863,
      "sampling/importance_sampling_ratio/max": 1.081651210784912,
      "sampling/importance_sampling_ratio/mean": 0.9523353576660156,
      "sampling/importance_sampling_ratio/min": 0.5488268733024597,
      "sampling/sampling_logp_difference/max": 0.2212257981300354,
      "sampling/sampling_logp_difference/mean": 0.025655653327703476,
      "step": 943,
      "step_time": 10.119044536999354
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.1875,
      "completions/max_length": 1664.0,
      "completions/max_terminated_length": 1664.0,
      "completions/mean_length": 420.5,
      "completions/mean_terminated_length": 381.7692565917969,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5690366867929697,
      "epoch": 0.00944,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.5529922842979431,
      "kl": 0.2676049116998911,
      "learning_rate": 7.356080606779981e-06,
      "loss": 0.1191,
      "num_tokens": 22254842.0,
      "reward": 0.9329558610916138,
      "reward_std": 0.18962953984737396,
      "rewards/rollout_reward_func/mean": 0.9329558610916138,
      "rewards/rollout_reward_func/std": 0.2681766748428345,
      "sampling/importance_sampling_ratio/max": 1.0869883298873901,
      "sampling/importance_sampling_ratio/mean": 1.0159012079238892,
      "sampling/importance_sampling_ratio/min": 0.914746105670929,
      "sampling/sampling_logp_difference/max": 0.12207841873168945,
      "sampling/sampling_logp_difference/mean": 0.011889362707734108,
      "step": 944,
      "step_time": 10.03134844700071
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1623.0,
      "completions/max_terminated_length": 1623.0,
      "completions/mean_length": 715.75,
      "completions/mean_terminated_length": 705.857177734375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2439613677561283,
      "epoch": 0.00945,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0860259532928467,
      "kl": 0.7668678313493729,
      "learning_rate": 7.354467442254858e-06,
      "loss": 0.1113,
      "num_tokens": 22281047.0,
      "reward": 0.6207582950592041,
      "reward_std": 0.7420318126678467,
      "rewards/rollout_reward_func/mean": 0.6207582950592041,
      "rewards/rollout_reward_func/std": 0.7169429063796997,
      "sampling/importance_sampling_ratio/max": 1.2290599346160889,
      "sampling/importance_sampling_ratio/mean": 0.986603856086731,
      "sampling/importance_sampling_ratio/min": 0.7666587233543396,
      "sampling/sampling_logp_difference/max": 0.1351637840270996,
      "sampling/sampling_logp_difference/mean": 0.019204601645469666,
      "step": 945,
      "step_time": 10.956950654997854
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1753.0,
      "completions/max_terminated_length": 1753.0,
      "completions/mean_length": 760.5625,
      "completions/mean_terminated_length": 700.6000366210938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2675348743796349,
      "epoch": 0.00946,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6446217894554138,
      "kl": 0.5534827839583158,
      "learning_rate": 7.353102424349307e-06,
      "loss": -0.0477,
      "num_tokens": 22308767.0,
      "reward": 0.36645299196243286,
      "reward_std": 0.8270710706710815,
      "rewards/rollout_reward_func/mean": 0.36645299196243286,
      "rewards/rollout_reward_func/std": 0.8028416633605957,
      "sampling/importance_sampling_ratio/max": 1.1810249090194702,
      "sampling/importance_sampling_ratio/mean": 1.0044605731964111,
      "sampling/importance_sampling_ratio/min": 0.7456684708595276,
      "sampling/sampling_logp_difference/max": 0.16882610321044922,
      "sampling/sampling_logp_difference/mean": 0.026258252561092377,
      "step": 946,
      "step_time": 11.163938218000112
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1762.0,
      "completions/max_terminated_length": 1762.0,
      "completions/mean_length": 453.9375,
      "completions/mean_terminated_length": 453.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.212828941643238,
      "epoch": 0.00947,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2676584720611572,
      "kl": 0.6713163331151009,
      "learning_rate": 7.351985569332115e-06,
      "loss": -0.0293,
      "num_tokens": 22330358.0,
      "reward": 0.46507012844085693,
      "reward_std": 0.6587989330291748,
      "rewards/rollout_reward_func/mean": 0.46507012844085693,
      "rewards/rollout_reward_func/std": 0.6616592407226562,
      "sampling/importance_sampling_ratio/max": 1.0175081491470337,
      "sampling/importance_sampling_ratio/mean": 0.9027461409568787,
      "sampling/importance_sampling_ratio/min": 4.652672969118754e-10,
      "sampling/sampling_logp_difference/max": 21.60713005065918,
      "sampling/sampling_logp_difference/mean": 0.29156628251075745,
      "step": 947,
      "step_time": 9.896149609998247
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 1851.0,
      "completions/max_terminated_length": 867.0,
      "completions/mean_length": 479.375,
      "completions/mean_terminated_length": 387.933349609375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.109714336693287,
      "epoch": 0.00948,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7191804647445679,
      "kl": 0.4527799990028143,
      "learning_rate": 7.351116890514361e-06,
      "loss": 0.052,
      "num_tokens": 22352411.0,
      "reward": 0.6213049292564392,
      "reward_std": 0.6387016773223877,
      "rewards/rollout_reward_func/mean": 0.6213049292564392,
      "rewards/rollout_reward_func/std": 0.7174550294876099,
      "sampling/importance_sampling_ratio/max": 1.484837532043457,
      "sampling/importance_sampling_ratio/mean": 1.0243580341339111,
      "sampling/importance_sampling_ratio/min": 0.6771385073661804,
      "sampling/sampling_logp_difference/max": 0.13318252563476562,
      "sampling/sampling_logp_difference/mean": 0.024372536689043045,
      "step": 948,
      "step_time": 10.647414021001168
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.125,
      "completions/max_length": 1743.0,
      "completions/max_terminated_length": 1743.0,
      "completions/mean_length": 724.4375,
      "completions/mean_terminated_length": 646.7142944335938,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.5495899803936481,
      "epoch": 0.00949,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8021070957183838,
      "kl": 0.7730380799621344,
      "learning_rate": 7.350496398249265e-06,
      "loss": -0.1117,
      "num_tokens": 22379214.0,
      "reward": 0.2987158000469208,
      "reward_std": 0.812942624092102,
      "rewards/rollout_reward_func/mean": 0.2987158000469208,
      "rewards/rollout_reward_func/std": 0.7892395853996277,
      "sampling/importance_sampling_ratio/max": 1.468388319015503,
      "sampling/importance_sampling_ratio/mean": 0.9712381958961487,
      "sampling/importance_sampling_ratio/min": 0.501706600189209,
      "sampling/sampling_logp_difference/max": 0.15959501266479492,
      "sampling/sampling_logp_difference/mean": 0.025095529854297638,
      "step": 949,
      "step_time": 11.25365519099978
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 856.0,
      "completions/max_terminated_length": 856.0,
      "completions/mean_length": 387.125,
      "completions/mean_terminated_length": 356.8000183105469,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0308204963803291,
      "epoch": 0.0095,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6873910427093506,
      "kl": 0.4297216609120369,
      "learning_rate": 7.350124099932081e-06,
      "loss": -0.117,
      "num_tokens": 22399435.0,
      "reward": 0.7956846952438354,
      "reward_std": 0.44419825077056885,
      "rewards/rollout_reward_func/mean": 0.7956846952438354,
      "rewards/rollout_reward_func/std": 0.43932199478149414,
      "sampling/importance_sampling_ratio/max": 1.7810028791427612,
      "sampling/importance_sampling_ratio/mean": 1.0276955366134644,
      "sampling/importance_sampling_ratio/min": 0.7516305446624756,
      "sampling/sampling_logp_difference/max": 0.21458172798156738,
      "sampling/sampling_logp_difference/mean": 0.024093022570014,
      "step": 950,
      "step_time": 7.706611600999167
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 950,
  "num_input_tokens_seen": 22399435,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}