{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.0015,
  "eval_steps": 500,
  "global_step": 75,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 555.0,
      "completions/max_terminated_length": 555.0,
      "completions/mean_length": 48.5,
      "completions/mean_terminated_length": 48.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0614224635064602,
      "epoch": 2e-05,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.9696364402770996,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": -0.0044,
      "num_tokens": 41003.0,
      "reward": -0.28125,
      "reward_std": 0.09983479976654053,
      "rewards/rollout_reward_func/mean": -0.28125,
      "rewards/rollout_reward_func/std": 0.12556324899196625,
      "sampling/importance_sampling_ratio/max": 1.2598837614059448,
      "sampling/importance_sampling_ratio/mean": 1.0303902626037598,
      "sampling/importance_sampling_ratio/min": 0.6541365385055542,
      "sampling/sampling_logp_difference/max": 0.3427009582519531,
      "sampling/sampling_logp_difference/mean": 0.04384341090917587,
      "step": 1,
      "step_time": 12.103902025002753
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 528.0,
      "completions/max_terminated_length": 528.0,
      "completions/mean_length": 82.8125,
      "completions/mean_terminated_length": 82.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1359193846583366,
      "epoch": 4e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.7091009616851807,
      "kl": 0.0,
      "learning_rate": 2.8571428571428575e-07,
      "loss": 0.1304,
      "num_tokens": 84006.0,
      "reward": -0.3125,
      "reward_std": 0.10659779608249664,
      "rewards/rollout_reward_func/mean": -0.3125,
      "rewards/rollout_reward_func/std": 0.1099853366613388,
      "sampling/importance_sampling_ratio/max": 1.5958999395370483,
      "sampling/importance_sampling_ratio/mean": 1.0259093046188354,
      "sampling/importance_sampling_ratio/min": 0.7485636472702026,
      "sampling/sampling_logp_difference/max": 0.39561665058135986,
      "sampling/sampling_logp_difference/mean": 0.045396752655506134,
      "step": 2,
      "step_time": 11.933418574000825
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 46.65625,
      "completions/mean_terminated_length": 46.65625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0438250675797462,
      "epoch": 6e-05,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.9018235802650452,
      "kl": 0.002862276043742895,
      "learning_rate": 5.714285714285715e-07,
      "loss": -0.0006,
      "num_tokens": 123834.0,
      "reward": -0.2875000238418579,
      "reward_std": 0.08960890024900436,
      "rewards/rollout_reward_func/mean": -0.2875000238418579,
      "rewards/rollout_reward_func/std": 0.1099853366613388,
      "sampling/importance_sampling_ratio/max": 1.2291228771209717,
      "sampling/importance_sampling_ratio/mean": 0.9879076480865479,
      "sampling/importance_sampling_ratio/min": 0.7175705432891846,
      "sampling/sampling_logp_difference/max": 0.278971791267395,
      "sampling/sampling_logp_difference/mean": 0.042785435914993286,
      "step": 3,
      "step_time": 11.816223641013494
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 537.0,
      "completions/max_terminated_length": 537.0,
      "completions/mean_length": 33.03125,
      "completions/mean_terminated_length": 33.03125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.092379666864872,
      "epoch": 8e-05,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.7882914543151855,
      "kl": 0.0027756020426750183,
      "learning_rate": 8.571428571428572e-07,
      "loss": 0.0111,
      "num_tokens": 164325.0,
      "reward": -0.3031250238418579,
      "reward_std": 0.051582735031843185,
      "rewards/rollout_reward_func/mean": -0.3031250238418579,
      "rewards/rollout_reward_func/std": 0.07398506253957748,
      "sampling/importance_sampling_ratio/max": 1.309051513671875,
      "sampling/importance_sampling_ratio/mean": 1.0035574436187744,
      "sampling/importance_sampling_ratio/min": 0.8704667091369629,
      "sampling/sampling_logp_difference/max": 0.2693389654159546,
      "sampling/sampling_logp_difference/mean": 0.03194209560751915,
      "step": 4,
      "step_time": 10.954096017005213
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 600.0,
      "completions/max_terminated_length": 600.0,
      "completions/mean_length": 55.375,
      "completions/mean_terminated_length": 55.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.125911958515644,
      "epoch": 0.0001,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2943179607391357,
      "kl": 0.002716648392379284,
      "learning_rate": 1.142857142857143e-06,
      "loss": 0.0062,
      "num_tokens": 205685.0,
      "reward": -0.27812498807907104,
      "reward_std": 0.15026018023490906,
      "rewards/rollout_reward_func/mean": -0.27812498807907104,
      "rewards/rollout_reward_func/std": 0.1717732548713684,
      "sampling/importance_sampling_ratio/max": 1.307915210723877,
      "sampling/importance_sampling_ratio/mean": 1.0372111797332764,
      "sampling/importance_sampling_ratio/min": 0.8186781406402588,
      "sampling/sampling_logp_difference/max": 0.20126968622207642,
      "sampling/sampling_logp_difference/mean": 0.038129210472106934,
      "step": 5,
      "step_time": 12.542300894005166
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 438.0,
      "completions/max_terminated_length": 438.0,
      "completions/mean_length": 66.71875,
      "completions/mean_terminated_length": 66.71875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.11374481767416,
      "epoch": 0.00012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2801172733306885,
      "kl": 0.004243878647685051,
      "learning_rate": 1.4285714285714286e-06,
      "loss": -0.0063,
      "num_tokens": 247433.0,
      "reward": -0.26250001788139343,
      "reward_std": 0.18602894246578217,
      "rewards/rollout_reward_func/mean": -0.26250001788139343,
      "rewards/rollout_reward_func/std": 0.20280294120311737,
      "sampling/importance_sampling_ratio/max": 1.240354061126709,
      "sampling/importance_sampling_ratio/mean": 0.9942352771759033,
      "sampling/importance_sampling_ratio/min": 0.7674867510795593,
      "sampling/sampling_logp_difference/max": 0.23222827911376953,
      "sampling/sampling_logp_difference/mean": 0.049182578921318054,
      "step": 6,
      "step_time": 10.926656089985045
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 600.0,
      "completions/max_terminated_length": 600.0,
      "completions/mean_length": 36.0,
      "completions/mean_terminated_length": 36.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1214893609285355,
      "epoch": 0.00014,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.031197428703308,
      "kl": 0.0014154938980937004,
      "learning_rate": 1.7142857142857145e-06,
      "loss": 0.0136,
      "num_tokens": 287715.0,
      "reward": -0.28437501192092896,
      "reward_std": 0.09690804779529572,
      "rewards/rollout_reward_func/mean": -0.28437501192092896,
      "rewards/rollout_reward_func/std": 0.12727764248847961,
      "sampling/importance_sampling_ratio/max": 1.2321161031723022,
      "sampling/importance_sampling_ratio/mean": 1.0050127506256104,
      "sampling/importance_sampling_ratio/min": 0.7858185172080994,
      "sampling/sampling_logp_difference/max": 0.24108994007110596,
      "sampling/sampling_logp_difference/mean": 0.049698200076818466,
      "step": 7,
      "step_time": 11.036665410996648
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234375,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 169.875,
      "completions/mean_terminated_length": 169.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0151596441864967,
      "epoch": 0.00016,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.4127973318099976,
      "kl": 0.003077693283557892,
      "learning_rate": 2.0000000000000003e-06,
      "loss": -0.0156,
      "num_tokens": 333102.0,
      "reward": -0.2656250298023224,
      "reward_std": 0.1774883270263672,
      "rewards/rollout_reward_func/mean": -0.2656250298023224,
      "rewards/rollout_reward_func/std": 0.19110776484012604,
      "sampling/importance_sampling_ratio/max": 1.46891450881958,
      "sampling/importance_sampling_ratio/mean": 0.9800823926925659,
      "sampling/importance_sampling_ratio/min": 0.6549047827720642,
      "sampling/sampling_logp_difference/max": 0.41107726097106934,
      "sampling/sampling_logp_difference/mean": 0.04922608286142349,
      "step": 8,
      "step_time": 13.175797829018848
    },
    {
      "clip_ratio/high_max": 0.0625,
      "clip_ratio/high_mean": 0.03125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 357.0,
      "completions/max_terminated_length": 357.0,
      "completions/mean_length": 23.0625,
      "completions/mean_terminated_length": 23.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0669935643672943,
      "epoch": 0.00018,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3136646747589111,
      "kl": 0.0034769903868436813,
      "learning_rate": 2.285714285714286e-06,
      "loss": 0.0207,
      "num_tokens": 371700.0,
      "reward": -0.2718750238418579,
      "reward_std": 0.14089259505271912,
      "rewards/rollout_reward_func/mean": -0.2718750238418579,
      "rewards/rollout_reward_func/std": 0.1590990275144577,
      "sampling/importance_sampling_ratio/max": 1.1991398334503174,
      "sampling/importance_sampling_ratio/mean": 1.0174425840377808,
      "sampling/importance_sampling_ratio/min": 0.7648656368255615,
      "sampling/sampling_logp_difference/max": 0.2679147720336914,
      "sampling/sampling_logp_difference/mean": 0.035627588629722595,
      "step": 9,
      "step_time": 10.69127149101405
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 474.0,
      "completions/max_terminated_length": 474.0,
      "completions/mean_length": 54.09375,
      "completions/mean_terminated_length": 54.09375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.034070409834385,
      "epoch": 0.0002,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.1416436433792114,
      "kl": 0.004923981614410877,
      "learning_rate": 2.571428571428571e-06,
      "loss": 0.0136,
      "num_tokens": 412189.0,
      "reward": -0.25,
      "reward_std": 0.15856999158859253,
      "rewards/rollout_reward_func/mean": -0.25,
      "rewards/rollout_reward_func/std": 0.20478154718875885,
      "sampling/importance_sampling_ratio/max": 1.3213411569595337,
      "sampling/importance_sampling_ratio/mean": 0.9911450147628784,
      "sampling/importance_sampling_ratio/min": 0.5153354406356812,
      "sampling/sampling_logp_difference/max": 0.45181727409362793,
      "sampling/sampling_logp_difference/mean": 0.057384196668863297,
      "step": 10,
      "step_time": 10.959724056992854
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 627.0,
      "completions/max_terminated_length": 627.0,
      "completions/mean_length": 74.1875,
      "completions/mean_terminated_length": 74.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0892708413302898,
      "epoch": 0.00022,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.0666534900665283,
      "kl": 0.0030549648217856884,
      "learning_rate": 2.8571428571428573e-06,
      "loss": 0.0054,
      "num_tokens": 454325.0,
      "reward": -0.2750000059604645,
      "reward_std": 0.1340271234512329,
      "rewards/rollout_reward_func/mean": -0.2750000059604645,
      "rewards/rollout_reward_func/std": 0.1586231142282486,
      "sampling/importance_sampling_ratio/max": 1.3382775783538818,
      "sampling/importance_sampling_ratio/mean": 1.0159631967544556,
      "sampling/importance_sampling_ratio/min": 0.7966440320014954,
      "sampling/sampling_logp_difference/max": 0.2912829518318176,
      "sampling/sampling_logp_difference/mean": 0.04749169573187828,
      "step": 11,
      "step_time": 12.903910137996718
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 663.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 61.125,
      "completions/mean_terminated_length": 61.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0594853572547436,
      "epoch": 0.00024,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.0740004777908325,
      "kl": 0.004419071599841118,
      "learning_rate": 3.142857142857143e-06,
      "loss": -0.0003,
      "num_tokens": 494660.0,
      "reward": -0.23749999701976776,
      "reward_std": 0.1643490195274353,
      "rewards/rollout_reward_func/mean": -0.23749999701976776,
      "rewards/rollout_reward_func/std": 0.18447834253311157,
      "sampling/importance_sampling_ratio/max": 1.1906286478042603,
      "sampling/importance_sampling_ratio/mean": 1.0346179008483887,
      "sampling/importance_sampling_ratio/min": 0.7173823118209839,
      "sampling/sampling_logp_difference/max": 0.3321366310119629,
      "sampling/sampling_logp_difference/mean": 0.03979075700044632,
      "step": 12,
      "step_time": 11.274669903999893
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 411.0,
      "completions/max_terminated_length": 411.0,
      "completions/mean_length": 14.78125,
      "completions/mean_terminated_length": 14.78125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0981016233563423,
      "epoch": 0.00026,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.070846438407898,
      "kl": 0.002780897542834282,
      "learning_rate": 3.428571428571429e-06,
      "loss": -0.008,
      "num_tokens": 535121.0,
      "reward": -0.28437501192092896,
      "reward_std": 0.04419417679309845,
      "rewards/rollout_reward_func/mean": -0.28437501192092896,
      "rewards/rollout_reward_func/std": 0.05741403251886368,
      "sampling/importance_sampling_ratio/max": 1.2942397594451904,
      "sampling/importance_sampling_ratio/mean": 0.9831522703170776,
      "sampling/importance_sampling_ratio/min": 0.7195752859115601,
      "sampling/sampling_logp_difference/max": 0.3291201591491699,
      "sampling/sampling_logp_difference/mean": 0.04754544422030449,
      "step": 13,
      "step_time": 10.63663990200439
    },
    {
      "clip_ratio/high_max": 0.0625,
      "clip_ratio/high_mean": 0.03125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 519.0,
      "completions/max_terminated_length": 519.0,
      "completions/mean_length": 49.1875,
      "completions/mean_terminated_length": 49.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0206766799092293,
      "epoch": 0.00028,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5201469659805298,
      "kl": 0.002846735529601574,
      "learning_rate": 3.7142857142857146e-06,
      "loss": -0.0247,
      "num_tokens": 574733.0,
      "reward": -0.27500003576278687,
      "reward_std": 0.10820101201534271,
      "rewards/rollout_reward_func/mean": -0.27500003576278687,
      "rewards/rollout_reward_func/std": 0.11359237134456635,
      "sampling/importance_sampling_ratio/max": 1.3940155506134033,
      "sampling/importance_sampling_ratio/mean": 1.030958890914917,
      "sampling/importance_sampling_ratio/min": 0.591517448425293,
      "sampling/sampling_logp_difference/max": 0.4781653881072998,
      "sampling/sampling_logp_difference/mean": 0.05315797030925751,
      "step": 14,
      "step_time": 13.021292253994034
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 573.0,
      "completions/max_terminated_length": 573.0,
      "completions/mean_length": 34.875,
      "completions/mean_terminated_length": 34.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1226625591516495,
      "epoch": 0.0003,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 0.9492051005363464,
      "kl": 0.0027973568066954613,
      "learning_rate": 4.000000000000001e-06,
      "loss": 0.0066,
      "num_tokens": 614427.0,
      "reward": -0.296875,
      "reward_std": 0.08859796822071075,
      "rewards/rollout_reward_func/mean": -0.296875,
      "rewards/rollout_reward_func/std": 0.12309025228023529,
      "sampling/importance_sampling_ratio/max": 1.3263733386993408,
      "sampling/importance_sampling_ratio/mean": 1.0220060348510742,
      "sampling/importance_sampling_ratio/min": 0.6851633787155151,
      "sampling/sampling_logp_difference/max": 0.37807559967041016,
      "sampling/sampling_logp_difference/mean": 0.04163235425949097,
      "step": 15,
      "step_time": 10.956243886997981
    },
    {
      "clip_ratio/high_max": 0.046875,
      "clip_ratio/high_mean": 0.0234375,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0390625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 654.0,
      "completions/max_terminated_length": 654.0,
      "completions/mean_length": 131.78125,
      "completions/mean_terminated_length": 131.78125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9810269959270954,
      "epoch": 0.00032,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.512181043624878,
      "kl": 0.009098081849515438,
      "learning_rate": 4.2857142857142855e-06,
      "loss": -0.0165,
      "num_tokens": 658600.0,
      "reward": -0.22187501192092896,
      "reward_std": 0.21038508415222168,
      "rewards/rollout_reward_func/mean": -0.22187501192092896,
      "rewards/rollout_reward_func/std": 0.22822509706020355,
      "sampling/importance_sampling_ratio/max": 1.5061415433883667,
      "sampling/importance_sampling_ratio/mean": 0.9676941633224487,
      "sampling/importance_sampling_ratio/min": 0.5662972331047058,
      "sampling/sampling_logp_difference/max": 0.43145811557769775,
      "sampling/sampling_logp_difference/mean": 0.04479137063026428,
      "step": 16,
      "step_time": 12.094649828999536
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 465.0,
      "completions/max_terminated_length": 465.0,
      "completions/mean_length": 63.65625,
      "completions/mean_terminated_length": 63.65625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0462176874279976,
      "epoch": 0.00034,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6570802927017212,
      "kl": 0.01116976048797369,
      "learning_rate": 4.571428571428572e-06,
      "loss": -0.0166,
      "num_tokens": 699999.0,
      "reward": -0.19687500596046448,
      "reward_std": 0.21800611913204193,
      "rewards/rollout_reward_func/mean": -0.19687500596046448,
      "rewards/rollout_reward_func/std": 0.25078505277633667,
      "sampling/importance_sampling_ratio/max": 1.377416729927063,
      "sampling/importance_sampling_ratio/mean": 1.0086041688919067,
      "sampling/importance_sampling_ratio/min": 0.8076944947242737,
      "sampling/sampling_logp_difference/max": 0.5140385627746582,
      "sampling/sampling_logp_difference/mean": 0.0497208833694458,
      "step": 17,
      "step_time": 12.058850760025962
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234375,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 609.0,
      "completions/max_terminated_length": 609.0,
      "completions/mean_length": 105.53125,
      "completions/mean_terminated_length": 108.41934967041016,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0595434084534645,
      "epoch": 0.00036,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1009786128997803,
      "kl": 0.00993515457957983,
      "learning_rate": 4.857142857142858e-06,
      "loss": -0.0136,
      "num_tokens": 743486.0,
      "reward": -0.2406250238418579,
      "reward_std": 0.19179844856262207,
      "rewards/rollout_reward_func/mean": -0.2406250238418579,
      "rewards/rollout_reward_func/std": 0.19651705026626587,
      "sampling/importance_sampling_ratio/max": 1.5368764400482178,
      "sampling/importance_sampling_ratio/mean": 1.034261703491211,
      "sampling/importance_sampling_ratio/min": 0.3597041964530945,
      "sampling/sampling_logp_difference/max": 0.7274980545043945,
      "sampling/sampling_logp_difference/mean": 0.07571011781692505,
      "step": 18,
      "step_time": 11.383665119981742
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 647.0,
      "completions/mean_length": 64.0,
      "completions/mean_terminated_length": 64.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0838632583618164,
      "epoch": 0.00038,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.1668524742126465,
      "kl": 0.023761347285471857,
      "learning_rate": 5.142857142857142e-06,
      "loss": -0.0057,
      "num_tokens": 785148.0,
      "reward": -0.24687501788139343,
      "reward_std": 0.17297786474227905,
      "rewards/rollout_reward_func/mean": -0.24687501788139343,
      "rewards/rollout_reward_func/std": 0.2015814334154129,
      "sampling/importance_sampling_ratio/max": 1.4032303094863892,
      "sampling/importance_sampling_ratio/mean": 0.9995061755180359,
      "sampling/importance_sampling_ratio/min": 0.55998694896698,
      "sampling/sampling_logp_difference/max": 0.36147332191467285,
      "sampling/sampling_logp_difference/mean": 0.0652083158493042,
      "step": 19,
      "step_time": 11.808384004980326
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 546.0,
      "completions/max_terminated_length": 546.0,
      "completions/mean_length": 54.25,
      "completions/mean_terminated_length": 54.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0250555127859116,
      "epoch": 0.0004,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0737942457199097,
      "kl": 0.029853208921849728,
      "learning_rate": 5.428571428571429e-06,
      "loss": 0.0082,
      "num_tokens": 826123.0,
      "reward": -0.24687501788139343,
      "reward_std": 0.15553557872772217,
      "rewards/rollout_reward_func/mean": -0.24687501788139343,
      "rewards/rollout_reward_func/std": 0.19835512340068817,
      "sampling/importance_sampling_ratio/max": 1.4954851865768433,
      "sampling/importance_sampling_ratio/mean": 1.0108838081359863,
      "sampling/importance_sampling_ratio/min": 0.5854961276054382,
      "sampling/sampling_logp_difference/max": 0.40249764919281006,
      "sampling/sampling_logp_difference/mean": 0.06120438873767853,
      "step": 20,
      "step_time": 12.069347435011878
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 366.0,
      "completions/max_terminated_length": 366.0,
      "completions/mean_length": 36.25,
      "completions/mean_terminated_length": 36.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9453472681343555,
      "epoch": 0.00042,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6686122417449951,
      "kl": 0.06423007696866989,
      "learning_rate": 5.7142857142857145e-06,
      "loss": -0.0375,
      "num_tokens": 866459.0,
      "reward": -0.2031250149011612,
      "reward_std": 0.1847882866859436,
      "rewards/rollout_reward_func/mean": -0.2031250149011612,
      "rewards/rollout_reward_func/std": 0.21324583888053894,
      "sampling/importance_sampling_ratio/max": 1.4023669958114624,
      "sampling/importance_sampling_ratio/mean": 0.9803745150566101,
      "sampling/importance_sampling_ratio/min": 0.42931392788887024,
      "sampling/sampling_logp_difference/max": 0.8456306457519531,
      "sampling/sampling_logp_difference/mean": 0.08974891155958176,
      "step": 21,
      "step_time": 10.78873161800584
    },
    {
      "clip_ratio/high_max": 0.0625,
      "clip_ratio/high_mean": 0.03125,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0390625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 564.0,
      "completions/max_terminated_length": 564.0,
      "completions/mean_length": 101.15625,
      "completions/mean_terminated_length": 101.15625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9814572706818581,
      "epoch": 0.00044,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.8636322021484375,
      "kl": 0.07755630370229483,
      "learning_rate": 6e-06,
      "loss": 0.0188,
      "num_tokens": 909458.0,
      "reward": -0.3531250059604645,
      "reward_std": 0.11343458294868469,
      "rewards/rollout_reward_func/mean": -0.3531250059604645,
      "rewards/rollout_reward_func/std": 0.11354798823595047,
      "sampling/importance_sampling_ratio/max": 2.0840201377868652,
      "sampling/importance_sampling_ratio/mean": 0.9108251333236694,
      "sampling/importance_sampling_ratio/min": 0.2514898180961609,
      "sampling/sampling_logp_difference/max": 1.041851282119751,
      "sampling/sampling_logp_difference/mean": 0.1208076924085617,
      "step": 22,
      "step_time": 12.667805460987438
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 438.0,
      "completions/max_terminated_length": 438.0,
      "completions/mean_length": 23.0625,
      "completions/mean_terminated_length": 23.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0368415117263794,
      "epoch": 0.00046,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.1755458116531372,
      "kl": 0.08299005404114723,
      "learning_rate": 6.285714285714286e-06,
      "loss": -0.0178,
      "num_tokens": 949922.0,
      "reward": -0.24687500298023224,
      "reward_std": 0.12045249342918396,
      "rewards/rollout_reward_func/mean": -0.24687500298023224,
      "rewards/rollout_reward_func/std": 0.1665288358926773,
      "sampling/importance_sampling_ratio/max": 1.7702778577804565,
      "sampling/importance_sampling_ratio/mean": 1.0359066724777222,
      "sampling/importance_sampling_ratio/min": 0.5668853521347046,
      "sampling/sampling_logp_difference/max": 0.5710101127624512,
      "sampling/sampling_logp_difference/mean": 0.09431438148021698,
      "step": 23,
      "step_time": 10.698794815019937
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.046875,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.046875,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 474.0,
      "completions/max_terminated_length": 474.0,
      "completions/mean_length": 77.96875,
      "completions/mean_terminated_length": 77.96875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8210285473614931,
      "epoch": 0.00048,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0721848011016846,
      "kl": 0.21201763674616814,
      "learning_rate": 6.571428571428572e-06,
      "loss": -0.0357,
      "num_tokens": 991922.0,
      "reward": -0.25312501192092896,
      "reward_std": 0.20965352654457092,
      "rewards/rollout_reward_func/mean": -0.25312501192092896,
      "rewards/rollout_reward_func/std": 0.21847620606422424,
      "sampling/importance_sampling_ratio/max": 2.069061279296875,
      "sampling/importance_sampling_ratio/mean": 0.9011770486831665,
      "sampling/importance_sampling_ratio/min": 0.27951085567474365,
      "sampling/sampling_logp_difference/max": 1.2179622650146484,
      "sampling/sampling_logp_difference/mean": 0.1695031076669693,
      "step": 24,
      "step_time": 11.061244069991517
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 564.0,
      "completions/max_terminated_length": 564.0,
      "completions/mean_length": 76.15625,
      "completions/mean_terminated_length": 76.15625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7968073785305023,
      "epoch": 0.0005,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5174390077590942,
      "kl": 0.3118744776584208,
      "learning_rate": 6.857142857142858e-06,
      "loss": -0.056,
      "num_tokens": 1032594.0,
      "reward": -0.14375001192092896,
      "reward_std": 0.24670757353305817,
      "rewards/rollout_reward_func/mean": -0.14375001192092896,
      "rewards/rollout_reward_func/std": 0.24354572594165802,
      "sampling/importance_sampling_ratio/max": 2.3097891807556152,
      "sampling/importance_sampling_ratio/mean": 0.9626590013504028,
      "sampling/importance_sampling_ratio/min": 0.2978106141090393,
      "sampling/sampling_logp_difference/max": 1.108457088470459,
      "sampling/sampling_logp_difference/mean": 0.19129329919815063,
      "step": 25,
      "step_time": 12.663319375016727
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 555.0,
      "completions/max_terminated_length": 555.0,
      "completions/mean_length": 59.15625,
      "completions/mean_terminated_length": 59.15625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8916913140565157,
      "epoch": 0.00052,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0048381090164185,
      "kl": 0.30913491314277053,
      "learning_rate": 7.1428571428571436e-06,
      "loss": -0.0006,
      "num_tokens": 1074813.0,
      "reward": -0.18125000596046448,
      "reward_std": 0.2179833948612213,
      "rewards/rollout_reward_func/mean": -0.18125000596046448,
      "rewards/rollout_reward_func/std": 0.25072476267814636,
      "sampling/importance_sampling_ratio/max": 2.4185590744018555,
      "sampling/importance_sampling_ratio/mean": 1.0351688861846924,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.4548168182373047,
      "sampling/sampling_logp_difference/mean": 0.19318988919258118,
      "step": 26,
      "step_time": 11.08408795001742
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.03125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 294.0,
      "completions/max_terminated_length": 294.0,
      "completions/mean_length": 11.125,
      "completions/mean_terminated_length": 11.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7630116026848555,
      "epoch": 0.00054,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.723469614982605,
      "kl": 0.5540827643126249,
      "learning_rate": 7.428571428571429e-06,
      "loss": -0.0345,
      "num_tokens": 1113701.0,
      "reward": -0.15000000596046448,
      "reward_std": 0.2582676410675049,
      "rewards/rollout_reward_func/mean": -0.15000000596046448,
      "rewards/rollout_reward_func/std": 0.26880860328674316,
      "sampling/importance_sampling_ratio/max": 2.1174392700195312,
      "sampling/importance_sampling_ratio/mean": 1.035858392715454,
      "sampling/importance_sampling_ratio/min": 0.3152281641960144,
      "sampling/sampling_logp_difference/max": 1.1544897556304932,
      "sampling/sampling_logp_difference/mean": 0.221282497048378,
      "step": 27,
      "step_time": 10.377809943005559
    },
    {
      "clip_ratio/high_max": 0.046875,
      "clip_ratio/high_mean": 0.0234375,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0390625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 591.0,
      "completions/max_terminated_length": 591.0,
      "completions/mean_length": 58.3125,
      "completions/mean_terminated_length": 58.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6682969238609076,
      "epoch": 0.00056,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.36026644706726074,
      "kl": 0.8262073025107384,
      "learning_rate": 7.714285714285716e-06,
      "loss": -0.0259,
      "num_tokens": 1153609.0,
      "reward": -0.11875000596046448,
      "reward_std": 0.2724887728691101,
      "rewards/rollout_reward_func/mean": -0.11875000596046448,
      "rewards/rollout_reward_func/std": 0.27642592787742615,
      "sampling/importance_sampling_ratio/max": 1.5184627771377563,
      "sampling/importance_sampling_ratio/mean": 0.8565590381622314,
      "sampling/importance_sampling_ratio/min": 0.19537629187107086,
      "sampling/sampling_logp_difference/max": 1.6328480243682861,
      "sampling/sampling_logp_difference/mean": 0.2516718804836273,
      "step": 28,
      "step_time": 12.692237003982882
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.03125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 474.0,
      "completions/max_terminated_length": 474.0,
      "completions/mean_length": 63.5,
      "completions/mean_terminated_length": 63.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7480962593108416,
      "epoch": 0.00058,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6819409132003784,
      "kl": 1.7183759827166796,
      "learning_rate": 8.000000000000001e-06,
      "loss": -0.0636,
      "num_tokens": 1195629.0,
      "reward": -0.15312500298023224,
      "reward_std": 0.23106396198272705,
      "rewards/rollout_reward_func/mean": -0.15312500298023224,
      "rewards/rollout_reward_func/std": 0.22716253995895386,
      "sampling/importance_sampling_ratio/max": 2.3989429473876953,
      "sampling/importance_sampling_ratio/mean": 0.9417358636856079,
      "sampling/importance_sampling_ratio/min": 0.09763173758983612,
      "sampling/sampling_logp_difference/max": 2.3263726234436035,
      "sampling/sampling_logp_difference/mean": 0.2625517249107361,
      "step": 29,
      "step_time": 10.827767232025508
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 447.0,
      "completions/max_terminated_length": 447.0,
      "completions/mean_length": 46.375,
      "completions/mean_terminated_length": 46.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6999865411780775,
      "epoch": 0.0006,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8341388702392578,
      "kl": 1.357317698188126,
      "learning_rate": 8.285714285714287e-06,
      "loss": -0.0341,
      "num_tokens": 1236866.0,
      "reward": -0.03750000149011612,
      "reward_std": 0.28391045331954956,
      "rewards/rollout_reward_func/mean": -0.03750000149011612,
      "rewards/rollout_reward_func/std": 0.28708773851394653,
      "sampling/importance_sampling_ratio/max": 2.022016763687134,
      "sampling/importance_sampling_ratio/mean": 1.0200661420822144,
      "sampling/importance_sampling_ratio/min": 0.10619739443063736,
      "sampling/sampling_logp_difference/max": 2.2425215244293213,
      "sampling/sampling_logp_difference/mean": 0.18624533712863922,
      "step": 30,
      "step_time": 10.752208585006883
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 564.0,
      "completions/max_terminated_length": 564.0,
      "completions/mean_length": 128.0,
      "completions/mean_terminated_length": 128.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.628165683709085,
      "epoch": 0.00062,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7558659911155701,
      "kl": 2.631310001015663,
      "learning_rate": 8.571428571428571e-06,
      "loss": -0.019,
      "num_tokens": 1280504.0,
      "reward": -0.06562500447034836,
      "reward_std": 0.2705993056297302,
      "rewards/rollout_reward_func/mean": -0.06562500447034836,
      "rewards/rollout_reward_func/std": 0.2634870111942291,
      "sampling/importance_sampling_ratio/max": 1.707987666130066,
      "sampling/importance_sampling_ratio/mean": 1.0561168193817139,
      "sampling/importance_sampling_ratio/min": 0.06691105663776398,
      "sampling/sampling_logp_difference/max": 2.70440673828125,
      "sampling/sampling_logp_difference/mean": 0.1615004986524582,
      "step": 31,
      "step_time": 13.473540206017788
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 600.0,
      "completions/max_terminated_length": 600.0,
      "completions/mean_length": 89.90625,
      "completions/mean_terminated_length": 89.90625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7277065492235124,
      "epoch": 0.00064,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7042682766914368,
      "kl": 3.7955982983112335,
      "learning_rate": 8.857142857142858e-06,
      "loss": -0.037,
      "num_tokens": 1323851.0,
      "reward": -0.11875000596046448,
      "reward_std": 0.23918089270591736,
      "rewards/rollout_reward_func/mean": -0.11875000596046448,
      "rewards/rollout_reward_func/std": 0.23751060664653778,
      "sampling/importance_sampling_ratio/max": 2.524008274078369,
      "sampling/importance_sampling_ratio/mean": 0.865868330001831,
      "sampling/importance_sampling_ratio/min": 0.03843312710523605,
      "sampling/sampling_logp_difference/max": 3.258748769760132,
      "sampling/sampling_logp_difference/mean": 0.22958534955978394,
      "step": 32,
      "step_time": 11.175307760975556
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 564.0,
      "completions/max_terminated_length": 564.0,
      "completions/mean_length": 94.28125,
      "completions/mean_terminated_length": 94.28125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.46378876199014485,
      "epoch": 0.00066,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5643417835235596,
      "kl": 0.5954106361605227,
      "learning_rate": 9.142857142857144e-06,
      "loss": 0.0109,
      "num_tokens": 1365697.0,
      "reward": 0.00937499850988388,
      "reward_std": 0.2471153736114502,
      "rewards/rollout_reward_func/mean": 0.00937499850988388,
      "rewards/rollout_reward_func/std": 0.25697919726371765,
      "sampling/importance_sampling_ratio/max": 1.7968086004257202,
      "sampling/importance_sampling_ratio/mean": 1.105307698249817,
      "sampling/importance_sampling_ratio/min": 0.40751877427101135,
      "sampling/sampling_logp_difference/max": 0.8977193832397461,
      "sampling/sampling_logp_difference/mean": 0.0992361456155777,
      "step": 33,
      "step_time": 11.007732397032669
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234375,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 111.5625,
      "completions/mean_terminated_length": 111.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5576102585764602,
      "epoch": 0.00068,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.7384986877441406,
      "kl": 11.82732129842043,
      "learning_rate": 9.42857142857143e-06,
      "loss": 0.0105,
      "num_tokens": 1409011.0,
      "reward": -0.03437500447034836,
      "reward_std": 0.27928784489631653,
      "rewards/rollout_reward_func/mean": -0.03437500447034836,
      "rewards/rollout_reward_func/std": 0.28010293841362,
      "sampling/importance_sampling_ratio/max": 1.7578835487365723,
      "sampling/importance_sampling_ratio/mean": 0.925423800945282,
      "sampling/importance_sampling_ratio/min": 0.06665799021720886,
      "sampling/sampling_logp_difference/max": 2.7078189849853516,
      "sampling/sampling_logp_difference/mean": 0.1947239488363266,
      "step": 34,
      "step_time": 13.245897905988386
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.03125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0390625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 546.0,
      "completions/max_terminated_length": 546.0,
      "completions/mean_length": 93.6875,
      "completions/mean_terminated_length": 93.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5237502428935841,
      "epoch": 0.0007,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9105224609375,
      "kl": 2.7520698439329863,
      "learning_rate": 9.714285714285715e-06,
      "loss": -0.0167,
      "num_tokens": 1452356.0,
      "reward": -0.04062499850988388,
      "reward_std": 0.2408279925584793,
      "rewards/rollout_reward_func/mean": -0.04062499850988388,
      "rewards/rollout_reward_func/std": 0.2380829006433487,
      "sampling/importance_sampling_ratio/max": 1.350341558456421,
      "sampling/importance_sampling_ratio/mean": 0.9021202325820923,
      "sampling/importance_sampling_ratio/min": 0.09995006024837494,
      "sampling/sampling_logp_difference/max": 2.303088665008545,
      "sampling/sampling_logp_difference/mean": 0.12590916454792023,
      "step": 35,
      "step_time": 10.927020231989445
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 582.0,
      "completions/max_terminated_length": 582.0,
      "completions/mean_length": 82.03125,
      "completions/mean_terminated_length": 82.03125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.36774852545931935,
      "epoch": 0.00072,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7954043745994568,
      "kl": 2.6637869011610746,
      "learning_rate": 1e-05,
      "loss": -0.0024,
      "num_tokens": 1493967.0,
      "reward": 0.02499999850988388,
      "reward_std": 0.2516058385372162,
      "rewards/rollout_reward_func/mean": 0.02499999850988388,
      "rewards/rollout_reward_func/std": 0.24756881594657898,
      "sampling/importance_sampling_ratio/max": 1.7330042123794556,
      "sampling/importance_sampling_ratio/mean": 0.999114453792572,
      "sampling/importance_sampling_ratio/min": 0.25052380561828613,
      "sampling/sampling_logp_difference/max": 1.3840765953063965,
      "sampling/sampling_logp_difference/mean": 0.0708904042840004,
      "step": 36,
      "step_time": 12.797441937000258
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 654.0,
      "completions/max_terminated_length": 654.0,
      "completions/mean_length": 159.28125,
      "completions/mean_terminated_length": 159.28125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5412645031465217,
      "epoch": 0.00074,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.106950283050537,
      "kl": 0.4659273913130164,
      "learning_rate": 9.999975267482496e-06,
      "loss": -0.0133,
      "num_tokens": 1538868.0,
      "reward": 0.02499999850988388,
      "reward_std": 0.242197185754776,
      "rewards/rollout_reward_func/mean": 0.02499999850988388,
      "rewards/rollout_reward_func/std": 0.2409658133983612,
      "sampling/importance_sampling_ratio/max": 1.3582439422607422,
      "sampling/importance_sampling_ratio/mean": 1.045370101928711,
      "sampling/importance_sampling_ratio/min": 0.615585446357727,
      "sampling/sampling_logp_difference/max": 0.48502540588378906,
      "sampling/sampling_logp_difference/mean": 0.03992624580860138,
      "step": 37,
      "step_time": 11.223586676991545
    },
    {
      "clip_ratio/high_max": 0.010416666977107525,
      "clip_ratio/high_mean": 0.0052083334885537624,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020833333488553762,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 573.0,
      "completions/max_terminated_length": 573.0,
      "completions/mean_length": 171.1875,
      "completions/mean_terminated_length": 171.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6118192651774734,
      "epoch": 0.00076,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9464697241783142,
      "kl": 0.7336778603494167,
      "learning_rate": 9.99990107025622e-06,
      "loss": -0.0108,
      "num_tokens": 1584561.0,
      "reward": 0.04375000298023224,
      "reward_std": 0.25051799416542053,
      "rewards/rollout_reward_func/mean": 0.04375000298023224,
      "rewards/rollout_reward_func/std": 0.274669885635376,
      "sampling/importance_sampling_ratio/max": 1.3502336740493774,
      "sampling/importance_sampling_ratio/mean": 0.8847198486328125,
      "sampling/importance_sampling_ratio/min": 1.3120121722030978e-20,
      "sampling/sampling_logp_difference/max": 31.884798049926758,
      "sampling/sampling_logp_difference/mean": 0.6219751834869385,
      "step": 38,
      "step_time": 11.301022464002017
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.03645833348855376,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03645833348855376,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 627.0,
      "completions/max_terminated_length": 627.0,
      "completions/mean_length": 142.125,
      "completions/mean_terminated_length": 142.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.37175329332239926,
      "epoch": 0.00078,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.693281888961792,
      "kl": 12.660895445384085,
      "learning_rate": 9.99977740929988e-06,
      "loss": 0.0256,
      "num_tokens": 1627965.0,
      "reward": -0.012499995529651642,
      "reward_std": 0.2926747500896454,
      "rewards/rollout_reward_func/mean": -0.012499995529651642,
      "rewards/rollout_reward_func/std": 0.29043906927108765,
      "sampling/importance_sampling_ratio/max": 1.6986039876937866,
      "sampling/importance_sampling_ratio/mean": 0.9517328143119812,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.3545312881469727,
      "sampling/sampling_logp_difference/mean": 0.08566101640462875,
      "step": 39,
      "step_time": 13.440720257000066
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0234375,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234375,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 681.0,
      "completions/max_terminated_length": 681.0,
      "completions/mean_length": 125.71875,
      "completions/mean_terminated_length": 125.71875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4266299228183925,
      "epoch": 0.0008,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4361065626144409,
      "kl": 1.4970917226746678,
      "learning_rate": 9.999604286244655e-06,
      "loss": -0.0285,
      "num_tokens": 1671502.0,
      "reward": -0.006250005215406418,
      "reward_std": 0.2776191830635071,
      "rewards/rollout_reward_func/mean": -0.006250005215406418,
      "rewards/rollout_reward_func/std": 0.274669885635376,
      "sampling/importance_sampling_ratio/max": 1.440024971961975,
      "sampling/importance_sampling_ratio/mean": 0.8878991603851318,
      "sampling/importance_sampling_ratio/min": 0.2080165296792984,
      "sampling/sampling_logp_difference/max": 1.5692877769470215,
      "sampling/sampling_logp_difference/mean": 0.11863303929567337,
      "step": 40,
      "step_time": 11.355874190034228
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 214.375,
      "completions/mean_terminated_length": 214.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.36467939894646406,
      "epoch": 0.00082,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6060211062431335,
      "kl": 0.9705618508160114,
      "learning_rate": 9.999381703374151e-06,
      "loss": 0.0112,
      "num_tokens": 1717611.0,
      "reward": 0.046875,
      "reward_std": 0.25271111726760864,
      "rewards/rollout_reward_func/mean": 0.046875,
      "rewards/rollout_reward_func/std": 0.2699873149394989,
      "sampling/importance_sampling_ratio/max": 1.241422414779663,
      "sampling/importance_sampling_ratio/mean": 0.9222193956375122,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.8252816200256348,
      "sampling/sampling_logp_difference/mean": 0.08485406637191772,
      "step": 41,
      "step_time": 11.466293533027056
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 681.0,
      "completions/max_terminated_length": 681.0,
      "completions/mean_length": 97.71875,
      "completions/mean_terminated_length": 97.71875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.20417141425423324,
      "epoch": 0.00084,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1066712141036987,
      "kl": 3.103068159893155,
      "learning_rate": 9.999109663624387e-06,
      "loss": -0.0213,
      "num_tokens": 1757157.0,
      "reward": 0.14375001192092896,
      "reward_std": 0.22776277363300323,
      "rewards/rollout_reward_func/mean": 0.14375001192092896,
      "rewards/rollout_reward_func/std": 0.222793847322464,
      "sampling/importance_sampling_ratio/max": 1.9615569114685059,
      "sampling/importance_sampling_ratio/mean": 0.9588294625282288,
      "sampling/importance_sampling_ratio/min": 0.12466630339622498,
      "sampling/sampling_logp_difference/max": 2.081918239593506,
      "sampling/sampling_logp_difference/mean": 0.07950436323881149,
      "step": 42,
      "step_time": 13.391698513994925
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 140.1875,
      "completions/mean_terminated_length": 140.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.37688443367369473,
      "epoch": 0.00086,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.877277672290802,
      "kl": 1.905312055721879,
      "learning_rate": 9.99878817058375e-06,
      "loss": -0.0202,
      "num_tokens": 1801643.0,
      "reward": 0.0062499986961483955,
      "reward_std": 0.2858549952507019,
      "rewards/rollout_reward_func/mean": 0.0062499986961483955,
      "rewards/rollout_reward_func/std": 0.2770088016986847,
      "sampling/importance_sampling_ratio/max": 1.1030826568603516,
      "sampling/importance_sampling_ratio/mean": 0.8611886501312256,
      "sampling/importance_sampling_ratio/min": 0.16169670224189758,
      "sampling/sampling_logp_difference/max": 1.8219761848449707,
      "sampling/sampling_logp_difference/mean": 0.12048870325088501,
      "step": 43,
      "step_time": 11.387192363996292
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 636.0,
      "completions/max_terminated_length": 636.0,
      "completions/mean_length": 210.0,
      "completions/mean_terminated_length": 210.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4463948835618794,
      "epoch": 0.00088,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5817294120788574,
      "kl": 0.8826885735616088,
      "learning_rate": 9.998417228492952e-06,
      "loss": -0.0136,
      "num_tokens": 1848910.0,
      "reward": -0.00937499850988388,
      "reward_std": 0.2822544574737549,
      "rewards/rollout_reward_func/mean": -0.00937499850988388,
      "rewards/rollout_reward_func/std": 0.2751649022102356,
      "sampling/importance_sampling_ratio/max": 1.110375165939331,
      "sampling/importance_sampling_ratio/mean": 0.7898576855659485,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.1669483184814453,
      "sampling/sampling_logp_difference/mean": 0.12066969275474548,
      "step": 44,
      "step_time": 11.407111445994815
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 207.5625,
      "completions/mean_terminated_length": 207.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.43740531243383884,
      "epoch": 0.0009,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9083879590034485,
      "kl": 4.853854786604643,
      "learning_rate": 9.99799684224497e-06,
      "loss": 0.0057,
      "num_tokens": 1895041.0,
      "reward": 0.03125,
      "reward_std": 0.2839195728302002,
      "rewards/rollout_reward_func/mean": 0.03125,
      "rewards/rollout_reward_func/std": 0.27875009179115295,
      "sampling/importance_sampling_ratio/max": 2.6709001064300537,
      "sampling/importance_sampling_ratio/mean": 0.9052444100379944,
      "sampling/importance_sampling_ratio/min": 0.11095882952213287,
      "sampling/sampling_logp_difference/max": 2.198289394378662,
      "sampling/sampling_logp_difference/mean": 0.12623152136802673,
      "step": 45,
      "step_time": 13.64592470700154
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234375,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 573.0,
      "completions/max_terminated_length": 573.0,
      "completions/mean_length": 139.34375,
      "completions/mean_terminated_length": 139.34375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2803000560961664,
      "epoch": 0.00092,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2158215343952179,
      "kl": 0.8745662607252598,
      "learning_rate": 9.997527017384983e-06,
      "loss": -0.0113,
      "num_tokens": 1938662.0,
      "reward": 0.04062500223517418,
      "reward_std": 0.22427159547805786,
      "rewards/rollout_reward_func/mean": 0.04062500223517418,
      "rewards/rollout_reward_func/std": 0.22698496282100677,
      "sampling/importance_sampling_ratio/max": 2.3719894886016846,
      "sampling/importance_sampling_ratio/mean": 0.8989532589912415,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.1719560623168945,
      "sampling/sampling_logp_difference/mean": 0.12344694137573242,
      "step": 46,
      "step_time": 11.222478270021384
    },
    {
      "clip_ratio/high_max": 0.010416666977107525,
      "clip_ratio/high_mean": 0.0052083334885537624,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02083333395421505,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 636.0,
      "completions/max_terminated_length": 636.0,
      "completions/mean_length": 146.375,
      "completions/mean_terminated_length": 146.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.27321664337068796,
      "epoch": 0.00094,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6159545183181763,
      "kl": 2.3801498748362064,
      "learning_rate": 9.997007760110298e-06,
      "loss": 0.0276,
      "num_tokens": 1981504.0,
      "reward": 0.05937500298023224,
      "reward_std": 0.28657251596450806,
      "rewards/rollout_reward_func/mean": 0.05937500298023224,
      "rewards/rollout_reward_func/std": 0.27575045824050903,
      "sampling/importance_sampling_ratio/max": 2.7518577575683594,
      "sampling/importance_sampling_ratio/mean": 1.01792573928833,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9951364994049072,
      "sampling/sampling_logp_difference/mean": 0.12307627499103546,
      "step": 47,
      "step_time": 11.423600950016407
    },
    {
      "clip_ratio/high_max": 0.026041666977107525,
      "clip_ratio/high_mean": 0.013020833488553762,
      "clip_ratio/low_mean": 0.020833333488553762,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.033854166977107525,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 654.0,
      "completions/max_terminated_length": 654.0,
      "completions/mean_length": 172.90625,
      "completions/mean_terminated_length": 172.90625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3133870712481439,
      "epoch": 0.00096,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5473414659500122,
      "kl": 1.4304386470466852,
      "learning_rate": 9.996439077270269e-06,
      "loss": -0.0019,
      "num_tokens": 2026272.0,
      "reward": 0.006250001490116119,
      "reward_std": 0.2522270083427429,
      "rewards/rollout_reward_func/mean": 0.006250001490116119,
      "rewards/rollout_reward_func/std": 0.25392085313796997,
      "sampling/importance_sampling_ratio/max": 1.9223732948303223,
      "sampling/importance_sampling_ratio/mean": 0.9544615149497986,
      "sampling/importance_sampling_ratio/min": 0.29666778445243835,
      "sampling/sampling_logp_difference/max": 1.2302255630493164,
      "sampling/sampling_logp_difference/mean": 0.09093736857175827,
      "step": 48,
      "step_time": 13.378115150990197
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 600.0,
      "completions/max_terminated_length": 600.0,
      "completions/mean_length": 164.46875,
      "completions/mean_terminated_length": 164.46875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2654049596749246,
      "epoch": 0.00098,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5221968293190002,
      "kl": 0.9643691387027502,
      "learning_rate": 9.995820976366208e-06,
      "loss": 0.0038,
      "num_tokens": 2072106.0,
      "reward": 0.02812499925494194,
      "reward_std": 0.25317180156707764,
      "rewards/rollout_reward_func/mean": 0.02812499925494194,
      "rewards/rollout_reward_func/std": 0.26668137311935425,
      "sampling/importance_sampling_ratio/max": 1.231547474861145,
      "sampling/importance_sampling_ratio/mean": 0.9199539422988892,
      "sampling/importance_sampling_ratio/min": 0.25612059235572815,
      "sampling/sampling_logp_difference/max": 1.4200241565704346,
      "sampling/sampling_logp_difference/mean": 0.08330152928829193,
      "step": 49,
      "step_time": 11.4312691600353
    },
    {
      "clip_ratio/high_max": 0.05208333395421505,
      "clip_ratio/high_mean": 0.026041666977107525,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.026041666977107525,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 609.0,
      "completions/max_terminated_length": 609.0,
      "completions/mean_length": 197.40625,
      "completions/mean_terminated_length": 197.40625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3112881346605718,
      "epoch": 0.001,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8779159188270569,
      "kl": 2.751499431207776,
      "learning_rate": 9.995153465551283e-06,
      "loss": -0.0479,
      "num_tokens": 2118155.0,
      "reward": 0.07187500596046448,
      "reward_std": 0.30170750617980957,
      "rewards/rollout_reward_func/mean": 0.07187500596046448,
      "rewards/rollout_reward_func/std": 0.29428356885910034,
      "sampling/importance_sampling_ratio/max": 2.896355390548706,
      "sampling/importance_sampling_ratio/mean": 0.9583613872528076,
      "sampling/importance_sampling_ratio/min": 0.1915157586336136,
      "sampling/sampling_logp_difference/max": 1.652543544769287,
      "sampling/sampling_logp_difference/mean": 0.12397453188896179,
      "step": 50,
      "step_time": 11.400272801009123
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0052083334885537624,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020833333488553762,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 214.875,
      "completions/mean_terminated_length": 214.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.21842088596895337,
      "epoch": 0.00102,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5779225826263428,
      "kl": 0.7069900678470731,
      "learning_rate": 9.994436553630411e-06,
      "loss": 0.0038,
      "num_tokens": 2164032.0,
      "reward": 0.07500000298023224,
      "reward_std": 0.25826045870780945,
      "rewards/rollout_reward_func/mean": 0.07500000298023224,
      "rewards/rollout_reward_func/std": 0.2488684207201004,
      "sampling/importance_sampling_ratio/max": 1.5296891927719116,
      "sampling/importance_sampling_ratio/mean": 0.9359626770019531,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.4481534957885742,
      "sampling/sampling_logp_difference/mean": 0.08004340529441833,
      "step": 51,
      "step_time": 13.095075463003013
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 190.34375,
      "completions/mean_terminated_length": 190.34375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.20333480136469007,
      "epoch": 0.00104,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.42373329401016235,
      "kl": 2.319945788010955,
      "learning_rate": 9.993670250060152e-06,
      "loss": 0.0076,
      "num_tokens": 2209214.0,
      "reward": 0.03437499701976776,
      "reward_std": 0.25926271080970764,
      "rewards/rollout_reward_func/mean": 0.03437499701976776,
      "rewards/rollout_reward_func/std": 0.2777900993824005,
      "sampling/importance_sampling_ratio/max": 1.3496242761611938,
      "sampling/importance_sampling_ratio/mean": 1.030142903327942,
      "sampling/importance_sampling_ratio/min": 0.18121764063835144,
      "sampling/sampling_logp_difference/max": 1.707810401916504,
      "sampling/sampling_logp_difference/mean": 0.08821949362754822,
      "step": 52,
      "step_time": 11.57385084600537
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.020833333488553762,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.028645833488553762,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 620.0,
      "completions/max_terminated_length": 620.0,
      "completions/mean_length": 219.0625,
      "completions/mean_terminated_length": 219.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.20170985534787178,
      "epoch": 0.00106,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.49643993377685547,
      "kl": 0.772532288916409,
      "learning_rate": 9.99285456494856e-06,
      "loss": 0.0392,
      "num_tokens": 2256608.0,
      "reward": 0.02812500298023224,
      "reward_std": 0.2515004575252533,
      "rewards/rollout_reward_func/mean": 0.02812500298023224,
      "rewards/rollout_reward_func/std": 0.2504633069038391,
      "sampling/importance_sampling_ratio/max": 2.932656764984131,
      "sampling/importance_sampling_ratio/mean": 1.0782846212387085,
      "sampling/importance_sampling_ratio/min": 0.29615864157676697,
      "sampling/sampling_logp_difference/max": 1.2217793464660645,
      "sampling/sampling_logp_difference/mean": 0.08285166323184967,
      "step": 53,
      "step_time": 12.397921280018636
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 591.0,
      "completions/max_terminated_length": 591.0,
      "completions/mean_length": 118.4375,
      "completions/mean_terminated_length": 118.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.15126608172431588,
      "epoch": 0.00108,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5388218760490417,
      "kl": 1.2472308743745089,
      "learning_rate": 9.991989509055075e-06,
      "loss": 0.009,
      "num_tokens": 2297497.0,
      "reward": 0.11249999701976776,
      "reward_std": 0.2365659922361374,
      "rewards/rollout_reward_func/mean": 0.11249999701976776,
      "rewards/rollout_reward_func/std": 0.2379346638917923,
      "sampling/importance_sampling_ratio/max": 1.2914180755615234,
      "sampling/importance_sampling_ratio/mean": 1.0325658321380615,
      "sampling/importance_sampling_ratio/min": 0.17973823845386505,
      "sampling/sampling_logp_difference/max": 1.7153735160827637,
      "sampling/sampling_logp_difference/mean": 0.06109348684549332,
      "step": 54,
      "step_time": 12.035794266033918
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0234375,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234375,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 186.25,
      "completions/mean_terminated_length": 186.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.19740394363179803,
      "epoch": 0.0011,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2732814848423004,
      "kl": 3.4689031448215246,
      "learning_rate": 9.991075093790372e-06,
      "loss": -0.0251,
      "num_tokens": 2343508.0,
      "reward": 0.01874999701976776,
      "reward_std": 0.22209849953651428,
      "rewards/rollout_reward_func/mean": 0.01874999701976776,
      "rewards/rollout_reward_func/std": 0.24552446603775024,
      "sampling/importance_sampling_ratio/max": 1.3932645320892334,
      "sampling/importance_sampling_ratio/mean": 0.9726418852806091,
      "sampling/importance_sampling_ratio/min": 0.17458941042423248,
      "sampling/sampling_logp_difference/max": 1.745373010635376,
      "sampling/sampling_logp_difference/mean": 0.11014913022518158,
      "step": 55,
      "step_time": 11.509712788028992
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 214.5625,
      "completions/mean_terminated_length": 214.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.24161778902634978,
      "epoch": 0.00112,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.23505429923534393,
      "kl": 1.2839185483753681,
      "learning_rate": 9.990111331216204e-06,
      "loss": 0.0152,
      "num_tokens": 2390046.0,
      "reward": -0.04062499850988388,
      "reward_std": 0.24682512879371643,
      "rewards/rollout_reward_func/mean": -0.04062499850988388,
      "rewards/rollout_reward_func/std": 0.25508931279182434,
      "sampling/importance_sampling_ratio/max": 1.4491592645645142,
      "sampling/importance_sampling_ratio/mean": 0.9946873784065247,
      "sampling/importance_sampling_ratio/min": 0.24229057133197784,
      "sampling/sampling_logp_difference/max": 1.423060655593872,
      "sampling/sampling_logp_difference/mean": 0.08438558131456375,
      "step": 56,
      "step_time": 13.343498381000245
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0234375,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 263.09375,
      "completions/mean_terminated_length": 263.09375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.24081898154690862,
      "epoch": 0.00114,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 33.2855339050293,
      "kl": 58.01132536120713,
      "learning_rate": 9.989098234045254e-06,
      "loss": 0.1413,
      "num_tokens": 2438602.0,
      "reward": -0.02500000223517418,
      "reward_std": 0.28471288084983826,
      "rewards/rollout_reward_func/mean": -0.02500000223517418,
      "rewards/rollout_reward_func/std": 0.2735666334629059,
      "sampling/importance_sampling_ratio/max": 1.2197866439819336,
      "sampling/importance_sampling_ratio/mean": 0.9039830565452576,
      "sampling/importance_sampling_ratio/min": 0.02309959940612316,
      "sampling/sampling_logp_difference/max": 3.611992835998535,
      "sampling/sampling_logp_difference/mean": 0.12853899598121643,
      "step": 57,
      "step_time": 11.64153567799076
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234375,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 175.59375,
      "completions/mean_terminated_length": 175.59375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2313585728406906,
      "epoch": 0.00116,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5263372659683228,
      "kl": 2.427494652569294,
      "learning_rate": 9.988035815640953e-06,
      "loss": -0.0145,
      "num_tokens": 2484064.0,
      "reward": 0.04375000298023224,
      "reward_std": 0.3044625222682953,
      "rewards/rollout_reward_func/mean": 0.04375000298023224,
      "rewards/rollout_reward_func/std": 0.29505330324172974,
      "sampling/importance_sampling_ratio/max": 1.1703529357910156,
      "sampling/importance_sampling_ratio/mean": 0.9389526844024658,
      "sampling/importance_sampling_ratio/min": 0.21210388839244843,
      "sampling/sampling_logp_difference/max": 1.5496492385864258,
      "sampling/sampling_logp_difference/mean": 0.09731487929821014,
      "step": 58,
      "step_time": 11.456254371994874
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 636.0,
      "completions/max_terminated_length": 636.0,
      "completions/mean_length": 251.125,
      "completions/mean_terminated_length": 251.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.22872668597847223,
      "epoch": 0.00118,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6488710045814514,
      "kl": 1.7432300951331854,
      "learning_rate": 9.986924090017322e-06,
      "loss": -0.0247,
      "num_tokens": 2533624.0,
      "reward": 0.03749999403953552,
      "reward_std": 0.22515708208084106,
      "rewards/rollout_reward_func/mean": 0.03749999403953552,
      "rewards/rollout_reward_func/std": 0.23383204638957977,
      "sampling/importance_sampling_ratio/max": 1.2473702430725098,
      "sampling/importance_sampling_ratio/mean": 0.9192129373550415,
      "sampling/importance_sampling_ratio/min": 0.11615930497646332,
      "sampling/sampling_logp_difference/max": 2.1710920333862305,
      "sampling/sampling_logp_difference/mean": 0.09392736852169037,
      "step": 59,
      "step_time": 13.274015463030082
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 681.0,
      "completions/max_terminated_length": 681.0,
      "completions/mean_length": 190.5,
      "completions/mean_terminated_length": 190.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.19564690859988332,
      "epoch": 0.0012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7282356023788452,
      "kl": 0.7743214350193739,
      "learning_rate": 9.985763071838774e-06,
      "loss": 0.0007,
      "num_tokens": 2578743.0,
      "reward": 0.09687500447034836,
      "reward_std": 0.2579509913921356,
      "rewards/rollout_reward_func/mean": 0.09687500447034836,
      "rewards/rollout_reward_func/std": 0.2546146512031555,
      "sampling/importance_sampling_ratio/max": 1.2842891216278076,
      "sampling/importance_sampling_ratio/mean": 1.0898473262786865,
      "sampling/importance_sampling_ratio/min": 0.8279550671577454,
      "sampling/sampling_logp_difference/max": 0.23374176025390625,
      "sampling/sampling_logp_difference/mean": 0.03387434035539627,
      "step": 60,
      "step_time": 11.90262256297865
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 227.96875,
      "completions/mean_terminated_length": 227.96875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.22771155647933483,
      "epoch": 0.00122,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.41098225116729736,
      "kl": 0.8445331808179617,
      "learning_rate": 9.98455277641992e-06,
      "loss": -0.0054,
      "num_tokens": 2625702.0,
      "reward": 0.07500000298023224,
      "reward_std": 0.23432482779026031,
      "rewards/rollout_reward_func/mean": 0.07500000298023224,
      "rewards/rollout_reward_func/std": 0.24756883084774017,
      "sampling/importance_sampling_ratio/max": 1.2830342054367065,
      "sampling/importance_sampling_ratio/mean": 1.0378845930099487,
      "sampling/importance_sampling_ratio/min": 0.35964587330818176,
      "sampling/sampling_logp_difference/max": 1.0222272872924805,
      "sampling/sampling_logp_difference/mean": 0.04020819440484047,
      "step": 61,
      "step_time": 11.48414945298282
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 654.0,
      "completions/max_terminated_length": 654.0,
      "completions/mean_length": 170.28125,
      "completions/mean_terminated_length": 170.28125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.23342515621334314,
      "epoch": 0.00124,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4464556574821472,
      "kl": 1.809425000101328,
      "learning_rate": 9.983293219725379e-06,
      "loss": 0.0113,
      "num_tokens": 2670425.0,
      "reward": 0.046875,
      "reward_std": 0.3064039945602417,
      "rewards/rollout_reward_func/mean": 0.046875,
      "rewards/rollout_reward_func/std": 0.29836583137512207,
      "sampling/importance_sampling_ratio/max": 1.208778738975525,
      "sampling/importance_sampling_ratio/mean": 1.025787115097046,
      "sampling/importance_sampling_ratio/min": 0.2051321268081665,
      "sampling/sampling_logp_difference/max": 1.5840506553649902,
      "sampling/sampling_logp_difference/mean": 0.06489424407482147,
      "step": 62,
      "step_time": 13.149487137037795
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 609.0,
      "completions/max_terminated_length": 609.0,
      "completions/mean_length": 226.75,
      "completions/mean_terminated_length": 226.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2739702956750989,
      "epoch": 0.00126,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.9997408390045166,
      "kl": 11.797076037153602,
      "learning_rate": 9.981984418369552e-06,
      "loss": 0.0122,
      "num_tokens": 2718052.0,
      "reward": -0.046875,
      "reward_std": 0.25116267800331116,
      "rewards/rollout_reward_func/mean": -0.046875,
      "rewards/rollout_reward_func/std": 0.2565080523490906,
      "sampling/importance_sampling_ratio/max": 1.3198902606964111,
      "sampling/importance_sampling_ratio/mean": 0.9307323694229126,
      "sampling/importance_sampling_ratio/min": 0.3230690360069275,
      "sampling/sampling_logp_difference/max": 1.1296305656433105,
      "sampling/sampling_logp_difference/mean": 0.0826588124036789,
      "step": 63,
      "step_time": 11.495983364031417
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 211.4375,
      "completions/mean_terminated_length": 211.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.22241126652806997,
      "epoch": 0.00128,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.29363980889320374,
      "kl": 1.5062697418034077,
      "learning_rate": 9.980626389616414e-06,
      "loss": -0.0011,
      "num_tokens": 2764040.0,
      "reward": 0.07500000298023224,
      "reward_std": 0.19970935583114624,
      "rewards/rollout_reward_func/mean": 0.07500000298023224,
      "rewards/rollout_reward_func/std": 0.25903046131134033,
      "sampling/importance_sampling_ratio/max": 1.2733521461486816,
      "sampling/importance_sampling_ratio/mean": 0.9930177330970764,
      "sampling/importance_sampling_ratio/min": 0.22384974360466003,
      "sampling/sampling_logp_difference/max": 1.508871078491211,
      "sampling/sampling_logp_difference/mean": 0.07086890190839767,
      "step": 64,
      "step_time": 11.527082602973678
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0234375,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234375,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 663.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 266.78125,
      "completions/mean_terminated_length": 266.78125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.25698775611817837,
      "epoch": 0.0013,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8869104385375977,
      "kl": 4.196637926623225,
      "learning_rate": 9.979219151379285e-06,
      "loss": 0.0089,
      "num_tokens": 2813465.0,
      "reward": 0.05000000447034836,
      "reward_std": 0.2999016046524048,
      "rewards/rollout_reward_func/mean": 0.05000000447034836,
      "rewards/rollout_reward_func/std": 0.3026762306690216,
      "sampling/importance_sampling_ratio/max": 1.3658215999603271,
      "sampling/importance_sampling_ratio/mean": 0.9861950278282166,
      "sampling/importance_sampling_ratio/min": 0.2892862558364868,
      "sampling/sampling_logp_difference/max": 1.2403202056884766,
      "sampling/sampling_logp_difference/mean": 0.07883322238922119,
      "step": 65,
      "step_time": 13.46161199499329
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 254.84375,
      "completions/mean_terminated_length": 254.84375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2509062495082617,
      "epoch": 0.00132,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9279988408088684,
      "kl": 3.308092936873436,
      "learning_rate": 9.977762722220585e-06,
      "loss": -0.0007,
      "num_tokens": 2861675.0,
      "reward": 0.02187499776482582,
      "reward_std": 0.301052063703537,
      "rewards/rollout_reward_func/mean": 0.02187499776482582,
      "rewards/rollout_reward_func/std": 0.2991756200790405,
      "sampling/importance_sampling_ratio/max": 1.2624356746673584,
      "sampling/importance_sampling_ratio/mean": 1.0402545928955078,
      "sampling/importance_sampling_ratio/min": 0.17986911535263062,
      "sampling/sampling_logp_difference/max": 1.8468008041381836,
      "sampling/sampling_logp_difference/mean": 0.06000932306051254,
      "step": 66,
      "step_time": 11.597071924057673
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.03125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 609.0,
      "completions/max_terminated_length": 609.0,
      "completions/mean_length": 157.1875,
      "completions/mean_terminated_length": 157.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.25066352915018797,
      "epoch": 0.00134,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6115333437919617,
      "kl": 1.0872764214873314,
      "learning_rate": 9.976257121351602e-06,
      "loss": -0.0061,
      "num_tokens": 2905066.0,
      "reward": 0.13750000298023224,
      "reward_std": 0.24354363977909088,
      "rewards/rollout_reward_func/mean": 0.13750000298023224,
      "rewards/rollout_reward_func/std": 0.2612007260322571,
      "sampling/importance_sampling_ratio/max": 1.2874200344085693,
      "sampling/importance_sampling_ratio/mean": 1.0013997554779053,
      "sampling/importance_sampling_ratio/min": 0.41545429825782776,
      "sampling/sampling_logp_difference/max": 0.878279447555542,
      "sampling/sampling_logp_difference/mean": 0.06989742815494537,
      "step": 67,
      "step_time": 12.412616250032443
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 681.0,
      "completions/max_terminated_length": 681.0,
      "completions/mean_length": 156.9375,
      "completions/mean_terminated_length": 156.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3040436040610075,
      "epoch": 0.00136,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8024974465370178,
      "kl": 2.9588886965066195,
      "learning_rate": 9.974702368632228e-06,
      "loss": -0.0409,
      "num_tokens": 2948732.0,
      "reward": 0.16562500596046448,
      "reward_std": 0.27447807788848877,
      "rewards/rollout_reward_func/mean": 0.16562500596046448,
      "rewards/rollout_reward_func/std": 0.28916940093040466,
      "sampling/importance_sampling_ratio/max": 1.2132750749588013,
      "sampling/importance_sampling_ratio/mean": 0.9726374745368958,
      "sampling/importance_sampling_ratio/min": 0.1697029173374176,
      "sampling/sampling_logp_difference/max": 1.5887651443481445,
      "sampling/sampling_logp_difference/mean": 0.08710584044456482,
      "step": 68,
      "step_time": 12.203586879972136
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 248.25,
      "completions/mean_terminated_length": 248.25,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.28679338470101357,
      "epoch": 0.00138,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.39162760972976685,
      "kl": 2.0344080086797476,
      "learning_rate": 9.973098484570702e-06,
      "loss": -0.0047,
      "num_tokens": 2996668.0,
      "reward": 0.02812499925494194,
      "reward_std": 0.324293315410614,
      "rewards/rollout_reward_func/mean": 0.02812499925494194,
      "rewards/rollout_reward_func/std": 0.3195100724697113,
      "sampling/importance_sampling_ratio/max": 1.2930116653442383,
      "sampling/importance_sampling_ratio/mean": 0.9549592733383179,
      "sampling/importance_sampling_ratio/min": 0.3066304624080658,
      "sampling/sampling_logp_difference/max": 1.1820893287658691,
      "sampling/sampling_logp_difference/mean": 0.08449231833219528,
      "step": 69,
      "step_time": 11.645247950960766
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.0078125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0078125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 663.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 248.9375,
      "completions/mean_terminated_length": 248.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2908786991611123,
      "epoch": 0.0014,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7563177943229675,
      "kl": 1.0613105613738298,
      "learning_rate": 9.97144549032334e-06,
      "loss": 0.0046,
      "num_tokens": 3044887.0,
      "reward": 0.03437500074505806,
      "reward_std": 0.2673013210296631,
      "rewards/rollout_reward_func/mean": 0.03437500074505806,
      "rewards/rollout_reward_func/std": 0.2622598707675934,
      "sampling/importance_sampling_ratio/max": 1.275259256362915,
      "sampling/importance_sampling_ratio/mean": 0.9882270693778992,
      "sampling/importance_sampling_ratio/min": 0.3845323622226715,
      "sampling/sampling_logp_difference/max": 0.9556173086166382,
      "sampling/sampling_logp_difference/mean": 0.07594569027423859,
      "step": 70,
      "step_time": 12.42379025502305
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 188.9375,
      "completions/mean_terminated_length": 188.9375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.33571683894842863,
      "epoch": 0.00142,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5575281381607056,
      "kl": 2.2665872871875763,
      "learning_rate": 9.969743407694254e-06,
      "loss": 0.0078,
      "num_tokens": 3089799.0,
      "reward": 0.08437500149011612,
      "reward_std": 0.2662077248096466,
      "rewards/rollout_reward_func/mean": 0.08437500149011612,
      "rewards/rollout_reward_func/std": 0.25791895389556885,
      "sampling/importance_sampling_ratio/max": 1.5227569341659546,
      "sampling/importance_sampling_ratio/mean": 0.9422699213027954,
      "sampling/importance_sampling_ratio/min": 0.21678699553012848,
      "sampling/sampling_logp_difference/max": 1.137342929840088,
      "sampling/sampling_logp_difference/mean": 0.11660201847553253,
      "step": 71,
      "step_time": 12.160292600019602
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 681.0,
      "completions/max_terminated_length": 681.0,
      "completions/mean_length": 256.09375,
      "completions/mean_terminated_length": 256.09375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3041485659778118,
      "epoch": 0.00144,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2881532907485962,
      "kl": 0.6617582477629185,
      "learning_rate": 9.967992259135063e-06,
      "loss": 0.0075,
      "num_tokens": 3138288.0,
      "reward": 0.10625000298023224,
      "reward_std": 0.2767048478126526,
      "rewards/rollout_reward_func/mean": 0.10625000298023224,
      "rewards/rollout_reward_func/std": 0.27231091260910034,
      "sampling/importance_sampling_ratio/max": 1.31902277469635,
      "sampling/importance_sampling_ratio/mean": 1.0397642850875854,
      "sampling/importance_sampling_ratio/min": 0.42800259590148926,
      "sampling/sampling_logp_difference/max": 0.8528444766998291,
      "sampling/sampling_logp_difference/mean": 0.05983080714941025,
      "step": 72,
      "step_time": 11.671126687026117
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0234375,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234375,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 663.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 240.375,
      "completions/mean_terminated_length": 240.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2916300296783447,
      "epoch": 0.00146,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5817001461982727,
      "kl": 3.6002681870013475,
      "learning_rate": 9.9661920677446e-06,
      "loss": -0.0054,
      "num_tokens": 3185247.0,
      "reward": 0.03125,
      "reward_std": 0.32543766498565674,
      "rewards/rollout_reward_func/mean": 0.03125,
      "rewards/rollout_reward_func/std": 0.32867035269737244,
      "sampling/importance_sampling_ratio/max": 1.2236007452011108,
      "sampling/importance_sampling_ratio/mean": 0.9291048049926758,
      "sampling/importance_sampling_ratio/min": 0.20215687155723572,
      "sampling/sampling_logp_difference/max": 1.4806010723114014,
      "sampling/sampling_logp_difference/mean": 0.1030847430229187,
      "step": 73,
      "step_time": 12.253923129028408
    },
    {
      "clip_ratio/high_max": 0.026041666977107525,
      "clip_ratio/high_mean": 0.013020833488553762,
      "clip_ratio/low_mean": 0.0078125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020833333488553762,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 663.0,
      "completions/max_terminated_length": 663.0,
      "completions/mean_length": 210.65625,
      "completions/mean_terminated_length": 210.65625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3424571454524994,
      "epoch": 0.00148,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6793689131736755,
      "kl": 3.160204190760851,
      "learning_rate": 9.964342857268609e-06,
      "loss": -0.0196,
      "num_tokens": 3232335.0,
      "reward": -0.02500000223517418,
      "reward_std": 0.28614965081214905,
      "rewards/rollout_reward_func/mean": -0.02500000223517418,
      "rewards/rollout_reward_func/std": 0.27708157896995544,
      "sampling/importance_sampling_ratio/max": 1.2865917682647705,
      "sampling/importance_sampling_ratio/mean": 0.9295843839645386,
      "sampling/importance_sampling_ratio/min": 0.14403089880943298,
      "sampling/sampling_logp_difference/max": 1.495539665222168,
      "sampling/sampling_logp_difference/mean": 0.10957419127225876,
      "step": 74,
      "step_time": 12.433173162018647
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 672.0,
      "completions/max_terminated_length": 672.0,
      "completions/mean_length": 233.71875,
      "completions/mean_terminated_length": 233.71875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.2938485397025943,
      "epoch": 0.0015,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.108181118965149,
      "kl": 0.9683016650378704,
      "learning_rate": 9.962444652099422e-06,
      "loss": 0.0056,
      "num_tokens": 3280415.0,
      "reward": 0.06874999403953552,
      "reward_std": 0.2604575753211975,
      "rewards/rollout_reward_func/mean": 0.06874999403953552,
      "rewards/rollout_reward_func/std": 0.287859171628952,
      "sampling/importance_sampling_ratio/max": 1.2297614812850952,
      "sampling/importance_sampling_ratio/mean": 1.0398441553115845,
      "sampling/importance_sampling_ratio/min": 0.33198797702789307,
      "sampling/sampling_logp_difference/max": 1.102529525756836,
      "sampling/sampling_logp_difference/mean": 0.04783129692077637,
      "step": 75,
      "step_time": 11.919177781994222
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 900,
  "num_input_tokens_seen": 3280415,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}