{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.00029,
  "eval_steps": 500,
  "global_step": 29,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1121.0,
      "completions/max_terminated_length": 1121.0,
      "completions/mean_length": 89.125,
      "completions/mean_terminated_length": 89.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8703252337872982,
      "epoch": 1e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.623584270477295,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": -0.0507,
      "num_tokens": 20371.0,
      "reward": -0.2766486704349518,
      "reward_std": 0.1640574336051941,
      "rewards/rollout_reward_func/mean": -0.2766486704349518,
      "rewards/rollout_reward_func/std": 0.20505212247371674,
      "sampling/importance_sampling_ratio/max": 1.2225033044815063,
      "sampling/importance_sampling_ratio/mean": 1.0341836214065552,
      "sampling/importance_sampling_ratio/min": 0.9178955554962158,
      "sampling/sampling_logp_difference/max": 0.13056421279907227,
      "sampling/sampling_logp_difference/mean": 0.028775660321116447,
      "step": 1,
      "step_time": 10.067080650000321
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 650.0,
      "completions/max_terminated_length": 650.0,
      "completions/mean_length": 125.5625,
      "completions/mean_terminated_length": 125.5625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9611260890960693,
      "epoch": 2e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.4665699005126953,
      "kl": 0.0,
      "learning_rate": 2.8571428571428575e-07,
      "loss": -0.0335,
      "num_tokens": 42911.0,
      "reward": -0.2309272140264511,
      "reward_std": 0.26045405864715576,
      "rewards/rollout_reward_func/mean": -0.2309272140264511,
      "rewards/rollout_reward_func/std": 0.2524268329143524,
      "sampling/importance_sampling_ratio/max": 1.2408462762832642,
      "sampling/importance_sampling_ratio/mean": 1.014411449432373,
      "sampling/importance_sampling_ratio/min": 0.862123966217041,
      "sampling/sampling_logp_difference/max": 0.21459484100341797,
      "sampling/sampling_logp_difference/mean": 0.02787204459309578,
      "step": 2,
      "step_time": 8.417592127000034
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1042.0,
      "completions/max_terminated_length": 1042.0,
      "completions/mean_length": 101.375,
      "completions/mean_terminated_length": 101.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8967500347644091,
      "epoch": 3e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2944509983062744,
      "kl": 0.001957286149263382,
      "learning_rate": 5.714285714285715e-07,
      "loss": -0.0382,
      "num_tokens": 63636.0,
      "reward": -0.2389892041683197,
      "reward_std": 0.2091013640165329,
      "rewards/rollout_reward_func/mean": -0.2389892041683197,
      "rewards/rollout_reward_func/std": 0.21935902535915375,
      "sampling/importance_sampling_ratio/max": 1.1709651947021484,
      "sampling/importance_sampling_ratio/mean": 1.010354995727539,
      "sampling/importance_sampling_ratio/min": 0.8460708260536194,
      "sampling/sampling_logp_difference/max": 0.16718029975891113,
      "sampling/sampling_logp_difference/mean": 0.03452146053314209,
      "step": 3,
      "step_time": 8.871616774999666
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 569.0,
      "completions/max_terminated_length": 569.0,
      "completions/mean_length": 107.0,
      "completions/mean_terminated_length": 107.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8927937652915716,
      "epoch": 4e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.373798131942749,
      "kl": 0.002377880737185478,
      "learning_rate": 8.571428571428572e-07,
      "loss": -0.0162,
      "num_tokens": 85333.0,
      "reward": -0.312600314617157,
      "reward_std": 0.15555822849273682,
      "rewards/rollout_reward_func/mean": -0.312600314617157,
      "rewards/rollout_reward_func/std": 0.1706695705652237,
      "sampling/importance_sampling_ratio/max": 1.2034525871276855,
      "sampling/importance_sampling_ratio/mean": 0.9583898186683655,
      "sampling/importance_sampling_ratio/min": 0.732940673828125,
      "sampling/sampling_logp_difference/max": 0.31058597564697266,
      "sampling/sampling_logp_difference/mean": 0.030293956398963928,
      "step": 4,
      "step_time": 7.5195989830002645
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 659.0,
      "completions/max_terminated_length": 659.0,
      "completions/mean_length": 97.125,
      "completions/mean_terminated_length": 97.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0254089757800102,
      "epoch": 5e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9663734436035156,
      "kl": 0.0008389949798583984,
      "learning_rate": 1.142857142857143e-06,
      "loss": 0.0031,
      "num_tokens": 106863.0,
      "reward": -0.2523829936981201,
      "reward_std": 0.25961124897003174,
      "rewards/rollout_reward_func/mean": -0.2523829936981201,
      "rewards/rollout_reward_func/std": 0.2587268650531769,
      "sampling/importance_sampling_ratio/max": 1.1407527923583984,
      "sampling/importance_sampling_ratio/mean": 0.9936575293540955,
      "sampling/importance_sampling_ratio/min": 0.7607207298278809,
      "sampling/sampling_logp_difference/max": 0.20561480522155762,
      "sampling/sampling_logp_difference/mean": 0.03536064550280571,
      "step": 5,
      "step_time": 8.257373196000003
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1123.0,
      "completions/max_terminated_length": 1123.0,
      "completions/mean_length": 109.1875,
      "completions/mean_terminated_length": 109.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.952575646340847,
      "epoch": 6e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.785276174545288,
      "kl": 0.0018332991749048233,
      "learning_rate": 1.4285714285714286e-06,
      "loss": -0.0255,
      "num_tokens": 128186.0,
      "reward": -0.3365856409072876,
      "reward_std": 0.08366663753986359,
      "rewards/rollout_reward_func/mean": -0.3365856409072876,
      "rewards/rollout_reward_func/std": 0.09609124809503555,
      "sampling/importance_sampling_ratio/max": 1.116741418838501,
      "sampling/importance_sampling_ratio/mean": 0.9997830390930176,
      "sampling/importance_sampling_ratio/min": 0.8368671536445618,
      "sampling/sampling_logp_difference/max": 0.2266085147857666,
      "sampling/sampling_logp_difference/mean": 0.032183341681957245,
      "step": 6,
      "step_time": 9.195131481000203
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0388711597770452,
      "epoch": 7e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5290614366531372,
      "kl": 0.0020035766065120697,
      "learning_rate": 1.7142857142857145e-06,
      "loss": -0.0055,
      "num_tokens": 147483.0,
      "reward": -0.35745370388031006,
      "reward_std": 0.03493613004684448,
      "rewards/rollout_reward_func/mean": -0.35745370388031006,
      "rewards/rollout_reward_func/std": 0.03385399281978607,
      "sampling/importance_sampling_ratio/max": 1.2162235975265503,
      "sampling/importance_sampling_ratio/mean": 1.009650707244873,
      "sampling/importance_sampling_ratio/min": 0.7808212637901306,
      "sampling/sampling_logp_difference/max": 0.2472679615020752,
      "sampling/sampling_logp_difference/mean": 0.03417329862713814,
      "step": 7,
      "step_time": 6.008490301999927
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 560.0,
      "completions/max_terminated_length": 560.0,
      "completions/mean_length": 115.4375,
      "completions/mean_terminated_length": 115.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9864411782473326,
      "epoch": 8e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2101781368255615,
      "kl": 0.0015320181846618652,
      "learning_rate": 2.0000000000000003e-06,
      "loss": -0.0185,
      "num_tokens": 169779.0,
      "reward": -0.25471001863479614,
      "reward_std": 0.23991067707538605,
      "rewards/rollout_reward_func/mean": -0.25471001863479614,
      "rewards/rollout_reward_func/std": 0.23637273907661438,
      "sampling/importance_sampling_ratio/max": 1.1370298862457275,
      "sampling/importance_sampling_ratio/mean": 0.9835214614868164,
      "sampling/importance_sampling_ratio/min": 0.6933529376983643,
      "sampling/sampling_logp_difference/max": 0.1895155906677246,
      "sampling/sampling_logp_difference/mean": 0.03515404462814331,
      "step": 8,
      "step_time": 8.127222840000059
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 668.0,
      "completions/max_terminated_length": 668.0,
      "completions/mean_length": 60.8125,
      "completions/mean_terminated_length": 60.8125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0561043992638588,
      "epoch": 9e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.332129716873169,
      "kl": 0.002313021570444107,
      "learning_rate": 2.285714285714286e-06,
      "loss": -0.0248,
      "num_tokens": 189529.0,
      "reward": -0.26064300537109375,
      "reward_std": 0.2326878309249878,
      "rewards/rollout_reward_func/mean": -0.26064300537109375,
      "rewards/rollout_reward_func/std": 0.2316972017288208,
      "sampling/importance_sampling_ratio/max": 1.3810378313064575,
      "sampling/importance_sampling_ratio/mean": 0.9809567928314209,
      "sampling/importance_sampling_ratio/min": 0.8159245848655701,
      "sampling/sampling_logp_difference/max": 0.32272958755493164,
      "sampling/sampling_logp_difference/mean": 0.049647510051727295,
      "step": 9,
      "step_time": 7.584354400999814
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 403.0,
      "completions/max_terminated_length": 403.0,
      "completions/mean_length": 48.75,
      "completions/mean_terminated_length": 48.75,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9946036152541637,
      "epoch": 0.0001,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.404557943344116,
      "kl": 0.008055873215198517,
      "learning_rate": 2.571428571428571e-06,
      "loss": -0.0304,
      "num_tokens": 209857.0,
      "reward": -0.21437498927116394,
      "reward_std": 0.2782182991504669,
      "rewards/rollout_reward_func/mean": -0.21437498927116394,
      "rewards/rollout_reward_func/std": 0.2919125258922577,
      "sampling/importance_sampling_ratio/max": 1.242602825164795,
      "sampling/importance_sampling_ratio/mean": 1.0025804042816162,
      "sampling/importance_sampling_ratio/min": 0.7827528715133667,
      "sampling/sampling_logp_difference/max": 0.2717958688735962,
      "sampling/sampling_logp_difference/mean": 0.03549234941601753,
      "step": 10,
      "step_time": 7.017077430000086
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 659.0,
      "completions/max_terminated_length": 659.0,
      "completions/mean_length": 75.1875,
      "completions/mean_terminated_length": 75.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.071075901389122,
      "epoch": 0.00011,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5599632263183594,
      "kl": 0.0099228136241436,
      "learning_rate": 2.8571428571428573e-06,
      "loss": -0.0138,
      "num_tokens": 231100.0,
      "reward": -0.3267521858215332,
      "reward_std": 0.09648184478282928,
      "rewards/rollout_reward_func/mean": -0.3267521858215332,
      "rewards/rollout_reward_func/std": 0.12373796850442886,
      "sampling/importance_sampling_ratio/max": 1.3085228204727173,
      "sampling/importance_sampling_ratio/mean": 0.9983298778533936,
      "sampling/importance_sampling_ratio/min": 0.8498186469078064,
      "sampling/sampling_logp_difference/max": 0.29364633560180664,
      "sampling/sampling_logp_difference/mean": 0.035756293684244156,
      "step": 11,
      "step_time": 8.156059186999983
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 623.0,
      "completions/max_terminated_length": 623.0,
      "completions/mean_length": 76.875,
      "completions/mean_terminated_length": 76.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0765549056231976,
      "epoch": 0.00012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6595567464828491,
      "kl": 0.013554699718952179,
      "learning_rate": 3.142857142857143e-06,
      "loss": -0.0194,
      "num_tokens": 252031.0,
      "reward": -0.21915188431739807,
      "reward_std": 0.23737147450447083,
      "rewards/rollout_reward_func/mean": -0.21915188431739807,
      "rewards/rollout_reward_func/std": 0.23919768631458282,
      "sampling/importance_sampling_ratio/max": 1.279653549194336,
      "sampling/importance_sampling_ratio/mean": 0.9805859923362732,
      "sampling/importance_sampling_ratio/min": 0.7882575392723083,
      "sampling/sampling_logp_difference/max": 0.2517733573913574,
      "sampling/sampling_logp_difference/mean": 0.04374777525663376,
      "step": 12,
      "step_time": 7.593313395000223
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 961.0,
      "completions/max_terminated_length": 961.0,
      "completions/mean_length": 92.0625,
      "completions/mean_terminated_length": 92.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.084555797278881,
      "epoch": 0.00013,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.7104790210723877,
      "kl": 0.020005052909255028,
      "learning_rate": 3.428571428571429e-06,
      "loss": -0.0669,
      "num_tokens": 272805.0,
      "reward": -0.25965023040771484,
      "reward_std": 0.22671905159950256,
      "rewards/rollout_reward_func/mean": -0.25965023040771484,
      "rewards/rollout_reward_func/std": 0.22195370495319366,
      "sampling/importance_sampling_ratio/max": 1.2232179641723633,
      "sampling/importance_sampling_ratio/mean": 1.020351767539978,
      "sampling/importance_sampling_ratio/min": 0.8783864378929138,
      "sampling/sampling_logp_difference/max": 0.2014932632446289,
      "sampling/sampling_logp_difference/mean": 0.028583938255906105,
      "step": 13,
      "step_time": 8.680951140000047
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.03125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1424.0,
      "completions/max_terminated_length": 1424.0,
      "completions/mean_length": 136.5,
      "completions/mean_terminated_length": 136.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0353318024426699,
      "epoch": 0.00014,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.6299335956573486,
      "kl": 0.04795516096055508,
      "learning_rate": 3.7142857142857146e-06,
      "loss": 0.0005,
      "num_tokens": 293335.0,
      "reward": -0.2034284919500351,
      "reward_std": 0.32647737860679626,
      "rewards/rollout_reward_func/mean": -0.2034284919500351,
      "rewards/rollout_reward_func/std": 0.31984227895736694,
      "sampling/importance_sampling_ratio/max": 1.4513046741485596,
      "sampling/importance_sampling_ratio/mean": 0.9897013902664185,
      "sampling/importance_sampling_ratio/min": 0.701361358165741,
      "sampling/sampling_logp_difference/max": 0.38347339630126953,
      "sampling/sampling_logp_difference/mean": 0.07703476399183273,
      "step": 14,
      "step_time": 10.529462050999996
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 970.0,
      "completions/max_terminated_length": 970.0,
      "completions/mean_length": 153.6875,
      "completions/mean_terminated_length": 153.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1377369370311499,
      "epoch": 0.00015,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.278023719787598,
      "kl": 0.08094949973747134,
      "learning_rate": 4.000000000000001e-06,
      "loss": -0.1217,
      "num_tokens": 315140.0,
      "reward": -0.26169753074645996,
      "reward_std": 0.244867742061615,
      "rewards/rollout_reward_func/mean": -0.26169753074645996,
      "rewards/rollout_reward_func/std": 0.2678700387477875,
      "sampling/importance_sampling_ratio/max": 1.8066322803497314,
      "sampling/importance_sampling_ratio/mean": 1.1207926273345947,
      "sampling/importance_sampling_ratio/min": 0.541466474533081,
      "sampling/sampling_logp_difference/max": 0.613422691822052,
      "sampling/sampling_logp_difference/mean": 0.10166005045175552,
      "step": 15,
      "step_time": 8.746920443999898
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1271.0,
      "completions/max_terminated_length": 1271.0,
      "completions/mean_length": 210.875,
      "completions/mean_terminated_length": 210.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.9875210858881474,
      "epoch": 0.00016,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.3659324645996094,
      "kl": 0.12660418590530753,
      "learning_rate": 4.2857142857142855e-06,
      "loss": -0.0252,
      "num_tokens": 338215.0,
      "reward": -0.2300102859735489,
      "reward_std": 0.3161442279815674,
      "rewards/rollout_reward_func/mean": -0.2300102859735489,
      "rewards/rollout_reward_func/std": 0.32718169689178467,
      "sampling/importance_sampling_ratio/max": 2.0122945308685303,
      "sampling/importance_sampling_ratio/mean": 1.2058320045471191,
      "sampling/importance_sampling_ratio/min": 0.6631097793579102,
      "sampling/sampling_logp_difference/max": 0.4927055835723877,
      "sampling/sampling_logp_difference/mean": 0.11450451612472534,
      "step": 16,
      "step_time": 9.826978175000136
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.03125,
      "clip_ratio/low_min": 0.03125,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/max_terminated_length": 1027.0,
      "completions/mean_length": 154.3125,
      "completions/mean_terminated_length": 154.3125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.2742707654833794,
      "epoch": 0.00017,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.975493907928467,
      "kl": 0.23780074575915933,
      "learning_rate": 4.571428571428572e-06,
      "loss": -0.0921,
      "num_tokens": 360683.0,
      "reward": -0.2627803385257721,
      "reward_std": 0.265349417924881,
      "rewards/rollout_reward_func/mean": -0.2627803385257721,
      "rewards/rollout_reward_func/std": 0.27273303270339966,
      "sampling/importance_sampling_ratio/max": 1.684834361076355,
      "sampling/importance_sampling_ratio/mean": 0.9126327633857727,
      "sampling/importance_sampling_ratio/min": 0.5164092183113098,
      "sampling/sampling_logp_difference/max": 0.6608381271362305,
      "sampling/sampling_logp_difference/mean": 0.13929885625839233,
      "step": 17,
      "step_time": 9.44654342299998
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.03125,
      "clip_ratio/low_min": 0.03125,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 560.0,
      "completions/max_terminated_length": 560.0,
      "completions/mean_length": 124.875,
      "completions/mean_terminated_length": 124.875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.1063523050397635,
      "epoch": 0.00018,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.501491069793701,
      "kl": 0.5189742557704449,
      "learning_rate": 4.857142857142858e-06,
      "loss": -0.0862,
      "num_tokens": 381892.0,
      "reward": -0.26139724254608154,
      "reward_std": 0.2675129473209381,
      "rewards/rollout_reward_func/mean": -0.26139724254608154,
      "rewards/rollout_reward_func/std": 0.272594690322876,
      "sampling/importance_sampling_ratio/max": 2.095355272293091,
      "sampling/importance_sampling_ratio/mean": 0.9401150941848755,
      "sampling/importance_sampling_ratio/min": 0.3087601959705353,
      "sampling/sampling_logp_difference/max": 1.175127625465393,
      "sampling/sampling_logp_difference/mean": 0.2186625748872757,
      "step": 18,
      "step_time": 7.373243494999883
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 759.0,
      "completions/max_terminated_length": 759.0,
      "completions/mean_length": 195.0625,
      "completions/mean_terminated_length": 161.40000915527344,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 1.0844341777265072,
      "epoch": 0.00019,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.8269429206848145,
      "kl": 1.1895455615594983,
      "learning_rate": 5.142857142857142e-06,
      "loss": -0.0441,
      "num_tokens": 404684.0,
      "reward": -0.24775463342666626,
      "reward_std": 0.24763348698616028,
      "rewards/rollout_reward_func/mean": -0.24775463342666626,
      "rewards/rollout_reward_func/std": 0.2522027790546417,
      "sampling/importance_sampling_ratio/max": 2.3281195163726807,
      "sampling/importance_sampling_ratio/mean": 1.0239425897598267,
      "sampling/importance_sampling_ratio/min": 0.19262944161891937,
      "sampling/sampling_logp_difference/max": 1.6467186212539673,
      "sampling/sampling_logp_difference/mean": 0.19436073303222656,
      "step": 19,
      "step_time": 8.138967235999758
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 964.0,
      "completions/max_terminated_length": 964.0,
      "completions/mean_length": 158.125,
      "completions/mean_terminated_length": 158.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.8784555904567242,
      "epoch": 0.0002,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.174610614776611,
      "kl": 2.745266660116613,
      "learning_rate": 5.428571428571429e-06,
      "loss": -0.0368,
      "num_tokens": 427032.0,
      "reward": -0.26112911105155945,
      "reward_std": 0.15938061475753784,
      "rewards/rollout_reward_func/mean": -0.26112911105155945,
      "rewards/rollout_reward_func/std": 0.16594691574573517,
      "sampling/importance_sampling_ratio/max": 1.3649011850357056,
      "sampling/importance_sampling_ratio/mean": 0.8665837049484253,
      "sampling/importance_sampling_ratio/min": 0.09711658209562302,
      "sampling/sampling_logp_difference/max": 2.3314242362976074,
      "sampling/sampling_logp_difference/mean": 0.2720341384410858,
      "step": 20,
      "step_time": 9.323931289999791
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 623.0,
      "completions/max_terminated_length": 623.0,
      "completions/mean_length": 214.4375,
      "completions/mean_terminated_length": 214.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7717013321816921,
      "epoch": 0.00021,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1020058393478394,
      "kl": 0.6415491066873074,
      "learning_rate": 5.7142857142857145e-06,
      "loss": -0.0769,
      "num_tokens": 449574.0,
      "reward": -0.22068029642105103,
      "reward_std": 0.26756125688552856,
      "rewards/rollout_reward_func/mean": -0.22068029642105103,
      "rewards/rollout_reward_func/std": 0.27299609780311584,
      "sampling/importance_sampling_ratio/max": 2.0030555725097656,
      "sampling/importance_sampling_ratio/mean": 1.1451401710510254,
      "sampling/importance_sampling_ratio/min": 0.20305787026882172,
      "sampling/sampling_logp_difference/max": 1.594210147857666,
      "sampling/sampling_logp_difference/mean": 0.17614634335041046,
      "step": 21,
      "step_time": 7.830380608000155
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1461.0,
      "completions/max_terminated_length": 1461.0,
      "completions/mean_length": 312.6875,
      "completions/mean_terminated_length": 312.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.7112401407212019,
      "epoch": 0.00022,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.7276053428649902,
      "kl": 4.1643759198486805,
      "learning_rate": 6e-06,
      "loss": -0.0314,
      "num_tokens": 474422.0,
      "reward": -0.14965856075286865,
      "reward_std": 0.3239442706108093,
      "rewards/rollout_reward_func/mean": -0.14965856075286865,
      "rewards/rollout_reward_func/std": 0.32331201434135437,
      "sampling/importance_sampling_ratio/max": 2.146582841873169,
      "sampling/importance_sampling_ratio/mean": 1.0021710395812988,
      "sampling/importance_sampling_ratio/min": 0.05034710839390755,
      "sampling/sampling_logp_difference/max": 2.988576889038086,
      "sampling/sampling_logp_difference/mean": 0.1974925547838211,
      "step": 22,
      "step_time": 10.44025386199985
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1123.0,
      "completions/max_terminated_length": 1123.0,
      "completions/mean_length": 325.4375,
      "completions/mean_terminated_length": 325.4375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.649243276566267,
      "epoch": 0.00023,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5880532264709473,
      "kl": 1.0535046725999564,
      "learning_rate": 6.285714285714286e-06,
      "loss": -0.0591,
      "num_tokens": 499592.0,
      "reward": -0.17797067761421204,
      "reward_std": 0.23259083926677704,
      "rewards/rollout_reward_func/mean": -0.17797067761421204,
      "rewards/rollout_reward_func/std": 0.23775041103363037,
      "sampling/importance_sampling_ratio/max": 2.2591254711151123,
      "sampling/importance_sampling_ratio/mean": 1.1730170249938965,
      "sampling/importance_sampling_ratio/min": 0.12334319204092026,
      "sampling/sampling_logp_difference/max": 2.0927066802978516,
      "sampling/sampling_logp_difference/mean": 0.18133001029491425,
      "step": 23,
      "step_time": 9.972772803999987
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1069.0,
      "completions/max_terminated_length": 1069.0,
      "completions/mean_length": 200.5,
      "completions/mean_terminated_length": 200.5,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5944561772048473,
      "epoch": 0.00024,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 19.25649070739746,
      "kl": 36.716627491638064,
      "learning_rate": 6.571428571428572e-06,
      "loss": -0.0262,
      "num_tokens": 521996.0,
      "reward": -0.08103395253419876,
      "reward_std": 0.30074936151504517,
      "rewards/rollout_reward_func/mean": -0.08103395253419876,
      "rewards/rollout_reward_func/std": 0.29864388704299927,
      "sampling/importance_sampling_ratio/max": 1.8443561792373657,
      "sampling/importance_sampling_ratio/mean": 1.1332334280014038,
      "sampling/importance_sampling_ratio/min": 0.014606568031013012,
      "sampling/sampling_logp_difference/max": 4.22576904296875,
      "sampling/sampling_logp_difference/mean": 0.24374164640903473,
      "step": 24,
      "step_time": 9.13615449800011
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 632.0,
      "completions/max_terminated_length": 632.0,
      "completions/mean_length": 230.0625,
      "completions/mean_terminated_length": 215.80001831054688,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.6250733993947506,
      "epoch": 0.00025,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 21.08238410949707,
      "kl": 54.73542622104287,
      "learning_rate": 6.857142857142858e-06,
      "loss": 0.0356,
      "num_tokens": 545181.0,
      "reward": -0.015234053134918213,
      "reward_std": 0.3025549054145813,
      "rewards/rollout_reward_func/mean": -0.015234053134918213,
      "rewards/rollout_reward_func/std": 0.37876686453819275,
      "sampling/importance_sampling_ratio/max": 2.128788948059082,
      "sampling/importance_sampling_ratio/mean": 1.1078401803970337,
      "sampling/importance_sampling_ratio/min": 0.017541952431201935,
      "sampling/sampling_logp_difference/max": 4.0429277420043945,
      "sampling/sampling_logp_difference/mean": 0.25233322381973267,
      "step": 25,
      "step_time": 7.908236747999922
    },
    {
      "clip_ratio/high_max": 0.015625,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.015625,
      "clip_ratio/region_mean": 0.03125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1175.0,
      "completions/max_terminated_length": 1175.0,
      "completions/mean_length": 366.6875,
      "completions/mean_terminated_length": 366.6875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.46145952865481377,
      "epoch": 0.00026,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.403465747833252,
      "kl": 1.1885524473618716,
      "learning_rate": 7.1428571428571436e-06,
      "loss": -0.0242,
      "num_tokens": 570618.0,
      "reward": -0.11929012089967728,
      "reward_std": 0.32874608039855957,
      "rewards/rollout_reward_func/mean": -0.11929012089967728,
      "rewards/rollout_reward_func/std": 0.32527726888656616,
      "sampling/importance_sampling_ratio/max": 1.505420446395874,
      "sampling/importance_sampling_ratio/mean": 0.7959405183792114,
      "sampling/importance_sampling_ratio/min": 0.0980805978178978,
      "sampling/sampling_logp_difference/max": 2.3273396492004395,
      "sampling/sampling_logp_difference/mean": 0.23728841543197632,
      "step": 26,
      "step_time": 10.291611079999939
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 641.0,
      "completions/max_terminated_length": 641.0,
      "completions/mean_length": 244.125,
      "completions/mean_terminated_length": 244.125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.5841637272387743,
      "epoch": 0.00027,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.8382925987243652,
      "kl": 1.3213790226727724,
      "learning_rate": 7.428571428571429e-06,
      "loss": -0.1023,
      "num_tokens": 593455.0,
      "reward": -0.18927982449531555,
      "reward_std": 0.24898916482925415,
      "rewards/rollout_reward_func/mean": -0.18927982449531555,
      "rewards/rollout_reward_func/std": 0.24998711049556732,
      "sampling/importance_sampling_ratio/max": 2.2187016010284424,
      "sampling/importance_sampling_ratio/mean": 0.9474911093711853,
      "sampling/importance_sampling_ratio/min": 0.11061210930347443,
      "sampling/sampling_logp_difference/max": 2.201629161834717,
      "sampling/sampling_logp_difference/mean": 0.22588896751403809,
      "step": 27,
      "step_time": 7.830289716000038
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1015.0,
      "completions/max_terminated_length": 1015.0,
      "completions/mean_length": 245.375,
      "completions/mean_terminated_length": 245.375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.4898219257593155,
      "epoch": 0.00028,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3174339532852173,
      "kl": 0.45907025039196014,
      "learning_rate": 7.714285714285716e-06,
      "loss": -0.056,
      "num_tokens": 615596.0,
      "reward": -0.11361882090568542,
      "reward_std": 0.2901168465614319,
      "rewards/rollout_reward_func/mean": -0.11361882090568542,
      "rewards/rollout_reward_func/std": 0.2821318209171295,
      "sampling/importance_sampling_ratio/max": 1.5459662675857544,
      "sampling/importance_sampling_ratio/mean": 0.8522274494171143,
      "sampling/importance_sampling_ratio/min": 0.3117474317550659,
      "sampling/sampling_logp_difference/max": 1.1996822357177734,
      "sampling/sampling_logp_difference/mean": 0.16573527455329895,
      "step": 28,
      "step_time": 8.996423046000018
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.015625,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 970.0,
      "completions/max_terminated_length": 970.0,
      "completions/mean_length": 354.0625,
      "completions/mean_terminated_length": 354.0625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 0.3801212254911661,
      "epoch": 0.00029,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.826854407787323,
      "kl": 1.4256584625691175,
      "learning_rate": 8.000000000000001e-06,
      "loss": -0.0559,
      "num_tokens": 641634.0,
      "reward": 0.06936728954315186,
      "reward_std": 0.3731154799461365,
      "rewards/rollout_reward_func/mean": 0.06936728954315186,
      "rewards/rollout_reward_func/std": 0.3700728714466095,
      "sampling/importance_sampling_ratio/max": 2.0329487323760986,
      "sampling/importance_sampling_ratio/mean": 0.9922663569450378,
      "sampling/importance_sampling_ratio/min": 0.10419858247041702,
      "sampling/sampling_logp_difference/max": 2.2612314224243164,
      "sampling/sampling_logp_difference/mean": 0.21723516285419464,
      "step": 29,
      "step_time": 9.553925147000086
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 1300,
  "num_input_tokens_seen": 641634,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}