{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.05776173285198556, "eval_steps": 500, "global_step": 16, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.875, "completions/max_length": 256.0, "completions/max_terminated_length": 209.0, "completions/mean_length": 239.875, "completions/mean_terminated_length": 127.0, "completions/min_length": 66.0, "completions/min_terminated_length": 66.0, "epoch": 0.0036101083032490976, "grad_norm": 0.16485562920570374, "kl": 0.0, "learning_rate": 0.0, "loss": 0.0924, "num_tokens": 8764.0, "reward": -3225.985595703125, "reward_std": 817.931884765625, "rewards/reward_short_completions/mean": -1077.40625, "rewards/reward_short_completions/std": 289.6771240234375, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.8125, "completions/max_length": 256.0, "completions/max_terminated_length": 154.0, "completions/mean_length": 222.8125, "completions/mean_terminated_length": 79.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "epoch": 0.007220216606498195, "grad_norm": 0.2007177472114563, "kl": 0.0, "learning_rate": 5e-05, "loss": 0.0861, "num_tokens": 16882.0, "reward": -3076.5556640625, "reward_std": 1139.9786376953125, "rewards/reward_short_completions/mean": -1027.5, "rewards/reward_short_completions/std": 402.35443115234375, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.96875, "completions/max_length": 256.0, "completions/max_terminated_length": 4.0, "completions/mean_length": 248.125, "completions/mean_terminated_length": 4.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.010830324909747292, "grad_norm": 0.15252335369586945, "kl": 0.0005898440431337804, "learning_rate": 0.0001, "loss": 0.0572, "num_tokens": 25710.0, "reward": -3099.10595703125, "reward_std": 719.3511352539062, "rewards/reward_short_completions/mean": -1035.03125, "rewards/reward_short_completions/std": 288.5079040527344, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.90625, "completions/max_length": 256.0, "completions/max_terminated_length": 231.0, "completions/mean_length": 244.5625, "completions/mean_terminated_length": 134.0, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 0.01444043321299639, "grad_norm": 0.17062342166900635, "kl": 0.0007059574127197266, "learning_rate": 9.924038765061042e-05, "loss": 0.0746, "num_tokens": 34300.0, "reward": -3156.18310546875, "reward_std": 706.9985961914062, "rewards/reward_short_completions/mean": -1054.09375, "rewards/reward_short_completions/std": 286.53179931640625, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.018050541516245487, "grad_norm": 0.17358577251434326, "kl": 0.0008631395467091352, "learning_rate": 9.698463103929542e-05, "loss": 0.0, "num_tokens": 43368.0, "reward": -3253.4951171875, "reward_std": 581.868408203125, "rewards/reward_short_completions/mean": -1086.59375, "rewards/reward_short_completions/std": 254.26222229003906, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.8125, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 234.09375, "completions/mean_terminated_length": 139.1666717529297, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "epoch": 0.021660649819494584, "grad_norm": 0.17494061589241028, "kl": 0.0012313149636611342, "learning_rate": 9.330127018922194e-05, "loss": 0.1467, "num_tokens": 51739.0, "reward": -3192.11376953125, "reward_std": 729.4127197265625, "rewards/reward_short_completions/mean": -1066.09375, "rewards/reward_short_completions/std": 342.6435546875, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.84375, "completions/max_length": 256.0, "completions/max_terminated_length": 198.0, "completions/mean_length": 230.5, "completions/mean_terminated_length": 92.80000305175781, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "epoch": 0.02527075812274368, "grad_norm": 0.19088327884674072, "kl": 0.0013786845956929028, "learning_rate": 8.83022221559489e-05, "loss": 0.1449, "num_tokens": 60243.0, "reward": -2963.8046875, "reward_std": 874.9676513671875, "rewards/reward_short_completions/mean": -989.84375, "rewards/reward_short_completions/std": 346.3221435546875, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.84375, "completions/max_length": 256.0, "completions/max_terminated_length": 242.0, "completions/mean_length": 234.3125, "completions/mean_terminated_length": 117.20000457763672, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "epoch": 0.02888086642599278, "grad_norm": 0.19052405655384064, "kl": 0.0019435517024248838, "learning_rate": 8.213938048432697e-05, "loss": 0.1222, "num_tokens": 68921.0, "reward": -3151.4111328125, "reward_std": 903.66650390625, "rewards/reward_short_completions/mean": -1052.5, "rewards/reward_short_completions/std": 332.7859802246094, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.84375, "completions/max_length": 256.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 230.84375, "completions/mean_terminated_length": 95.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.032490974729241874, "grad_norm": 0.16807924211025238, "kl": 0.0023136032978072762, "learning_rate": 7.500000000000001e-05, "loss": 0.0992, "num_tokens": 77292.0, "reward": -2821.11181640625, "reward_std": 873.7684326171875, "rewards/reward_short_completions/mean": -942.1875, "rewards/reward_short_completions/std": 345.1814270019531, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.90625, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 246.6875, "completions/mean_terminated_length": 156.6666717529297, "completions/min_length": 65.0, "completions/min_terminated_length": 65.0, "epoch": 0.036101083032490974, "grad_norm": 0.16984766721725464, "kl": 0.002263773581944406, "learning_rate": 6.710100716628344e-05, "loss": 0.0557, "num_tokens": 86006.0, "reward": -3042.40283203125, "reward_std": 554.4484252929688, "rewards/reward_short_completions/mean": -1016.09375, "rewards/reward_short_completions/std": 238.2257537841797, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6875, "completions/max_length": 256.0, "completions/max_terminated_length": 204.0, "completions/mean_length": 193.40625, "completions/mean_terminated_length": 55.70000076293945, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "epoch": 0.039711191335740074, "grad_norm": 0.3081122040748596, "kl": 0.005528636509552598, "learning_rate": 5.868240888334653e-05, "loss": 0.2781, "num_tokens": 92979.0, "reward": -2254.26953125, "reward_std": 1479.353515625, "rewards/reward_short_completions/mean": -752.875, "rewards/reward_short_completions/std": 485.4302673339844, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.75, "completions/max_length": 256.0, "completions/max_terminated_length": 231.0, "completions/mean_length": 218.65625, "completions/mean_terminated_length": 106.625, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "epoch": 0.04332129963898917, "grad_norm": 0.27785155177116394, "kl": 0.006482115248218179, "learning_rate": 5e-05, "loss": 0.1416, "num_tokens": 101036.0, "reward": -2619.00244140625, "reward_std": 958.5040283203125, "rewards/reward_short_completions/mean": -874.6875, "rewards/reward_short_completions/std": 373.1864929199219, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.875, "completions/max_length": 256.0, "completions/max_terminated_length": 84.0, "completions/mean_length": 228.40625, "completions/mean_terminated_length": 35.25, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "epoch": 0.04693140794223827, "grad_norm": 0.17809373140335083, "kl": 0.006034299731254578, "learning_rate": 4.131759111665349e-05, "loss": 0.204, "num_tokens": 109277.0, "reward": -2761.41455078125, "reward_std": 1031.23828125, "rewards/reward_short_completions/mean": -922.25, "rewards/reward_short_completions/std": 359.5296325683594, "step": 13 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 256.0, "completions/max_terminated_length": 191.0, "completions/mean_length": 182.90625, "completions/mean_terminated_length": 76.0769271850586, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "epoch": 0.05054151624548736, "grad_norm": 0.2206607162952423, "kl": 0.007222140673547983, "learning_rate": 3.289899283371657e-05, "loss": 0.3145, "num_tokens": 116158.0, "reward": -2017.7264404296875, "reward_std": 1226.109375, "rewards/reward_short_completions/mean": -673.875, "rewards/reward_short_completions/std": 398.3905944824219, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 256.0, "completions/max_terminated_length": 229.0, "completions/mean_length": 167.8125, "completions/mean_terminated_length": 54.42857360839844, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.05415162454873646, "grad_norm": 0.32962244749069214, "kl": 0.016009118407964706, "learning_rate": 2.500000000000001e-05, "loss": 0.4223, "num_tokens": 122636.0, "reward": -1981.421630859375, "reward_std": 1539.54248046875, "rewards/reward_short_completions/mean": -661.75, "rewards/reward_short_completions/std": 477.202392578125, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.84375, "completions/max_length": 256.0, "completions/max_terminated_length": 20.0, "completions/mean_length": 217.09375, "completions/mean_terminated_length": 7.0, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "epoch": 0.05776173285198556, "grad_norm": 0.19252434372901917, "kl": 0.00875580357387662, "learning_rate": 1.7860619515673033e-05, "loss": 0.2923, "num_tokens": 130343.0, "reward": -2564.35791015625, "reward_std": 1129.935302734375, "rewards/reward_short_completions/mean": -856.4375, "rewards/reward_short_completions/std": 436.13079833984375, "step": 16 } ], "logging_steps": 1, "max_steps": 20, "num_input_tokens_seen": 130343, "num_train_epochs": 1, "save_steps": 4, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }