| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.05776173285198556, |
| "eval_steps": 500, |
| "global_step": 16, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 209.0, |
| "completions/mean_length": 239.875, |
| "completions/mean_terminated_length": 127.0, |
| "completions/min_length": 66.0, |
| "completions/min_terminated_length": 66.0, |
| "epoch": 0.0036101083032490976, |
| "grad_norm": 0.16485562920570374, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": 0.0924, |
| "num_tokens": 8764.0, |
| "reward": -3225.985595703125, |
| "reward_std": 817.931884765625, |
| "rewards/reward_short_completions/mean": -1077.40625, |
| "rewards/reward_short_completions/std": 289.6771240234375, |
| "step": 1 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8125, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 154.0, |
| "completions/mean_length": 222.8125, |
| "completions/mean_terminated_length": 79.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "epoch": 0.007220216606498195, |
| "grad_norm": 0.2007177472114563, |
| "kl": 0.0, |
| "learning_rate": 5e-05, |
| "loss": 0.0861, |
| "num_tokens": 16882.0, |
| "reward": -3076.5556640625, |
| "reward_std": 1139.9786376953125, |
| "rewards/reward_short_completions/mean": -1027.5, |
| "rewards/reward_short_completions/std": 402.35443115234375, |
| "step": 2 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.96875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 4.0, |
| "completions/mean_length": 248.125, |
| "completions/mean_terminated_length": 4.0, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "epoch": 0.010830324909747292, |
| "grad_norm": 0.15252335369586945, |
| "kl": 0.0005898440431337804, |
| "learning_rate": 0.0001, |
| "loss": 0.0572, |
| "num_tokens": 25710.0, |
| "reward": -3099.10595703125, |
| "reward_std": 719.3511352539062, |
| "rewards/reward_short_completions/mean": -1035.03125, |
| "rewards/reward_short_completions/std": 288.5079040527344, |
| "step": 3 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.90625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 231.0, |
| "completions/mean_length": 244.5625, |
| "completions/mean_terminated_length": 134.0, |
| "completions/min_length": 24.0, |
| "completions/min_terminated_length": 24.0, |
| "epoch": 0.01444043321299639, |
| "grad_norm": 0.17062342166900635, |
| "kl": 0.0007059574127197266, |
| "learning_rate": 9.924038765061042e-05, |
| "loss": 0.0746, |
| "num_tokens": 34300.0, |
| "reward": -3156.18310546875, |
| "reward_std": 706.9985961914062, |
| "rewards/reward_short_completions/mean": -1054.09375, |
| "rewards/reward_short_completions/std": 286.53179931640625, |
| "step": 4 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 1.0, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 0.0, |
| "completions/mean_length": 256.0, |
| "completions/mean_terminated_length": 0.0, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 0.0, |
| "epoch": 0.018050541516245487, |
| "grad_norm": 0.17358577251434326, |
| "kl": 0.0008631395467091352, |
| "learning_rate": 9.698463103929542e-05, |
| "loss": 0.0, |
| "num_tokens": 43368.0, |
| "reward": -3253.4951171875, |
| "reward_std": 581.868408203125, |
| "rewards/reward_short_completions/mean": -1086.59375, |
| "rewards/reward_short_completions/std": 254.26222229003906, |
| "step": 5 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.8125, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 252.0, |
| "completions/mean_length": 234.09375, |
| "completions/mean_terminated_length": 139.1666717529297, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "epoch": 0.021660649819494584, |
| "grad_norm": 0.17494061589241028, |
| "kl": 0.0012313149636611342, |
| "learning_rate": 9.330127018922194e-05, |
| "loss": 0.1467, |
| "num_tokens": 51739.0, |
| "reward": -3192.11376953125, |
| "reward_std": 729.4127197265625, |
| "rewards/reward_short_completions/mean": -1066.09375, |
| "rewards/reward_short_completions/std": 342.6435546875, |
| "step": 6 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.84375, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 198.0, |
| "completions/mean_length": 230.5, |
| "completions/mean_terminated_length": 92.80000305175781, |
| "completions/min_length": 26.0, |
| "completions/min_terminated_length": 26.0, |
| "epoch": 0.02527075812274368, |
| "grad_norm": 0.19088327884674072, |
| "kl": 0.0013786845956929028, |
| "learning_rate": 8.83022221559489e-05, |
| "loss": 0.1449, |
| "num_tokens": 60243.0, |
| "reward": -2963.8046875, |
| "reward_std": 874.9676513671875, |
| "rewards/reward_short_completions/mean": -989.84375, |
| "rewards/reward_short_completions/std": 346.3221435546875, |
| "step": 7 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.84375, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 242.0, |
| "completions/mean_length": 234.3125, |
| "completions/mean_terminated_length": 117.20000457763672, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "epoch": 0.02888086642599278, |
| "grad_norm": 0.19052405655384064, |
| "kl": 0.0019435517024248838, |
| "learning_rate": 8.213938048432697e-05, |
| "loss": 0.1222, |
| "num_tokens": 68921.0, |
| "reward": -3151.4111328125, |
| "reward_std": 903.66650390625, |
| "rewards/reward_short_completions/mean": -1052.5, |
| "rewards/reward_short_completions/std": 332.7859802246094, |
| "step": 8 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.84375, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 230.84375, |
| "completions/mean_terminated_length": 95.0, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.032490974729241874, |
| "grad_norm": 0.16807924211025238, |
| "kl": 0.0023136032978072762, |
| "learning_rate": 7.500000000000001e-05, |
| "loss": 0.0992, |
| "num_tokens": 77292.0, |
| "reward": -2821.11181640625, |
| "reward_std": 873.7684326171875, |
| "rewards/reward_short_completions/mean": -942.1875, |
| "rewards/reward_short_completions/std": 345.1814270019531, |
| "step": 9 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.90625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 251.0, |
| "completions/mean_length": 246.6875, |
| "completions/mean_terminated_length": 156.6666717529297, |
| "completions/min_length": 65.0, |
| "completions/min_terminated_length": 65.0, |
| "epoch": 0.036101083032490974, |
| "grad_norm": 0.16984766721725464, |
| "kl": 0.002263773581944406, |
| "learning_rate": 6.710100716628344e-05, |
| "loss": 0.0557, |
| "num_tokens": 86006.0, |
| "reward": -3042.40283203125, |
| "reward_std": 554.4484252929688, |
| "rewards/reward_short_completions/mean": -1016.09375, |
| "rewards/reward_short_completions/std": 238.2257537841797, |
| "step": 10 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.6875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 204.0, |
| "completions/mean_length": 193.40625, |
| "completions/mean_terminated_length": 55.70000076293945, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "epoch": 0.039711191335740074, |
| "grad_norm": 0.3081122040748596, |
| "kl": 0.005528636509552598, |
| "learning_rate": 5.868240888334653e-05, |
| "loss": 0.2781, |
| "num_tokens": 92979.0, |
| "reward": -2254.26953125, |
| "reward_std": 1479.353515625, |
| "rewards/reward_short_completions/mean": -752.875, |
| "rewards/reward_short_completions/std": 485.4302673339844, |
| "step": 11 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.75, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 231.0, |
| "completions/mean_length": 218.65625, |
| "completions/mean_terminated_length": 106.625, |
| "completions/min_length": 19.0, |
| "completions/min_terminated_length": 19.0, |
| "epoch": 0.04332129963898917, |
| "grad_norm": 0.27785155177116394, |
| "kl": 0.006482115248218179, |
| "learning_rate": 5e-05, |
| "loss": 0.1416, |
| "num_tokens": 101036.0, |
| "reward": -2619.00244140625, |
| "reward_std": 958.5040283203125, |
| "rewards/reward_short_completions/mean": -874.6875, |
| "rewards/reward_short_completions/std": 373.1864929199219, |
| "step": 12 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.875, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 84.0, |
| "completions/mean_length": 228.40625, |
| "completions/mean_terminated_length": 35.25, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "epoch": 0.04693140794223827, |
| "grad_norm": 0.17809373140335083, |
| "kl": 0.006034299731254578, |
| "learning_rate": 4.131759111665349e-05, |
| "loss": 0.204, |
| "num_tokens": 109277.0, |
| "reward": -2761.41455078125, |
| "reward_std": 1031.23828125, |
| "rewards/reward_short_completions/mean": -922.25, |
| "rewards/reward_short_completions/std": 359.5296325683594, |
| "step": 13 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.59375, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 191.0, |
| "completions/mean_length": 182.90625, |
| "completions/mean_terminated_length": 76.0769271850586, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "epoch": 0.05054151624548736, |
| "grad_norm": 0.2206607162952423, |
| "kl": 0.007222140673547983, |
| "learning_rate": 3.289899283371657e-05, |
| "loss": 0.3145, |
| "num_tokens": 116158.0, |
| "reward": -2017.7264404296875, |
| "reward_std": 1226.109375, |
| "rewards/reward_short_completions/mean": -673.875, |
| "rewards/reward_short_completions/std": 398.3905944824219, |
| "step": 14 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.5625, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 229.0, |
| "completions/mean_length": 167.8125, |
| "completions/mean_terminated_length": 54.42857360839844, |
| "completions/min_length": 1.0, |
| "completions/min_terminated_length": 1.0, |
| "epoch": 0.05415162454873646, |
| "grad_norm": 0.32962244749069214, |
| "kl": 0.016009118407964706, |
| "learning_rate": 2.500000000000001e-05, |
| "loss": 0.4223, |
| "num_tokens": 122636.0, |
| "reward": -1981.421630859375, |
| "reward_std": 1539.54248046875, |
| "rewards/reward_short_completions/mean": -661.75, |
| "rewards/reward_short_completions/std": 477.202392578125, |
| "step": 15 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.84375, |
| "completions/max_length": 256.0, |
| "completions/max_terminated_length": 20.0, |
| "completions/mean_length": 217.09375, |
| "completions/mean_terminated_length": 7.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "epoch": 0.05776173285198556, |
| "grad_norm": 0.19252434372901917, |
| "kl": 0.00875580357387662, |
| "learning_rate": 1.7860619515673033e-05, |
| "loss": 0.2923, |
| "num_tokens": 130343.0, |
| "reward": -2564.35791015625, |
| "reward_std": 1129.935302734375, |
| "rewards/reward_short_completions/mean": -856.4375, |
| "rewards/reward_short_completions/std": 436.13079833984375, |
| "step": 16 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 20, |
| "num_input_tokens_seen": 130343, |
| "num_train_epochs": 1, |
| "save_steps": 4, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
