Training in progress, step 470, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:269490a7deebb617c30aa9ef5c8ab977f0409d15fe2b4e98e33163a150da8fc2
+oid sha256:e97f0845aa6a67e07d4147bdbc645881cd725e485863c57bcc61520e8aa526f0
 size 194563400

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0831cfdbc23b67a5dd6fcbd7ed643941532ebde5ee793f4a4f8246d61fce7c06
+oid sha256:e076fb439be773d56148898b13cdfb393391bdee8311e86bd9314c0519333d5a
 size 100256339

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e35b7423ba0e6873a0c4bc838b1db796a6804557a4eb80dd29c8a2a14b7b2e37
+oid sha256:f759ddb7949658691c66ca99f40d508b0c01db148b293eb4b7d316ed5233a57e
 size 14645

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab8e53ebff7441e7eeaf464f67a666948d42ca62edc867d22536c1b5009f6e9a
+oid sha256:8471f0babb163e30a8ef2747dbd9c4e7f32c7d3318b6dd67c171a8d396f73ab6
 size 1383

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0bfddf1ecfdce6e753af140ec77040fea6a7f5ce90d81c2887eb4bbf5b50389
+oid sha256:b15150ddaa3faf35858f43938a74f78c04812ac2062419c41ac0afaa4a8ae794
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 24.210526315789473,
+  "epoch": 24.736842105263158,
   "eval_steps": 500,
-  "global_step": 460,
+  "global_step": 470,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -10128,11 +10128,231 @@
       "rewards/compute_weighted_reward/mean": 0.4879167079925537,
       "rewards/compute_weighted_reward/std": 1.3894351720809937,
       "step": 460
+    },
+    {
+      "completion_length": 1400.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 1400.0,
+      "completions/max_terminated_length": 975.0,
+      "completions/mean_length": 795.3333740234375,
+      "completions/mean_terminated_length": 593.7777709960938,
+      "completions/min_length": 327.0,
+      "completions/min_terminated_length": 327.0,
+      "epoch": 24.263157894736842,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 2.256335496902466,
+      "kl": 0.006880732020363212,
+      "learning_rate": 4.472163787765637e-09,
+      "loss": 0.0069,
+      "num_tokens": 12572861.0,
+      "reward": 1.6516666412353516,
+      "reward_std": 1.1727336645126343,
+      "rewards/compute_weighted_reward/mean": 1.6516666412353516,
+      "rewards/compute_weighted_reward/std": 1.6009553670883179,
+      "step": 461
+    },
+    {
+      "completion_length": 1400.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 1400.0,
+      "completions/max_terminated_length": 1265.0,
+      "completions/mean_length": 871.6666870117188,
+      "completions/mean_terminated_length": 695.5555419921875,
+      "completions/min_length": 264.0,
+      "completions/min_terminated_length": 264.0,
+      "epoch": 24.31578947368421,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 2.1168675422668457,
+      "kl": 0.005777199752628803,
+      "learning_rate": 4.2680502467932756e-09,
+      "loss": 0.0058,
+      "num_tokens": 12601245.0,
+      "reward": 0.7454167604446411,
+      "reward_std": 0.9587024450302124,
+      "rewards/compute_weighted_reward/mean": 0.7454167008399963,
+      "rewards/compute_weighted_reward/std": 1.4346518516540527,
+      "step": 462
+    },
+    {
+      "completion_length": 1400.0,
+      "completions/clipped_ratio": 0.29166666666666663,
+      "completions/max_length": 1400.0,
+      "completions/max_terminated_length": 1360.0,
+      "completions/mean_length": 902.2083740234375,
+      "completions/mean_terminated_length": 697.2352905273438,
+      "completions/min_length": 335.0,
+      "completions/min_terminated_length": 335.0,
+      "epoch": 24.36842105263158,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 17.3585147857666,
+      "kl": 0.0072209787322208285,
+      "learning_rate": 4.068602545994249e-09,
+      "loss": 0.0072,
+      "num_tokens": 12630596.0,
+      "reward": 0.4395833909511566,
+      "reward_std": 1.1506423950195312,
+      "rewards/compute_weighted_reward/mean": 0.4395833909511566,
+      "rewards/compute_weighted_reward/std": 1.45299232006073,
+      "step": 463
+    },
+    {
+      "completion_length": 1400.0,
+      "completions/clipped_ratio": 0.08333333333333337,
+      "completions/max_length": 1400.0,
+      "completions/max_terminated_length": 1076.0,
+      "completions/mean_length": 646.2916870117188,
+      "completions/mean_terminated_length": 577.7727661132812,
+      "completions/min_length": 328.0,
+      "completions/min_terminated_length": 328.0,
+      "epoch": 24.42105263157895,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 4.130858898162842,
+      "kl": 0.011352596920914948,
+      "learning_rate": 3.873830406168111e-09,
+      "loss": 0.0114,
+      "num_tokens": 12653349.0,
+      "reward": 1.3262500762939453,
+      "reward_std": 0.8212506175041199,
+      "rewards/compute_weighted_reward/mean": 1.3262500762939453,
+      "rewards/compute_weighted_reward/std": 1.3137494325637817,
+      "step": 464
+    },
+    {
+      "completion_length": 1400.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 1400.0,
+      "completions/max_terminated_length": 1138.0,
+      "completions/mean_length": 720.2083740234375,
+      "completions/mean_terminated_length": 623.0952758789062,
+      "completions/min_length": 260.0,
+      "completions/min_terminated_length": 260.0,
+      "epoch": 24.473684210526315,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 5.011742115020752,
+      "kl": 0.007324963924475014,
+      "learning_rate": 3.6837433202341896e-09,
+      "loss": 0.0073,
+      "num_tokens": 12677888.0,
+      "reward": 1.6141667366027832,
+      "reward_std": 1.100345492362976,
+      "rewards/compute_weighted_reward/mean": 1.6141667366027832,
+      "rewards/compute_weighted_reward/std": 1.6239241361618042,
+      "step": 465
+    },
+    {
+      "completion_length": 1400.0,
+      "completions/clipped_ratio": 0.375,
+      "completions/max_length": 1400.0,
+      "completions/max_terminated_length": 1268.0,
+      "completions/mean_length": 916.6666870117188,
+      "completions/mean_terminated_length": 626.6666870117188,
+      "completions/min_length": 227.0,
+      "completions/min_terminated_length": 227.0,
+      "epoch": 24.526315789473685,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.586663722991943,
+      "kl": 0.006580363435205072,
+      "learning_rate": 3.4983505527688583e-09,
+      "loss": 0.0066,
+      "num_tokens": 12707922.0,
+      "reward": 0.42875003814697266,
+      "reward_std": 1.1746701002120972,
+      "rewards/compute_weighted_reward/mean": 0.42875003814697266,
+      "rewards/compute_weighted_reward/std": 1.6151788234710693,
+      "step": 466
+    },
+    {
+      "completion_length": 1400.0,
+      "completions/clipped_ratio": 0.45833333333333337,
+      "completions/max_length": 1400.0,
+      "completions/max_terminated_length": 1269.0,
+      "completions/mean_length": 1025.25,
+      "completions/mean_terminated_length": 708.1538696289062,
+      "completions/min_length": 122.0,
+      "completions/min_terminated_length": 122.0,
+      "epoch": 24.57894736842105,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 7.444450855255127,
+      "kl": 0.008480201940983534,
+      "learning_rate": 3.317661139554062e-09,
+      "loss": 0.0085,
+      "num_tokens": 12740580.0,
+      "reward": -0.2133333384990692,
+      "reward_std": 1.4371541738510132,
+      "rewards/compute_weighted_reward/mean": -0.21333330869674683,
+      "rewards/compute_weighted_reward/std": 1.4814025163650513,
+      "step": 467
+    },
+    {
+      "completion_length": 1400.0,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 1400.0,
+      "completions/max_terminated_length": 1121.0,
+      "completions/mean_length": 957.75,
+      "completions/mean_terminated_length": 515.5,
+      "completions/min_length": 291.0,
+      "completions/min_terminated_length": 291.0,
+      "epoch": 24.63157894736842,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 12.957950592041016,
+      "kl": 0.0020423264359124005,
+      "learning_rate": 3.141683887136892e-09,
+      "loss": 0.002,
+      "num_tokens": 12770844.0,
+      "reward": 0.46041667461395264,
+      "reward_std": 0.7552928924560547,
+      "rewards/compute_weighted_reward/mean": 0.460416704416275,
+      "rewards/compute_weighted_reward/std": 1.6309572458267212,
+      "step": 468
+    },
+    {
+      "completion_length": 1400.0,
+      "completions/clipped_ratio": 0.33333333333333337,
+      "completions/max_length": 1400.0,
+      "completions/max_terminated_length": 1305.0,
+      "completions/mean_length": 895.875,
+      "completions/mean_terminated_length": 643.8125,
+      "completions/min_length": 227.0,
+      "completions/min_terminated_length": 227.0,
+      "epoch": 24.68421052631579,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 2.4682981967926025,
+      "kl": 0.004297248087823391,
+      "learning_rate": 2.9704273724003525e-09,
+      "loss": 0.0043,
+      "num_tokens": 12799539.0,
+      "reward": 0.7029167413711548,
+      "reward_std": 0.8361326456069946,
+      "rewards/compute_weighted_reward/mean": 0.70291668176651,
+      "rewards/compute_weighted_reward/std": 1.548310399055481,
+      "step": 469
+    },
+    {
+      "completion_length": 1400.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 1400.0,
+      "completions/max_terminated_length": 1051.0,
+      "completions/mean_length": 685.0416870117188,
+      "completions/mean_terminated_length": 582.90478515625,
+      "completions/min_length": 152.0,
+      "completions/min_terminated_length": 152.0,
+      "epoch": 24.736842105263158,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 8.62110710144043,
+      "kl": 0.0092578538460657,
+      "learning_rate": 2.803899942145382e-09,
+      "loss": 0.0093,
+      "num_tokens": 12823528.0,
+      "reward": 0.8570834994316101,
+      "reward_std": 1.151677131652832,
+      "rewards/compute_weighted_reward/mean": 0.8570833802223206,
+      "rewards/compute_weighted_reward/std": 1.384502649307251,
+      "step": 470
     }
   ],
   "logging_steps": 1,
   "max_steps": 500,
-  "num_input_tokens_seen": 12546519,
+  "num_input_tokens_seen": 12823528,
   "num_train_epochs": 27,
   "save_steps": 10,
   "stateful_callbacks": {