Training in progress, step 490, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65eb9c7682f606c7eedce0b4ba801f5bce84965b524de4d7a696e98178cb8567
+oid sha256:1402844c18aeb9c024b30f9f23a9c32c507ea60a64defb20c169a2539f9385d1
 size 194563400

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:720b9f75c33bc32b37084e17c701d88f36db88b0c4783e5eb687d5546a252121
+oid sha256:8d70bace88c1389d4b58e3fb6f248400c0142e1a1ca3e51d35870aa8eb36ebce
 size 100256339

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77864e13c88d06004fb6352ebb2cf276581d7aa7fca11c0011ab8f5f84e04bae
+oid sha256:055ac3c369a5e930fe04e9758176075cca45da315a2b1d5fc770be3582b8afee
 size 14645

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d48ee9d9909680ca611f0a95c8cefcadb338dd2851b722337f41dd0606fbe3b
+oid sha256:f1992ea3bf098e43650cdec4c9702d09710b1b566a09c123b480dba8f6266dd0
 size 1383

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fd4d399305074df04285b10af1df43ce0c6062922653613f50696673d2b5b78
+oid sha256:9e559e024e64978c57350fefa54232ddfa7d2568f6cf9d5df8a027da19545eb1
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.2018348623853212,
+  "epoch": 2.2477064220183487,
   "eval_steps": 500,
-  "global_step": 480,
+  "global_step": 490,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -10568,11 +10568,231 @@
       "rewards/compute_weighted_reward/mean": 0.07333331555128098,
       "rewards/compute_weighted_reward/std": 0.9759306311607361,
       "step": 480
+    },
+    {
+      "completion_length": 1422.7083333333333,
+      "completions/clipped_ratio": 0.625,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 1969.0,
+      "completions/mean_length": 1507.875,
+      "completions/mean_terminated_length": 687.6666870117188,
+      "completions/min_length": 392.0,
+      "completions/min_terminated_length": 392.0,
+      "epoch": 2.206422018348624,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.36272257566452026,
+      "kl": 0.0001912083589559188,
+      "learning_rate": 4.298349500846628e-08,
+      "loss": 0.1471,
+      "num_tokens": 15153385.0,
+      "reward": -0.25708338618278503,
+      "reward_std": 0.6896874308586121,
+      "rewards/compute_weighted_reward/mean": -0.25708332657814026,
+      "rewards/compute_weighted_reward/std": 1.1804289817810059,
+      "step": 481
+    },
+    {
+      "completion_length": 1335.875,
+      "completions/clipped_ratio": 0.5416666666666667,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 1679.0,
+      "completions/mean_length": 1403.375,
+      "completions/mean_terminated_length": 698.2727661132812,
+      "completions/min_length": 270.0,
+      "completions/min_terminated_length": 270.0,
+      "epoch": 2.2110091743119265,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5260336399078369,
+      "kl": 0.0002945731584986788,
+      "learning_rate": 4.294303276506441e-08,
+      "loss": 0.2248,
+      "num_tokens": 15195334.0,
+      "reward": -0.29208335280418396,
+      "reward_std": 0.6641569137573242,
+      "rewards/compute_weighted_reward/mean": -0.29208335280418396,
+      "rewards/compute_weighted_reward/std": 0.8769609928131104,
+      "step": 482
+    },
+    {
+      "completion_length": 1571.375,
+      "completions/clipped_ratio": 0.625,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 1997.0,
+      "completions/mean_length": 1571.375,
+      "completions/mean_terminated_length": 857.0,
+      "completions/min_length": 270.0,
+      "completions/min_terminated_length": 270.0,
+      "epoch": 2.2155963302752295,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.38027381896972656,
+      "kl": 0.0002809951434604348,
+      "learning_rate": 4.290247335263362e-08,
+      "loss": 0.0275,
+      "num_tokens": 15237241.0,
+      "reward": 0.1479167640209198,
+      "reward_std": 0.5390461683273315,
+      "rewards/compute_weighted_reward/mean": 0.14791667461395264,
+      "rewards/compute_weighted_reward/std": 1.8478398323059082,
+      "step": 483
+    },
+    {
+      "completion_length": 1442.0833333333333,
+      "completions/clipped_ratio": 0.5416666666666667,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 1958.0,
+      "completions/mean_length": 1442.0833740234375,
+      "completions/mean_terminated_length": 782.727294921875,
+      "completions/min_length": 454.0,
+      "completions/min_terminated_length": 454.0,
+      "epoch": 2.220183486238532,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.42852839827537537,
+      "kl": 0.0002766387536515443,
+      "learning_rate": 4.2861816990820083e-08,
+      "loss": 0.1764,
+      "num_tokens": 15276579.0,
+      "reward": -0.39499998092651367,
+      "reward_std": 0.7189545631408691,
+      "rewards/compute_weighted_reward/mean": -0.39499998092651367,
+      "rewards/compute_weighted_reward/std": 0.8256170153617859,
+      "step": 484
+    },
+    {
+      "completion_length": 1310.7916666666667,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 966.0,
+      "completions/mean_length": 1313.4583740234375,
+      "completions/mean_terminated_length": 626.9166870117188,
+      "completions/min_length": 345.0,
+      "completions/min_terminated_length": 345.0,
+      "epoch": 2.2247706422018347,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.3990933299064636,
+      "kl": 0.0002945650800635728,
+      "learning_rate": 4.282106389979501e-08,
+      "loss": 0.1089,
+      "num_tokens": 15316136.0,
+      "reward": -0.3725000321865082,
+      "reward_std": 0.6126512289047241,
+      "rewards/compute_weighted_reward/mean": -0.3724999725818634,
+      "rewards/compute_weighted_reward/std": 0.906643271446228,
+      "step": 485
+    },
+    {
+      "completion_length": 1326.75,
+      "completions/clipped_ratio": 0.41666666666666663,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 1965.0,
+      "completions/mean_length": 1331.875,
+      "completions/mean_terminated_length": 854.6428833007812,
+      "completions/min_length": 401.0,
+      "completions/min_terminated_length": 401.0,
+      "epoch": 2.229357798165138,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5519124269485474,
+      "kl": 0.0003618740717380812,
+      "learning_rate": 4.2780214300253425e-08,
+      "loss": 0.137,
+      "num_tokens": 15354791.0,
+      "reward": 0.06541667878627777,
+      "reward_std": 1.19478178024292,
+      "rewards/compute_weighted_reward/mean": 0.06541666388511658,
+      "rewards/compute_weighted_reward/std": 1.288430094718933,
+      "step": 486
+    },
+    {
+      "completion_length": 1770.0416666666667,
+      "completions/clipped_ratio": 0.7916666666666666,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 1416.0,
+      "completions/mean_length": 1770.041748046875,
+      "completions/mean_terminated_length": 896.2000122070312,
+      "completions/min_length": 429.0,
+      "completions/min_terminated_length": 429.0,
+      "epoch": 2.2339449541284404,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.5295096635818481,
+      "kl": 0.0003984466287268636,
+      "learning_rate": 4.273926841341302e-08,
+      "loss": 0.147,
+      "num_tokens": 15402444.0,
+      "reward": -0.47291669249534607,
+      "reward_std": 0.8791345357894897,
+      "rewards/compute_weighted_reward/mean": -0.47291669249534607,
+      "rewards/compute_weighted_reward/std": 1.13621985912323,
+      "step": 487
+    },
+    {
+      "completion_length": 1174.9166666666667,
+      "completions/clipped_ratio": 0.375,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 1251.0,
+      "completions/mean_length": 1174.916748046875,
+      "completions/mean_terminated_length": 679.86669921875,
+      "completions/min_length": 389.0,
+      "completions/min_terminated_length": 389.0,
+      "epoch": 2.238532110091743,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.31133127212524414,
+      "kl": 0.00023924819227734892,
+      "learning_rate": 4.269822646101289e-08,
+      "loss": 0.1609,
+      "num_tokens": 15437032.0,
+      "reward": 0.6670833826065063,
+      "reward_std": 0.39568936824798584,
+      "rewards/compute_weighted_reward/mean": 0.6670833230018616,
+      "rewards/compute_weighted_reward/std": 1.9870023727416992,
+      "step": 488
+    },
+    {
+      "completion_length": 1224.8333333333333,
+      "completions/clipped_ratio": 0.41666666666666663,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 1414.0,
+      "completions/mean_length": 1224.8333740234375,
+      "completions/mean_terminated_length": 671.1428833007812,
+      "completions/min_length": 205.0,
+      "completions/min_terminated_length": 205.0,
+      "epoch": 2.243119266055046,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.36838299036026,
+      "kl": 0.00021752280918008182,
+      "learning_rate": 4.265708866531238e-08,
+      "loss": 0.1666,
+      "num_tokens": 15471570.0,
+      "reward": 0.31291672587394714,
+      "reward_std": 0.687069296836853,
+      "rewards/compute_weighted_reward/mean": 0.31291666626930237,
+      "rewards/compute_weighted_reward/std": 1.5205562114715576,
+      "step": 489
+    },
+    {
+      "completion_length": 1431.0833333333333,
+      "completions/clipped_ratio": 0.5416666666666667,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 1937.0,
+      "completions/mean_length": 1431.0833740234375,
+      "completions/mean_terminated_length": 758.727294921875,
+      "completions/min_length": 281.0,
+      "completions/min_terminated_length": 281.0,
+      "epoch": 2.2477064220183487,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.47285446524620056,
+      "kl": 0.0003024169091077056,
+      "learning_rate": 4.2615855249089863e-08,
+      "loss": 0.0715,
+      "num_tokens": 15511040.0,
+      "reward": -0.021666675806045532,
+      "reward_std": 0.8609253764152527,
+      "rewards/compute_weighted_reward/mean": -0.021666666492819786,
+      "rewards/compute_weighted_reward/std": 1.4374394416809082,
+      "step": 490
     }
   ],
   "logging_steps": 1,
   "max_steps": 1500,
-  "num_input_tokens_seen": 15111658,
+  "num_input_tokens_seen": 15511040,
   "num_train_epochs": 7,
   "save_steps": 10,
   "stateful_callbacks": {