| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.3227513227513228, |
| "eval_steps": 500, |
| "global_step": 250, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "completion_length": 469.0, |
| "epoch": 0.005291005291005291, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 5.0000000000000004e-08, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 1 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 0.010582010582010581, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 1.0000000000000001e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 2 |
| }, |
| { |
| "completion_length": 460.0, |
| "epoch": 0.015873015873015872, |
| "grad_norm": 0.0, |
| "kl": 0.0, |
| "learning_rate": 1.5000000000000002e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 3 |
| }, |
| { |
| "completion_length": 443.0, |
| "epoch": 0.021164021164021163, |
| "grad_norm": 0.3466161787509918, |
| "kl": 0.0, |
| "learning_rate": 2.0000000000000002e-07, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103816509247, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 4 |
| }, |
| { |
| "completion_length": 456.0, |
| "epoch": 0.026455026455026454, |
| "grad_norm": 0.0013971382286399603, |
| "kl": 0.0007008439861238003, |
| "learning_rate": 2.5000000000000004e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 5 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.031746031746031744, |
| "grad_norm": 0.0018188758986070752, |
| "kl": 0.0008783861412666738, |
| "learning_rate": 3.0000000000000004e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 6 |
| }, |
| { |
| "completion_length": 466.0, |
| "epoch": 0.037037037037037035, |
| "grad_norm": 0.0013314716052263975, |
| "kl": 0.0007734335376881063, |
| "learning_rate": 3.5000000000000004e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 7 |
| }, |
| { |
| "completion_length": 465.0, |
| "epoch": 0.042328042328042326, |
| "grad_norm": 0.332011878490448, |
| "kl": 0.0007378095760941505, |
| "learning_rate": 4.0000000000000003e-07, |
| "loss": 0.0, |
| "reward": -0.3021666705608368, |
| "reward_std": 0.8029496669769287, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.3021666705608368, |
| "step": 8 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 0.047619047619047616, |
| "grad_norm": 0.001706001698039472, |
| "kl": 0.0008193795802071691, |
| "learning_rate": 4.5000000000000003e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 9 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.05291005291005291, |
| "grad_norm": 0.00121624197345227, |
| "kl": 0.0006714992923662066, |
| "learning_rate": 5.000000000000001e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 10 |
| }, |
| { |
| "completion_length": 343.3333435058594, |
| "epoch": 0.0582010582010582, |
| "grad_norm": 0.001374781015329063, |
| "kl": 0.0007051153806969523, |
| "learning_rate": 5.5e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 11 |
| }, |
| { |
| "completion_length": 471.0, |
| "epoch": 0.06349206349206349, |
| "grad_norm": 0.0015524772461503744, |
| "kl": 0.0008028542506508529, |
| "learning_rate": 6.000000000000001e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 12 |
| }, |
| { |
| "completion_length": 473.0, |
| "epoch": 0.06878306878306878, |
| "grad_norm": 0.4030776917934418, |
| "kl": 0.0007009574910625815, |
| "learning_rate": 6.5e-07, |
| "loss": 0.0, |
| "reward": 0.0416666679084301, |
| "reward_std": 0.06454972922801971, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0416666679084301, |
| "step": 13 |
| }, |
| { |
| "completion_length": 448.0, |
| "epoch": 0.07407407407407407, |
| "grad_norm": 0.0019437902374193072, |
| "kl": 0.0007818953017704189, |
| "learning_rate": 7.000000000000001e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 14 |
| }, |
| { |
| "completion_length": 458.0, |
| "epoch": 0.07936507936507936, |
| "grad_norm": 0.0011251139221712947, |
| "kl": 0.0006453194655478001, |
| "learning_rate": 7.5e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 15 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.08465608465608465, |
| "grad_norm": 0.001244411221705377, |
| "kl": 0.0007461973000317812, |
| "learning_rate": 8.000000000000001e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 16 |
| }, |
| { |
| "completion_length": 449.0, |
| "epoch": 0.08994708994708994, |
| "grad_norm": 0.37585949897766113, |
| "kl": 0.0008572853985242546, |
| "learning_rate": 8.500000000000001e-07, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103443980217, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 17 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.09523809523809523, |
| "grad_norm": 0.0010893117869272828, |
| "kl": 0.0006775903748348355, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 18 |
| }, |
| { |
| "completion_length": 467.0, |
| "epoch": 0.10052910052910052, |
| "grad_norm": 0.0011105411686003208, |
| "kl": 0.0007528206333518028, |
| "learning_rate": 9.500000000000001e-07, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 19 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 0.10582010582010581, |
| "grad_norm": 0.2887333929538727, |
| "kl": 0.0005192860262468457, |
| "learning_rate": 1.0000000000000002e-06, |
| "loss": 0.0, |
| "reward": -0.36116668581962585, |
| "reward_std": 0.884674072265625, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.36116668581962585, |
| "step": 20 |
| }, |
| { |
| "completion_length": 476.0, |
| "epoch": 0.1111111111111111, |
| "grad_norm": 0.0012039246503263712, |
| "kl": 0.0007195804500952363, |
| "learning_rate": 1.0500000000000001e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 21 |
| }, |
| { |
| "completion_length": 448.0, |
| "epoch": 0.1164021164021164, |
| "grad_norm": 0.001689268508926034, |
| "kl": 0.0008192076347768307, |
| "learning_rate": 1.1e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 22 |
| }, |
| { |
| "completion_length": 425.0, |
| "epoch": 0.12169312169312169, |
| "grad_norm": 0.30522194504737854, |
| "kl": 0.0007009351393207908, |
| "learning_rate": 1.1500000000000002e-06, |
| "loss": 0.0, |
| "reward": -0.4636666774749756, |
| "reward_std": 0.8273141384124756, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.4636666774749756, |
| "step": 23 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.12698412698412698, |
| "grad_norm": 0.0020346820820122957, |
| "kl": 0.0007367305806837976, |
| "learning_rate": 1.2000000000000002e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 24 |
| }, |
| { |
| "completion_length": 457.0, |
| "epoch": 0.13227513227513227, |
| "grad_norm": 0.0015893435338512063, |
| "kl": 0.0007120617083273828, |
| "learning_rate": 1.25e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 25 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 0.13756613756613756, |
| "grad_norm": 0.0015231599099934101, |
| "kl": 0.0007700793212279677, |
| "learning_rate": 1.3e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 26 |
| }, |
| { |
| "completion_length": 468.0, |
| "epoch": 0.14285714285714285, |
| "grad_norm": 0.001753391814418137, |
| "kl": 0.000772205414250493, |
| "learning_rate": 1.3500000000000002e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 27 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 0.14814814814814814, |
| "grad_norm": 0.0015386062441393733, |
| "kl": 0.0008366336114704609, |
| "learning_rate": 1.4000000000000001e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 28 |
| }, |
| { |
| "completion_length": 434.0, |
| "epoch": 0.15343915343915343, |
| "grad_norm": 0.0011081281118094921, |
| "kl": 0.0006436242256313562, |
| "learning_rate": 1.45e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 29 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 0.15873015873015872, |
| "grad_norm": 0.0014530145563185215, |
| "kl": 0.0007206773152574897, |
| "learning_rate": 1.5e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 30 |
| }, |
| { |
| "completion_length": 447.0, |
| "epoch": 0.164021164021164, |
| "grad_norm": 0.0011599598219618201, |
| "kl": 0.0007213893113657832, |
| "learning_rate": 1.5500000000000002e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 31 |
| }, |
| { |
| "completion_length": 417.8333435058594, |
| "epoch": 0.1693121693121693, |
| "grad_norm": 0.0011569266207516193, |
| "kl": 0.0006827408215031028, |
| "learning_rate": 1.6000000000000001e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 32 |
| }, |
| { |
| "completion_length": 453.0, |
| "epoch": 0.1746031746031746, |
| "grad_norm": 0.001164209214039147, |
| "kl": 0.000635787844657898, |
| "learning_rate": 1.6500000000000003e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 33 |
| }, |
| { |
| "completion_length": 426.3333435058594, |
| "epoch": 0.17989417989417988, |
| "grad_norm": 0.38202932476997375, |
| "kl": 0.0005964495940133929, |
| "learning_rate": 1.7000000000000002e-06, |
| "loss": 0.0, |
| "reward": -0.006666666828095913, |
| "reward_std": 0.01632993295788765, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.006666666828095913, |
| "step": 34 |
| }, |
| { |
| "completion_length": 451.0, |
| "epoch": 0.18518518518518517, |
| "grad_norm": 0.0010626944713294506, |
| "kl": 0.0006257618078961968, |
| "learning_rate": 1.75e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 35 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 0.19047619047619047, |
| "grad_norm": 0.0011656994465738535, |
| "kl": 0.0007656628149561584, |
| "learning_rate": 1.8000000000000001e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 36 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 0.19576719576719576, |
| "grad_norm": 0.37526777386665344, |
| "kl": 0.000684558879584074, |
| "learning_rate": 1.85e-06, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103816509247, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 37 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 0.20105820105820105, |
| "grad_norm": 0.0012193089351058006, |
| "kl": 0.0007294993847608566, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 38 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.20634920634920634, |
| "grad_norm": 0.00116525508929044, |
| "kl": 0.0007017645402811468, |
| "learning_rate": 1.9500000000000004e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 39 |
| }, |
| { |
| "completion_length": 456.0, |
| "epoch": 0.21164021164021163, |
| "grad_norm": 0.0014412697637453675, |
| "kl": 0.0008313995786011219, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 40 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 0.21693121693121692, |
| "grad_norm": 0.3337116837501526, |
| "kl": 0.0008217081194743514, |
| "learning_rate": 2.05e-06, |
| "loss": 0.0, |
| "reward": -0.3778333365917206, |
| "reward_std": 0.9254988431930542, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.3778333365917206, |
| "step": 41 |
| }, |
| { |
| "completion_length": 453.0, |
| "epoch": 0.2222222222222222, |
| "grad_norm": 0.3406112790107727, |
| "kl": 0.0007513266755267978, |
| "learning_rate": 2.1000000000000002e-06, |
| "loss": 0.0, |
| "reward": -0.6726667284965515, |
| "reward_std": 1.0454801321029663, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.6726667284965515, |
| "step": 42 |
| }, |
| { |
| "completion_length": 446.0, |
| "epoch": 0.2275132275132275, |
| "grad_norm": 0.31627708673477173, |
| "kl": 0.000685308943502605, |
| "learning_rate": 2.15e-06, |
| "loss": 0.0, |
| "reward": -0.27950000762939453, |
| "reward_std": 0.7475436925888062, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.27950000762939453, |
| "step": 43 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 0.2328042328042328, |
| "grad_norm": 0.0014166582841426134, |
| "kl": 0.0007032315479591489, |
| "learning_rate": 2.2e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 44 |
| }, |
| { |
| "completion_length": 456.0, |
| "epoch": 0.23809523809523808, |
| "grad_norm": 0.3792738616466522, |
| "kl": 0.0007219819235615432, |
| "learning_rate": 2.25e-06, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103443980217, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 45 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.24338624338624337, |
| "grad_norm": 0.0012094344710931182, |
| "kl": 0.0006781464326195419, |
| "learning_rate": 2.3000000000000004e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 46 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 0.24867724867724866, |
| "grad_norm": 0.0013666359009221196, |
| "kl": 0.0009113170090131462, |
| "learning_rate": 2.35e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 47 |
| }, |
| { |
| "completion_length": 467.0, |
| "epoch": 0.25396825396825395, |
| "grad_norm": 0.0038678154814988375, |
| "kl": 0.0008524904260411859, |
| "learning_rate": 2.4000000000000003e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 48 |
| }, |
| { |
| "completion_length": 467.0, |
| "epoch": 0.25925925925925924, |
| "grad_norm": 0.0016692781355232, |
| "kl": 0.0008949865004979074, |
| "learning_rate": 2.4500000000000003e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 49 |
| }, |
| { |
| "completion_length": 458.0, |
| "epoch": 0.26455026455026454, |
| "grad_norm": 0.0015817458042874932, |
| "kl": 0.0008414339972659945, |
| "learning_rate": 2.5e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 50 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 0.2698412698412698, |
| "grad_norm": 0.0011008064029738307, |
| "kl": 0.0006569117540493608, |
| "learning_rate": 2.55e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 51 |
| }, |
| { |
| "completion_length": 465.0, |
| "epoch": 0.2751322751322751, |
| "grad_norm": 0.001251985551789403, |
| "kl": 0.0007252182113006711, |
| "learning_rate": 2.6e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 52 |
| }, |
| { |
| "completion_length": 469.0, |
| "epoch": 0.2804232804232804, |
| "grad_norm": 0.001667379285208881, |
| "kl": 0.0008470327593386173, |
| "learning_rate": 2.6500000000000005e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 53 |
| }, |
| { |
| "completion_length": 460.0, |
| "epoch": 0.2857142857142857, |
| "grad_norm": 0.0010524148819968104, |
| "kl": 0.0006404528394341469, |
| "learning_rate": 2.7000000000000004e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 54 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 0.291005291005291, |
| "grad_norm": 0.0013384610647335649, |
| "kl": 0.0007196539081633091, |
| "learning_rate": 2.7500000000000004e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 55 |
| }, |
| { |
| "completion_length": 443.0, |
| "epoch": 0.2962962962962963, |
| "grad_norm": 0.0025668947491794825, |
| "kl": 0.000717508140951395, |
| "learning_rate": 2.8000000000000003e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 56 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.30158730158730157, |
| "grad_norm": 0.0013547488488256931, |
| "kl": 0.000740337185561657, |
| "learning_rate": 2.85e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 57 |
| }, |
| { |
| "completion_length": 435.0, |
| "epoch": 0.30687830687830686, |
| "grad_norm": 0.0015899285208433867, |
| "kl": 0.0007815321441739798, |
| "learning_rate": 2.9e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 58 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 0.31216931216931215, |
| "grad_norm": 0.3050638735294342, |
| "kl": 0.0006648754933848977, |
| "learning_rate": 2.95e-06, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103443980217, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 59 |
| }, |
| { |
| "completion_length": 460.0, |
| "epoch": 0.31746031746031744, |
| "grad_norm": 0.001313161919824779, |
| "kl": 0.0006786473095417023, |
| "learning_rate": 3e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 60 |
| }, |
| { |
| "completion_length": 466.0, |
| "epoch": 0.32275132275132273, |
| "grad_norm": 0.0018586755031719804, |
| "kl": 0.0009205904207192361, |
| "learning_rate": 3.05e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 61 |
| }, |
| { |
| "completion_length": 444.16668701171875, |
| "epoch": 0.328042328042328, |
| "grad_norm": 0.0012059886939823627, |
| "kl": 0.0006423206068575382, |
| "learning_rate": 3.1000000000000004e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 62 |
| }, |
| { |
| "completion_length": 458.3333435058594, |
| "epoch": 0.3333333333333333, |
| "grad_norm": 0.0010612922487780452, |
| "kl": 0.0006938511505723, |
| "learning_rate": 3.1500000000000003e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 63 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 0.3386243386243386, |
| "grad_norm": 0.0013961538206785917, |
| "kl": 0.0007388316444121301, |
| "learning_rate": 3.2000000000000003e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 64 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 0.3439153439153439, |
| "grad_norm": 0.0014836119953542948, |
| "kl": 0.0007349005318246782, |
| "learning_rate": 3.2500000000000002e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 65 |
| }, |
| { |
| "completion_length": 471.0, |
| "epoch": 0.3492063492063492, |
| "grad_norm": 0.0011990249622613192, |
| "kl": 0.0007487541297450662, |
| "learning_rate": 3.3000000000000006e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 66 |
| }, |
| { |
| "completion_length": 464.16668701171875, |
| "epoch": 0.3544973544973545, |
| "grad_norm": 0.3155646026134491, |
| "kl": 0.000580943189561367, |
| "learning_rate": 3.3500000000000005e-06, |
| "loss": 0.0, |
| "reward": -0.624666690826416, |
| "reward_std": 0.9964100122451782, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.624666690826416, |
| "step": 67 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.35978835978835977, |
| "grad_norm": 0.0013993227621540427, |
| "kl": 0.0007153385085985065, |
| "learning_rate": 3.4000000000000005e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 68 |
| }, |
| { |
| "completion_length": 449.16668701171875, |
| "epoch": 0.36507936507936506, |
| "grad_norm": 0.3121412694454193, |
| "kl": 0.0007234203512780368, |
| "learning_rate": 3.45e-06, |
| "loss": 0.0, |
| "reward": -0.22433333098888397, |
| "reward_std": 0.5495022535324097, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.22433333098888397, |
| "step": 69 |
| }, |
| { |
| "completion_length": 471.0, |
| "epoch": 0.37037037037037035, |
| "grad_norm": 0.001299115945585072, |
| "kl": 0.0007632293272763491, |
| "learning_rate": 3.5e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 70 |
| }, |
| { |
| "completion_length": 399.0, |
| "epoch": 0.37566137566137564, |
| "grad_norm": 0.3708711564540863, |
| "kl": 0.0007443460053764284, |
| "learning_rate": 3.5500000000000003e-06, |
| "loss": 0.0, |
| "reward": -0.33800002932548523, |
| "reward_std": 0.8279275298118591, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.33800002932548523, |
| "step": 71 |
| }, |
| { |
| "completion_length": 453.0, |
| "epoch": 0.38095238095238093, |
| "grad_norm": 0.001166418194770813, |
| "kl": 0.000631289032753557, |
| "learning_rate": 3.6000000000000003e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 72 |
| }, |
| { |
| "completion_length": 463.0, |
| "epoch": 0.3862433862433862, |
| "grad_norm": 0.0011989667546004057, |
| "kl": 0.0007275048992596567, |
| "learning_rate": 3.65e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 73 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 0.3915343915343915, |
| "grad_norm": 0.0014157023979350924, |
| "kl": 0.0008615436381660402, |
| "learning_rate": 3.7e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 74 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.3968253968253968, |
| "grad_norm": 0.0012928849318996072, |
| "kl": 0.000718214432708919, |
| "learning_rate": 3.7500000000000005e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 75 |
| }, |
| { |
| "completion_length": 467.0, |
| "epoch": 0.4021164021164021, |
| "grad_norm": 0.4323344826698303, |
| "kl": 0.000800282577984035, |
| "learning_rate": 3.8000000000000005e-06, |
| "loss": 0.0, |
| "reward": -0.3345000147819519, |
| "reward_std": 0.8193542957305908, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.3345000147819519, |
| "step": 76 |
| }, |
| { |
| "completion_length": 453.16668701171875, |
| "epoch": 0.4074074074074074, |
| "grad_norm": 0.0015292431926354766, |
| "kl": 0.0008153495145961642, |
| "learning_rate": 3.85e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 77 |
| }, |
| { |
| "completion_length": 466.0, |
| "epoch": 0.4126984126984127, |
| "grad_norm": 0.0012229210697114468, |
| "kl": 0.0006657217163592577, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 78 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.41798941798941797, |
| "grad_norm": 0.0013483399525284767, |
| "kl": 0.0007513434975408018, |
| "learning_rate": 3.95e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 79 |
| }, |
| { |
| "completion_length": 465.0, |
| "epoch": 0.42328042328042326, |
| "grad_norm": 0.0011019837111234665, |
| "kl": 0.0006735255010426044, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 80 |
| }, |
| { |
| "completion_length": 458.0, |
| "epoch": 0.42857142857142855, |
| "grad_norm": 0.0013857161393389106, |
| "kl": 0.0007106320699676871, |
| "learning_rate": 4.05e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 81 |
| }, |
| { |
| "completion_length": 466.0, |
| "epoch": 0.43386243386243384, |
| "grad_norm": 0.0013176508946344256, |
| "kl": 0.0008324295049533248, |
| "learning_rate": 4.1e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 82 |
| }, |
| { |
| "completion_length": 426.66668701171875, |
| "epoch": 0.43915343915343913, |
| "grad_norm": 0.0014097305247560143, |
| "kl": 0.0006901246379129589, |
| "learning_rate": 4.15e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 83 |
| }, |
| { |
| "completion_length": 451.3333435058594, |
| "epoch": 0.4444444444444444, |
| "grad_norm": 0.001386586925946176, |
| "kl": 0.0007090188446454704, |
| "learning_rate": 4.2000000000000004e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 84 |
| }, |
| { |
| "completion_length": 471.0, |
| "epoch": 0.4497354497354497, |
| "grad_norm": 0.0011919698445126414, |
| "kl": 0.0007535028271377087, |
| "learning_rate": 4.25e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 85 |
| }, |
| { |
| "completion_length": 455.0, |
| "epoch": 0.455026455026455, |
| "grad_norm": 0.0015907199122011662, |
| "kl": 0.0008195896516554058, |
| "learning_rate": 4.3e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 86 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 0.4603174603174603, |
| "grad_norm": 0.001058085821568966, |
| "kl": 0.0006564512150362134, |
| "learning_rate": 4.350000000000001e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 87 |
| }, |
| { |
| "completion_length": 456.0, |
| "epoch": 0.4656084656084656, |
| "grad_norm": 0.0016100555658340454, |
| "kl": 0.0008427806897088885, |
| "learning_rate": 4.4e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 88 |
| }, |
| { |
| "completion_length": 442.16668701171875, |
| "epoch": 0.4708994708994709, |
| "grad_norm": 0.0013671847991645336, |
| "kl": 0.0007155524799600244, |
| "learning_rate": 4.450000000000001e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 89 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.47619047619047616, |
| "grad_norm": 0.001224961131811142, |
| "kl": 0.0007802361506037414, |
| "learning_rate": 4.5e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 90 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.48148148148148145, |
| "grad_norm": 0.0012146050576120615, |
| "kl": 0.0006493218243122101, |
| "learning_rate": 4.5500000000000005e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 91 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.48677248677248675, |
| "grad_norm": 0.0017682723701000214, |
| "kl": 0.0009243417298421264, |
| "learning_rate": 4.600000000000001e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 92 |
| }, |
| { |
| "completion_length": 473.0, |
| "epoch": 0.49206349206349204, |
| "grad_norm": 0.0011983285658061504, |
| "kl": 0.0006397532997652888, |
| "learning_rate": 4.65e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 93 |
| }, |
| { |
| "completion_length": 453.0, |
| "epoch": 0.4973544973544973, |
| "grad_norm": 0.00133048789575696, |
| "kl": 0.0007033076835796237, |
| "learning_rate": 4.7e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 94 |
| }, |
| { |
| "completion_length": 437.0, |
| "epoch": 0.5026455026455027, |
| "grad_norm": 0.0018940640147775412, |
| "kl": 0.000845337170176208, |
| "learning_rate": 4.75e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 95 |
| }, |
| { |
| "completion_length": 456.0, |
| "epoch": 0.5079365079365079, |
| "grad_norm": 0.0012317766668274999, |
| "kl": 0.0007042339420877397, |
| "learning_rate": 4.800000000000001e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 96 |
| }, |
| { |
| "completion_length": 468.0, |
| "epoch": 0.5132275132275133, |
| "grad_norm": 0.0014763603685423732, |
| "kl": 0.0007702677394263446, |
| "learning_rate": 4.85e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 97 |
| }, |
| { |
| "completion_length": 379.0, |
| "epoch": 0.5185185185185185, |
| "grad_norm": 0.3322179615497589, |
| "kl": 0.0005753381410613656, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0, |
| "reward": -0.21566668152809143, |
| "reward_std": 0.4929104447364807, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.21566668152809143, |
| "step": 98 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 0.5238095238095238, |
| "grad_norm": 0.0011984164593741298, |
| "kl": 0.0007127061253413558, |
| "learning_rate": 4.95e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 99 |
| }, |
| { |
| "completion_length": 437.5, |
| "epoch": 0.5291005291005291, |
| "grad_norm": 0.001482065999880433, |
| "kl": 0.0007626632577739656, |
| "learning_rate": 5e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 100 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 0.5343915343915344, |
| "grad_norm": 0.0012092319084331393, |
| "kl": 0.0007269163033924997, |
| "learning_rate": 4.999984769144476e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 101 |
| }, |
| { |
| "completion_length": 456.0, |
| "epoch": 0.5396825396825397, |
| "grad_norm": 0.0011810030555352569, |
| "kl": 0.0007304114988073707, |
| "learning_rate": 4.999939076763487e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 102 |
| }, |
| { |
| "completion_length": 419.3333435058594, |
| "epoch": 0.544973544973545, |
| "grad_norm": 0.0013815524289384484, |
| "kl": 0.0007696993416175246, |
| "learning_rate": 4.999862923413781e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 103 |
| }, |
| { |
| "completion_length": 464.16668701171875, |
| "epoch": 0.5502645502645502, |
| "grad_norm": 0.0011437179055064917, |
| "kl": 0.0006861802539788187, |
| "learning_rate": 4.999756310023261e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 104 |
| }, |
| { |
| "completion_length": 457.0, |
| "epoch": 0.5555555555555556, |
| "grad_norm": 0.001996035221964121, |
| "kl": 0.0008951328927651048, |
| "learning_rate": 4.9996192378909785e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 105 |
| }, |
| { |
| "completion_length": 451.16668701171875, |
| "epoch": 0.5608465608465608, |
| "grad_norm": 0.3518621027469635, |
| "kl": 0.0007362122414633632, |
| "learning_rate": 4.999451708687114e-06, |
| "loss": 0.0, |
| "reward": -0.5885000228881836, |
| "reward_std": 0.9123517274856567, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.5885000228881836, |
| "step": 106 |
| }, |
| { |
| "completion_length": 452.3333435058594, |
| "epoch": 0.5661375661375662, |
| "grad_norm": 0.3178879916667938, |
| "kl": 0.0005817896453663707, |
| "learning_rate": 4.9992537244529585e-06, |
| "loss": 0.0, |
| "reward": -0.23800000548362732, |
| "reward_std": 0.5829786062240601, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.23800000548362732, |
| "step": 107 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.5714285714285714, |
| "grad_norm": 0.0015168412355706096, |
| "kl": 0.000762041425332427, |
| "learning_rate": 4.999025287600886e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 108 |
| }, |
| { |
| "completion_length": 452.0, |
| "epoch": 0.5767195767195767, |
| "grad_norm": 0.0021211248822510242, |
| "kl": 0.0008137742406688631, |
| "learning_rate": 4.998766400914329e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 109 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.582010582010582, |
| "grad_norm": 0.0015144016360864043, |
| "kl": 0.000795575964730233, |
| "learning_rate": 4.99847706754774e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 110 |
| }, |
| { |
| "completion_length": 418.0, |
| "epoch": 0.5873015873015873, |
| "grad_norm": 0.001428043469786644, |
| "kl": 0.0007960469229146838, |
| "learning_rate": 4.998157291026553e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 111 |
| }, |
| { |
| "completion_length": 428.0, |
| "epoch": 0.5925925925925926, |
| "grad_norm": 0.30288076400756836, |
| "kl": 0.0006850928766652942, |
| "learning_rate": 4.997807075247147e-06, |
| "loss": 0.0, |
| "reward": -0.2956666946411133, |
| "reward_std": 0.72423255443573, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.2956666946411133, |
| "step": 112 |
| }, |
| { |
| "completion_length": 452.3333435058594, |
| "epoch": 0.5978835978835979, |
| "grad_norm": 0.0015472040977329016, |
| "kl": 0.0007640746189281344, |
| "learning_rate": 4.997426424476787e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 113 |
| }, |
| { |
| "completion_length": 389.0, |
| "epoch": 0.6031746031746031, |
| "grad_norm": 0.32950592041015625, |
| "kl": 0.000562019064091146, |
| "learning_rate": 4.9970153433535855e-06, |
| "loss": 0.0, |
| "reward": -0.4096667170524597, |
| "reward_std": 0.7116295695304871, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.4096667170524597, |
| "step": 114 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 0.6084656084656085, |
| "grad_norm": 0.001373196137137711, |
| "kl": 0.0008566213655285537, |
| "learning_rate": 4.9965738368864345e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 115 |
| }, |
| { |
| "completion_length": 460.0, |
| "epoch": 0.6137566137566137, |
| "grad_norm": 0.00112072192132473, |
| "kl": 0.0006975589203648269, |
| "learning_rate": 4.996101910454953e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 116 |
| }, |
| { |
| "completion_length": 458.0, |
| "epoch": 0.6190476190476191, |
| "grad_norm": 0.30892816185951233, |
| "kl": 0.0006624362431466579, |
| "learning_rate": 4.995599569809414e-06, |
| "loss": 0.0, |
| "reward": 0.0416666679084301, |
| "reward_std": 0.10206206887960434, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0416666679084301, |
| "step": 117 |
| }, |
| { |
| "completion_length": 469.0, |
| "epoch": 0.6243386243386243, |
| "grad_norm": 0.0015334797790274024, |
| "kl": 0.0007985906559042633, |
| "learning_rate": 4.9950668210706795e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 118 |
| }, |
| { |
| "completion_length": 465.0, |
| "epoch": 0.6296296296296297, |
| "grad_norm": 0.00187064113561064, |
| "kl": 0.0009239478386007249, |
| "learning_rate": 4.994503670730126e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 119 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 0.6349206349206349, |
| "grad_norm": 0.0031258563976734877, |
| "kl": 0.0009402809082530439, |
| "learning_rate": 4.993910125649561e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 120 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 0.6402116402116402, |
| "grad_norm": 0.34855061769485474, |
| "kl": 0.0008161421865224838, |
| "learning_rate": 4.993286193061145e-06, |
| "loss": 0.0, |
| "reward": 0.0416666679084301, |
| "reward_std": 0.06454972922801971, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0416666679084301, |
| "step": 121 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.6455026455026455, |
| "grad_norm": 0.0015622343635186553, |
| "kl": 0.0008013999904505908, |
| "learning_rate": 4.992631880567301e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 122 |
| }, |
| { |
| "completion_length": 438.5, |
| "epoch": 0.6507936507936508, |
| "grad_norm": 0.40107929706573486, |
| "kl": 0.0007979162037372589, |
| "learning_rate": 4.991947196140619e-06, |
| "loss": 0.0, |
| "reward": -0.18533334136009216, |
| "reward_std": 0.45397210121154785, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.18533334136009216, |
| "step": 123 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.656084656084656, |
| "grad_norm": 0.0018841363489627838, |
| "kl": 0.0009175824234262109, |
| "learning_rate": 4.9912321481237616e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 124 |
| }, |
| { |
| "completion_length": 463.0, |
| "epoch": 0.6613756613756614, |
| "grad_norm": 0.0017101428238674998, |
| "kl": 0.0007935116300359368, |
| "learning_rate": 4.990486745229364e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 125 |
| }, |
| { |
| "completion_length": 424.66668701171875, |
| "epoch": 0.6666666666666666, |
| "grad_norm": 0.37114962935447693, |
| "kl": 0.0006903379107825458, |
| "learning_rate": 4.989710996539926e-06, |
| "loss": 0.0, |
| "reward": -0.3701666593551636, |
| "reward_std": 0.9067193865776062, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.3701666593551636, |
| "step": 126 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 0.671957671957672, |
| "grad_norm": 0.0015492148231714964, |
| "kl": 0.0007499443599954247, |
| "learning_rate": 4.9889049115077e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 127 |
| }, |
| { |
| "completion_length": 465.0, |
| "epoch": 0.6772486772486772, |
| "grad_norm": 0.001491202274337411, |
| "kl": 0.0008261007023975253, |
| "learning_rate": 4.988068499954578e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 128 |
| }, |
| { |
| "completion_length": 450.16668701171875, |
| "epoch": 0.6825396825396826, |
| "grad_norm": 0.34939536452293396, |
| "kl": 0.0007104433025233448, |
| "learning_rate": 4.987201772071971e-06, |
| "loss": 0.0, |
| "reward": -0.23516666889190674, |
| "reward_std": 0.6392340660095215, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.23516666889190674, |
| "step": 129 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 0.6878306878306878, |
| "grad_norm": 0.0014395629987120628, |
| "kl": 0.0008329777047038078, |
| "learning_rate": 4.986304738420684e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 130 |
| }, |
| { |
| "completion_length": 465.0, |
| "epoch": 0.6931216931216931, |
| "grad_norm": 0.0013965043472126126, |
| "kl": 0.0007900685886852443, |
| "learning_rate": 4.985377409930789e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 131 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 0.6984126984126984, |
| "grad_norm": 0.3567965626716614, |
| "kl": 0.000759742921218276, |
| "learning_rate": 4.984419797901491e-06, |
| "loss": 0.0, |
| "reward": -0.3136666715145111, |
| "reward_std": 0.8310660123825073, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.3136666715145111, |
| "step": 132 |
| }, |
| { |
| "completion_length": 457.0, |
| "epoch": 0.7037037037037037, |
| "grad_norm": 0.001598275383003056, |
| "kl": 0.0008906321600079536, |
| "learning_rate": 4.983431914000991e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 133 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 0.708994708994709, |
| "grad_norm": 0.0014098617248237133, |
| "kl": 0.0008211454842239618, |
| "learning_rate": 4.9824137702663424e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 134 |
| }, |
| { |
| "completion_length": 467.0, |
| "epoch": 0.7142857142857143, |
| "grad_norm": 0.0015612418064847589, |
| "kl": 0.0008501239935867488, |
| "learning_rate": 4.981365379103306e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 135 |
| }, |
| { |
| "completion_length": 460.0, |
| "epoch": 0.7195767195767195, |
| "grad_norm": 0.0012797295348718762, |
| "kl": 0.0006794001674279571, |
| "learning_rate": 4.980286753286196e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 136 |
| }, |
| { |
| "completion_length": 467.0, |
| "epoch": 0.7248677248677249, |
| "grad_norm": 0.0011214141268283129, |
| "kl": 0.0006867767660878599, |
| "learning_rate": 4.979177905957726e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 137 |
| }, |
| { |
| "completion_length": 469.0, |
| "epoch": 0.7301587301587301, |
| "grad_norm": 0.0012405989691615105, |
| "kl": 0.0007601610850542784, |
| "learning_rate": 4.978038850628855e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 138 |
| }, |
| { |
| "completion_length": 409.0, |
| "epoch": 0.7354497354497355, |
| "grad_norm": 0.32333922386169434, |
| "kl": 0.0007358253351412714, |
| "learning_rate": 4.9768696011786095e-06, |
| "loss": 0.0, |
| "reward": -0.3255000114440918, |
| "reward_std": 0.7973089218139648, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.3255000114440918, |
| "step": 139 |
| }, |
| { |
| "completion_length": 457.0, |
| "epoch": 0.7407407407407407, |
| "grad_norm": 0.0012477230047807097, |
| "kl": 0.0007025137892924249, |
| "learning_rate": 4.975670171853926e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 140 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 0.746031746031746, |
| "grad_norm": 0.2948945164680481, |
| "kl": 0.0005189929506741464, |
| "learning_rate": 4.974440577269473e-06, |
| "loss": 0.0, |
| "reward": -0.30883336067199707, |
| "reward_std": 0.7564841508865356, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.30883336067199707, |
| "step": 141 |
| }, |
| { |
| "completion_length": 450.0, |
| "epoch": 0.7513227513227513, |
| "grad_norm": 0.0016048979014158249, |
| "kl": 0.0006551402038894594, |
| "learning_rate": 4.973180832407471e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 142 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.7566137566137566, |
| "grad_norm": 0.0013635422801598907, |
| "kl": 0.0007275959360413253, |
| "learning_rate": 4.971890952617515e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 143 |
| }, |
| { |
| "completion_length": 453.0, |
| "epoch": 0.7619047619047619, |
| "grad_norm": 0.00150936224963516, |
| "kl": 0.0008993734372779727, |
| "learning_rate": 4.970570953616383e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 144 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.7671957671957672, |
| "grad_norm": 0.0016224040882661939, |
| "kl": 0.0007900248747318983, |
| "learning_rate": 4.9692208514878445e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 145 |
| }, |
| { |
| "completion_length": 443.0, |
| "epoch": 0.7724867724867724, |
| "grad_norm": 0.4084126353263855, |
| "kl": 0.0008380330400541425, |
| "learning_rate": 4.96784066268247e-06, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103443980217, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 146 |
| }, |
| { |
| "completion_length": 471.0, |
| "epoch": 0.7777777777777778, |
| "grad_norm": 0.0013737526023760438, |
| "kl": 0.0008382964297197759, |
| "learning_rate": 4.966430404017424e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 147 |
| }, |
| { |
| "completion_length": 463.66668701171875, |
| "epoch": 0.783068783068783, |
| "grad_norm": 0.332684189081192, |
| "kl": 0.0007040002383291721, |
| "learning_rate": 4.964990092676263e-06, |
| "loss": 0.0, |
| "reward": -0.640166699886322, |
| "reward_std": 0.99271559715271, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.640166699886322, |
| "step": 148 |
| }, |
| { |
| "completion_length": 450.66668701171875, |
| "epoch": 0.7883597883597884, |
| "grad_norm": 0.33231139183044434, |
| "kl": 0.000760066497605294, |
| "learning_rate": 4.963519746208726e-06, |
| "loss": 0.0, |
| "reward": -0.2459999918937683, |
| "reward_std": 0.6025744676589966, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.2459999918937683, |
| "step": 149 |
| }, |
| { |
| "completion_length": 467.0, |
| "epoch": 0.7936507936507936, |
| "grad_norm": 0.001477265846915543, |
| "kl": 0.0008438455406576395, |
| "learning_rate": 4.962019382530521e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 150 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 0.798941798941799, |
| "grad_norm": 0.0013378444127738476, |
| "kl": 0.000796476611867547, |
| "learning_rate": 4.960489019923105e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 151 |
| }, |
| { |
| "completion_length": 466.0, |
| "epoch": 0.8042328042328042, |
| "grad_norm": 0.0014132327632978559, |
| "kl": 0.0007687346078455448, |
| "learning_rate": 4.958928677033465e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 152 |
| }, |
| { |
| "completion_length": 469.0, |
| "epoch": 0.8095238095238095, |
| "grad_norm": 0.001528494176454842, |
| "kl": 0.0008697471348568797, |
| "learning_rate": 4.957338372873886e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 153 |
| }, |
| { |
| "completion_length": 468.0, |
| "epoch": 0.8148148148148148, |
| "grad_norm": 0.0012945622438564897, |
| "kl": 0.0008287787204608321, |
| "learning_rate": 4.9557181268217225e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 154 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 0.8201058201058201, |
| "grad_norm": 0.0014368314296007156, |
| "kl": 0.0007519798236899078, |
| "learning_rate": 4.9540679586191605e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 155 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.8253968253968254, |
| "grad_norm": 0.0013108905404806137, |
| "kl": 0.000834781676530838, |
| "learning_rate": 4.9523878883729794e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 156 |
| }, |
| { |
| "completion_length": 472.0, |
| "epoch": 0.8306878306878307, |
| "grad_norm": 0.0017808660631999373, |
| "kl": 0.0008636314305476844, |
| "learning_rate": 4.9506779365543054e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 157 |
| }, |
| { |
| "completion_length": 465.0, |
| "epoch": 0.8359788359788359, |
| "grad_norm": 0.0014730064431205392, |
| "kl": 0.0008062751730903983, |
| "learning_rate": 4.94893812399836e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 158 |
| }, |
| { |
| "completion_length": 437.3333435058594, |
| "epoch": 0.8412698412698413, |
| "grad_norm": 0.35100266337394714, |
| "kl": 0.0008240961469709873, |
| "learning_rate": 4.947168471904213e-06, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103443980217, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 159 |
| }, |
| { |
| "completion_length": 457.0, |
| "epoch": 0.8465608465608465, |
| "grad_norm": 0.38602274656295776, |
| "kl": 0.0007352498359978199, |
| "learning_rate": 4.9453690018345144e-06, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103443980217, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 160 |
| }, |
| { |
| "completion_length": 405.0, |
| "epoch": 0.8518518518518519, |
| "grad_norm": 0.001434896606951952, |
| "kl": 0.0007721909787505865, |
| "learning_rate": 4.9435397357152406e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 161 |
| }, |
| { |
| "completion_length": 430.16668701171875, |
| "epoch": 0.8571428571428571, |
| "grad_norm": 0.3277605473995209, |
| "kl": 0.0008263689232990146, |
| "learning_rate": 4.9416806958354206e-06, |
| "loss": 0.0, |
| "reward": 0.0416666679084301, |
| "reward_std": 0.10206207633018494, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0416666679084301, |
| "step": 162 |
| }, |
| { |
| "completion_length": 463.0, |
| "epoch": 0.8624338624338624, |
| "grad_norm": 0.00203329767100513, |
| "kl": 0.0007360164308920503, |
| "learning_rate": 4.939791904846869e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 163 |
| }, |
| { |
| "completion_length": 450.8333435058594, |
| "epoch": 0.8677248677248677, |
| "grad_norm": 0.29477033019065857, |
| "kl": 0.000802068505436182, |
| "learning_rate": 4.937873385763909e-06, |
| "loss": 0.0, |
| "reward": -0.21383333206176758, |
| "reward_std": 0.5237826108932495, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.21383333206176758, |
| "step": 164 |
| }, |
| { |
| "completion_length": 449.0, |
| "epoch": 0.873015873015873, |
| "grad_norm": 0.34382081031799316, |
| "kl": 0.0008281710324808955, |
| "learning_rate": 4.935925161963089e-06, |
| "loss": 0.0, |
| "reward": -0.23883333802223206, |
| "reward_std": 0.6481883525848389, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.23883333802223206, |
| "step": 165 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 0.8783068783068783, |
| "grad_norm": 0.0018430388299748302, |
| "kl": 0.0008553884690627456, |
| "learning_rate": 4.933947257182901e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 166 |
| }, |
| { |
| "completion_length": 460.0, |
| "epoch": 0.8835978835978836, |
| "grad_norm": 0.001679479144513607, |
| "kl": 0.0008453860646113753, |
| "learning_rate": 4.9319396955234925e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 167 |
| }, |
| { |
| "completion_length": 468.0, |
| "epoch": 0.8888888888888888, |
| "grad_norm": 0.0012063832255080342, |
| "kl": 0.0007442209171131253, |
| "learning_rate": 4.9299025014463665e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 168 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 0.8941798941798942, |
| "grad_norm": 0.001603979035280645, |
| "kl": 0.0008512127096764743, |
| "learning_rate": 4.92783569977409e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 169 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 0.8994708994708994, |
| "grad_norm": 0.001622421550564468, |
| "kl": 0.0007938953931443393, |
| "learning_rate": 4.925739315689991e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 170 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 0.9047619047619048, |
| "grad_norm": 0.3555748462677002, |
| "kl": 0.0008335469174198806, |
| "learning_rate": 4.923613374737848e-06, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103443980217, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 171 |
| }, |
| { |
| "completion_length": 447.0, |
| "epoch": 0.91005291005291, |
| "grad_norm": 0.41411712765693665, |
| "kl": 0.0008553611114621162, |
| "learning_rate": 4.921457902821578e-06, |
| "loss": 0.0, |
| "reward": -0.2763333320617676, |
| "reward_std": 0.6768757700920105, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.2763333320617676, |
| "step": 172 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 0.9153439153439153, |
| "grad_norm": 0.0011325571686029434, |
| "kl": 0.0005372216692194343, |
| "learning_rate": 4.9192729262049285e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 173 |
| }, |
| { |
| "completion_length": 443.0, |
| "epoch": 0.9206349206349206, |
| "grad_norm": 0.3447147607803345, |
| "kl": 0.0007221453706733882, |
| "learning_rate": 4.917058471511149e-06, |
| "loss": 0.0, |
| "reward": 0.036000002175569534, |
| "reward_std": 0.08818163722753525, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.036000002175569534, |
| "step": 174 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 0.9259259259259259, |
| "grad_norm": 0.0019907455425709486, |
| "kl": 0.0009091004030779004, |
| "learning_rate": 4.914814565722671e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 175 |
| }, |
| { |
| "completion_length": 466.0, |
| "epoch": 0.9312169312169312, |
| "grad_norm": 0.0013075470924377441, |
| "kl": 0.0007732388330623507, |
| "learning_rate": 4.912541236180779e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 176 |
| }, |
| { |
| "completion_length": 455.0, |
| "epoch": 0.9365079365079365, |
| "grad_norm": 0.0017355694435536861, |
| "kl": 0.0008823199896141887, |
| "learning_rate": 4.910238510585275e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 177 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.9417989417989417, |
| "grad_norm": 0.0015237254556268454, |
| "kl": 0.0008256490691564977, |
| "learning_rate": 4.907906416994146e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 178 |
| }, |
| { |
| "completion_length": 469.0, |
| "epoch": 0.9470899470899471, |
| "grad_norm": 0.0012383477296680212, |
| "kl": 0.0006993532879278064, |
| "learning_rate": 4.905544983823214e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 179 |
| }, |
| { |
| "completion_length": 458.0, |
| "epoch": 0.9523809523809523, |
| "grad_norm": 0.4569125771522522, |
| "kl": 0.0008987709297798574, |
| "learning_rate": 4.903154239845798e-06, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103443980217, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 180 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.9576719576719577, |
| "grad_norm": 0.0015631187707185745, |
| "kl": 0.0008607901399955153, |
| "learning_rate": 4.900734214192358e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 181 |
| }, |
| { |
| "completion_length": 416.8333435058594, |
| "epoch": 0.9629629629629629, |
| "grad_norm": 0.001391576835885644, |
| "kl": 0.0008419002406299114, |
| "learning_rate": 4.898284936350144e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 182 |
| }, |
| { |
| "completion_length": 456.0, |
| "epoch": 0.9682539682539683, |
| "grad_norm": 0.001448813476599753, |
| "kl": 0.0007961566443555057, |
| "learning_rate": 4.8958064361628334e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 183 |
| }, |
| { |
| "completion_length": 463.0, |
| "epoch": 0.9735449735449735, |
| "grad_norm": 0.0015433750813826919, |
| "kl": 0.0008045169524848461, |
| "learning_rate": 4.893298743830168e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 184 |
| }, |
| { |
| "completion_length": 460.0, |
| "epoch": 0.9788359788359788, |
| "grad_norm": 0.0017878510989248753, |
| "kl": 0.00079670938430354, |
| "learning_rate": 4.890761889907589e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 185 |
| }, |
| { |
| "completion_length": 471.5, |
| "epoch": 0.9841269841269841, |
| "grad_norm": 0.3490905165672302, |
| "kl": 0.0008125050226226449, |
| "learning_rate": 4.888195905305859e-06, |
| "loss": 0.0, |
| "reward": 0.0416666679084301, |
| "reward_std": 0.06454972922801971, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0416666679084301, |
| "step": 186 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.9894179894179894, |
| "grad_norm": 0.002016286598518491, |
| "kl": 0.0009556232835166156, |
| "learning_rate": 4.885600821290692e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 187 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 0.9947089947089947, |
| "grad_norm": 0.0029590048361569643, |
| "kl": 0.0009121996117755771, |
| "learning_rate": 4.882976669482368e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 188 |
| }, |
| { |
| "completion_length": 448.66668701171875, |
| "epoch": 1.0, |
| "grad_norm": 0.0014420989900827408, |
| "kl": 0.0007961898809298873, |
| "learning_rate": 4.880323481855347e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 189 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 1.0052910052910053, |
| "grad_norm": 0.0013571071904152632, |
| "kl": 0.0007572842296212912, |
| "learning_rate": 4.8776412907378845e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 190 |
| }, |
| { |
| "completion_length": 458.0, |
| "epoch": 1.0105820105820107, |
| "grad_norm": 0.002301849192008376, |
| "kl": 0.0008989504422061145, |
| "learning_rate": 4.874930128811631e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 191 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 1.0158730158730158, |
| "grad_norm": 0.0017323839711025357, |
| "kl": 0.0008269649697467685, |
| "learning_rate": 4.8721900291112415e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 192 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 1.0211640211640212, |
| "grad_norm": 0.0015301669482141733, |
| "kl": 0.0008518424583598971, |
| "learning_rate": 4.869421025023965e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 193 |
| }, |
| { |
| "completion_length": 466.0, |
| "epoch": 1.0264550264550265, |
| "grad_norm": 0.0016707084141671658, |
| "kl": 0.0007979063666425645, |
| "learning_rate": 4.866623150289241e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 194 |
| }, |
| { |
| "completion_length": 471.0, |
| "epoch": 1.0317460317460316, |
| "grad_norm": 0.0018011715728789568, |
| "kl": 0.0009513082914054394, |
| "learning_rate": 4.863796438998293e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 195 |
| }, |
| { |
| "completion_length": 467.0, |
| "epoch": 1.037037037037037, |
| "grad_norm": 0.002170830499380827, |
| "kl": 0.0008750210981816053, |
| "learning_rate": 4.860940925593703e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 196 |
| }, |
| { |
| "completion_length": 471.0, |
| "epoch": 1.0423280423280423, |
| "grad_norm": 0.0014533688081428409, |
| "kl": 0.0008201130549423397, |
| "learning_rate": 4.858056644869002e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 197 |
| }, |
| { |
| "completion_length": 471.0, |
| "epoch": 1.0476190476190477, |
| "grad_norm": 0.0014290348626673222, |
| "kl": 0.0008280356414616108, |
| "learning_rate": 4.855143631968242e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 198 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 1.052910052910053, |
| "grad_norm": 0.0016640143003314734, |
| "kl": 0.0008244509226642549, |
| "learning_rate": 4.852201922385564e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 199 |
| }, |
| { |
| "completion_length": 394.8333435058594, |
| "epoch": 1.0582010582010581, |
| "grad_norm": 0.0013414421118795872, |
| "kl": 0.000833874917589128, |
| "learning_rate": 4.849231551964771e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 200 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 1.0634920634920635, |
| "grad_norm": 0.0014326253440231085, |
| "kl": 0.0007075238972902298, |
| "learning_rate": 4.84623255689889e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 201 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 1.0687830687830688, |
| "grad_norm": 0.0012063175672665238, |
| "kl": 0.0006964670028537512, |
| "learning_rate": 4.84320497372973e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 202 |
| }, |
| { |
| "completion_length": 464.16668701171875, |
| "epoch": 1.074074074074074, |
| "grad_norm": 0.0014673115219920874, |
| "kl": 0.0007663235883228481, |
| "learning_rate": 4.840148839347434e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 203 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 1.0793650793650793, |
| "grad_norm": 0.0014457846991717815, |
| "kl": 0.0006882932502776384, |
| "learning_rate": 4.837064190990036e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 204 |
| }, |
| { |
| "completion_length": 466.0, |
| "epoch": 1.0846560846560847, |
| "grad_norm": 0.001710255746729672, |
| "kl": 0.0008291201665997505, |
| "learning_rate": 4.833951066243004e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 205 |
| }, |
| { |
| "completion_length": 456.0, |
| "epoch": 1.08994708994709, |
| "grad_norm": 0.0013682242715731263, |
| "kl": 0.0007607044535689056, |
| "learning_rate": 4.830809503038781e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 206 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 1.0952380952380953, |
| "grad_norm": 0.0012869981583207846, |
| "kl": 0.0008233741391450167, |
| "learning_rate": 4.8276395396563215e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 207 |
| }, |
| { |
| "completion_length": 456.0, |
| "epoch": 1.1005291005291005, |
| "grad_norm": 0.0014508500462397933, |
| "kl": 0.000837475701700896, |
| "learning_rate": 4.824441214720629e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 208 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 1.1058201058201058, |
| "grad_norm": 0.001393472426570952, |
| "kl": 0.0008049909374676645, |
| "learning_rate": 4.821214567202284e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 209 |
| }, |
| { |
| "completion_length": 448.0, |
| "epoch": 1.1111111111111112, |
| "grad_norm": 0.0017248099902644753, |
| "kl": 0.0008640000596642494, |
| "learning_rate": 4.817959636416969e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 210 |
| }, |
| { |
| "completion_length": 465.0, |
| "epoch": 1.1164021164021163, |
| "grad_norm": 0.0013942235382273793, |
| "kl": 0.0008910846663638949, |
| "learning_rate": 4.814676462024988e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 211 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 1.1216931216931216, |
| "grad_norm": 0.001571068074554205, |
| "kl": 0.0008616911945864558, |
| "learning_rate": 4.811365084030784e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 212 |
| }, |
| { |
| "completion_length": 450.0, |
| "epoch": 1.126984126984127, |
| "grad_norm": 0.001418941537849605, |
| "kl": 0.0007757723797112703, |
| "learning_rate": 4.808025542782453e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 213 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 1.1322751322751323, |
| "grad_norm": 0.0011643663747236133, |
| "kl": 0.0007587658474221826, |
| "learning_rate": 4.804657878971252e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 214 |
| }, |
| { |
| "completion_length": 463.0, |
| "epoch": 1.1375661375661377, |
| "grad_norm": 0.0016112123848870397, |
| "kl": 0.0008451254107058048, |
| "learning_rate": 4.801262133631101e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 215 |
| }, |
| { |
| "completion_length": 447.0, |
| "epoch": 1.1428571428571428, |
| "grad_norm": 0.001440851017832756, |
| "kl": 0.0007516282494179904, |
| "learning_rate": 4.7978383481380865e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 216 |
| }, |
| { |
| "completion_length": 457.0, |
| "epoch": 1.1481481481481481, |
| "grad_norm": 0.0015707529382780194, |
| "kl": 0.0007482049404643476, |
| "learning_rate": 4.794386564209953e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 217 |
| }, |
| { |
| "completion_length": 465.0, |
| "epoch": 1.1534391534391535, |
| "grad_norm": 0.0016596770146861672, |
| "kl": 0.0009118273737840354, |
| "learning_rate": 4.790906823905599e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 218 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 1.1587301587301586, |
| "grad_norm": 0.0017757281893864274, |
| "kl": 0.0009562873165123165, |
| "learning_rate": 4.787399169624562e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 219 |
| }, |
| { |
| "completion_length": 411.8333435058594, |
| "epoch": 1.164021164021164, |
| "grad_norm": 0.31777122616767883, |
| "kl": 0.0007744102622382343, |
| "learning_rate": 4.783863644106502e-06, |
| "loss": 0.0, |
| "reward": -0.25466668605804443, |
| "reward_std": 0.6238033771514893, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.25466668605804443, |
| "step": 220 |
| }, |
| { |
| "completion_length": 453.0, |
| "epoch": 1.1693121693121693, |
| "grad_norm": 0.0017664993647485971, |
| "kl": 0.0009532844414934516, |
| "learning_rate": 4.780300290430683e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 221 |
| }, |
| { |
| "completion_length": 415.8333435058594, |
| "epoch": 1.1746031746031746, |
| "grad_norm": 0.001434182282537222, |
| "kl": 0.0007502149092033505, |
| "learning_rate": 4.776709152015443e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 222 |
| }, |
| { |
| "completion_length": 435.0, |
| "epoch": 1.17989417989418, |
| "grad_norm": 0.0017983750440180302, |
| "kl": 0.0008337934268638492, |
| "learning_rate": 4.773090272617672e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 223 |
| }, |
| { |
| "completion_length": 419.66668701171875, |
| "epoch": 1.1851851851851851, |
| "grad_norm": 0.32918581366539, |
| "kl": 0.0007857311284169555, |
| "learning_rate": 4.769443696332272e-06, |
| "loss": 0.0, |
| "reward": -0.21400001645088196, |
| "reward_std": 0.5241908431053162, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.21400001645088196, |
| "step": 224 |
| }, |
| { |
| "completion_length": 453.0, |
| "epoch": 1.1904761904761905, |
| "grad_norm": 0.001956671942025423, |
| "kl": 0.0007811601972207427, |
| "learning_rate": 4.765769467591626e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 225 |
| }, |
| { |
| "completion_length": 462.0, |
| "epoch": 1.1957671957671958, |
| "grad_norm": 0.4346919059753418, |
| "kl": 0.0007796824793331325, |
| "learning_rate": 4.762067631165049e-06, |
| "loss": 0.0, |
| "reward": 0.0625, |
| "reward_std": 0.10458251088857651, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0625, |
| "step": 226 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 1.201058201058201, |
| "grad_norm": 0.0019021382322534919, |
| "kl": 0.0009116040891967714, |
| "learning_rate": 4.7583382321582525e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 227 |
| }, |
| { |
| "completion_length": 466.0, |
| "epoch": 1.2063492063492063, |
| "grad_norm": 0.0016885882942005992, |
| "kl": 0.0008838378125801682, |
| "learning_rate": 4.754581316012785e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 228 |
| }, |
| { |
| "completion_length": 471.0, |
| "epoch": 1.2116402116402116, |
| "grad_norm": 0.0015435084933415055, |
| "kl": 0.0009091456304304302, |
| "learning_rate": 4.750796928505484e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 229 |
| }, |
| { |
| "completion_length": 457.8333435058594, |
| "epoch": 1.216931216931217, |
| "grad_norm": 0.001574477064423263, |
| "kl": 0.0007901958888396621, |
| "learning_rate": 4.746985115747918e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 230 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 1.2222222222222223, |
| "grad_norm": 0.0019161113305017352, |
| "kl": 0.0010463364887982607, |
| "learning_rate": 4.743145924185821e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 231 |
| }, |
| { |
| "completion_length": 468.0, |
| "epoch": 1.2275132275132274, |
| "grad_norm": 0.001791435875929892, |
| "kl": 0.0008977344259619713, |
| "learning_rate": 4.7392794005985324e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 232 |
| }, |
| { |
| "completion_length": 436.3333435058594, |
| "epoch": 1.2328042328042328, |
| "grad_norm": 0.34113064408302307, |
| "kl": 0.0008040121174417436, |
| "learning_rate": 4.735385592098421e-06, |
| "loss": 0.0, |
| "reward": -0.3551666736602783, |
| "reward_std": 0.8699771761894226, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.3551666736602783, |
| "step": 233 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 1.2380952380952381, |
| "grad_norm": 0.0017350412672385573, |
| "kl": 0.0008584399474784732, |
| "learning_rate": 4.731464546130315e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 234 |
| }, |
| { |
| "completion_length": 459.0, |
| "epoch": 1.2433862433862433, |
| "grad_norm": 0.001306744059547782, |
| "kl": 0.0008234748966060579, |
| "learning_rate": 4.72751631047092e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 235 |
| }, |
| { |
| "completion_length": 461.0, |
| "epoch": 1.2486772486772486, |
| "grad_norm": 0.001822986756451428, |
| "kl": 0.0009081090684048831, |
| "learning_rate": 4.723540933228245e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 236 |
| }, |
| { |
| "completion_length": 456.0, |
| "epoch": 1.253968253968254, |
| "grad_norm": 0.295440673828125, |
| "kl": 0.0006796099478378892, |
| "learning_rate": 4.719538462841003e-06, |
| "loss": 0.0, |
| "reward": -0.3166666626930237, |
| "reward_std": 0.7756717205047607, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.3166666626930237, |
| "step": 237 |
| }, |
| { |
| "completion_length": 437.8333435058594, |
| "epoch": 1.2592592592592593, |
| "grad_norm": 0.001254405127838254, |
| "kl": 0.0007211491465568542, |
| "learning_rate": 4.715508948078037e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 238 |
| }, |
| { |
| "completion_length": 437.0, |
| "epoch": 1.2645502645502646, |
| "grad_norm": 0.42025598883628845, |
| "kl": 0.0009347737650386989, |
| "learning_rate": 4.71145243803771e-06, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103443980217, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 239 |
| }, |
| { |
| "completion_length": 476.0, |
| "epoch": 1.2698412698412698, |
| "grad_norm": 0.0015842639841139317, |
| "kl": 0.0007844850770197809, |
| "learning_rate": 4.707368982147318e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 240 |
| }, |
| { |
| "completion_length": 447.16668701171875, |
| "epoch": 1.2751322751322751, |
| "grad_norm": 0.3255479633808136, |
| "kl": 0.0007349971565417945, |
| "learning_rate": 4.703258630162481e-06, |
| "loss": 0.0, |
| "reward": -0.29883334040641785, |
| "reward_std": 0.7319891452789307, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": -0.29883334040641785, |
| "step": 241 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 1.2804232804232805, |
| "grad_norm": 0.3012905716896057, |
| "kl": 0.0008209796505980194, |
| "learning_rate": 4.699121432166542e-06, |
| "loss": 0.0, |
| "reward": 0.02083333395421505, |
| "reward_std": 0.05103103816509247, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.02083333395421505, |
| "step": 242 |
| }, |
| { |
| "completion_length": 470.0, |
| "epoch": 1.2857142857142856, |
| "grad_norm": 0.0020072192419320345, |
| "kl": 0.000969195447396487, |
| "learning_rate": 4.6949574385699514e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 243 |
| }, |
| { |
| "completion_length": 453.0, |
| "epoch": 1.291005291005291, |
| "grad_norm": 0.0013635673094540834, |
| "kl": 0.0007304962491616607, |
| "learning_rate": 4.690766700109659e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 244 |
| }, |
| { |
| "completion_length": 457.0, |
| "epoch": 1.2962962962962963, |
| "grad_norm": 0.0012732620816677809, |
| "kl": 0.0007430926198139787, |
| "learning_rate": 4.68654926784849e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 245 |
| }, |
| { |
| "completion_length": 460.0, |
| "epoch": 1.3015873015873016, |
| "grad_norm": 0.0016206332948058844, |
| "kl": 0.0008289730176329613, |
| "learning_rate": 4.682305193174524e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 246 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 1.306878306878307, |
| "grad_norm": 0.0013530352152884007, |
| "kl": 0.0007417545421048999, |
| "learning_rate": 4.6780345278004744e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 247 |
| }, |
| { |
| "completion_length": 414.8333435058594, |
| "epoch": 1.312169312169312, |
| "grad_norm": 0.001749272458255291, |
| "kl": 0.000826512579806149, |
| "learning_rate": 4.673737323763048e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 248 |
| }, |
| { |
| "completion_length": 454.0, |
| "epoch": 1.3174603174603174, |
| "grad_norm": 0.0013661340344697237, |
| "kl": 0.0008204582845792174, |
| "learning_rate": 4.669413633422322e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 249 |
| }, |
| { |
| "completion_length": 464.0, |
| "epoch": 1.3227513227513228, |
| "grad_norm": 0.0014948459574952722, |
| "kl": 0.0008490080363117158, |
| "learning_rate": 4.665063509461098e-06, |
| "loss": 0.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/correctness_reward_func": 0.0, |
| "rewards/int_reward_func": 0.0, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.0, |
| "rewards/xmlcount_reward_func": 0.0, |
| "step": 250 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 1000, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 6, |
| "save_steps": 250, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
