diff --git "a/last-checkpoint/trainer_state.json" "b/last-checkpoint/trainer_state.json" --- "a/last-checkpoint/trainer_state.json" +++ "b/last-checkpoint/trainer_state.json" @@ -2,9 +2,9 @@ "best_global_step": 1500, "best_metric": 0.582668250799179, "best_model_checkpoint": "/root/workspace/Shaer/grpo/outputs/train/shaer_grpo_20260413_071842/checkpoint-1500", - "epoch": 0.21094927172275238, + "epoch": 0.21597187343043697, "eval_steps": 50, - "global_step": 2100, + "global_step": 2150, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, @@ -115718,11 +115718,2766 @@ "eval_sampling/sampling_logp_difference/mean": 0.055311382934451106, "eval_steps_per_second": 0.161, "step": 2100 + }, + { + "clip_ratio/high_max": 0.11735270358622074, + "clip_ratio/high_mean": 0.11735270358622074, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.11735270358622074, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 89.0, + "completions/mean_length": 128.5, + "completions/mean_terminated_length": 73.71428680419922, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "entropy": 0.7288566678762436, + "epoch": 0.2110497237569061, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.856912612915039, + "learning_rate": 3.6363636363636366e-06, + "loss": -0.1714, + "num_tokens": 3779100.0, + "reward": 0.37015044689178467, + "reward_count_adherence_mean": 0.9166666865348816, + "reward_count_adherence_std": 0.2357022762298584, + "reward_hard_gate_mean": 0.875, + "reward_hard_gate_std": 0.3535533845424652, + "reward_judge_quality_mean": 0.14125001430511475, + "reward_judge_quality_std": 0.038335926830768585, + "reward_meter_mean": 0.7168045043945312, + "reward_meter_std": 0.3875977396965027, + "reward_repeat_soft_mean": 0.952052652835846, + "reward_repeat_soft_std": 0.033339813351631165, + "reward_std": 0.15158022940158844, + "reward_total_composite_mean": 0.37015044689178467, + "reward_total_composite_std": 0.15158022940158844, + "reward_total_mean": 0.37015044689178467, + "rewards/count_adherence/mean": 0.9166666865348816, + "rewards/count_adherence/std": 0.2357022762298584, + "rewards/hard_gate/mean": 0.875, + "rewards/hard_gate/std": 0.3535533845424652, + "rewards/judge_quality/mean": 0.14125001430511475, + "rewards/judge_quality/std": 0.038335926830768585, + "rewards/meter/mean": 0.7168045043945312, + "rewards/meter/std": 0.3875977396965027, + "rewards/repeat_soft/mean": 0.952052652835846, + "rewards/repeat_soft/std": 0.033339813351631165, + "rewards/total_composite/mean": 0.37015044689178467, + "rewards/total_composite/std": 0.15158022940158844, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9991225600242615, + "sampling/importance_sampling_ratio/min": 0.1656540483236313, + "sampling/sampling_logp_difference/max": 1.797853708267212, + "sampling/sampling_logp_difference/mean": 0.11523141711950302, + "step": 2101 + }, + { + "clip_ratio/high_max": 0.07393320184201002, + "clip_ratio/high_mean": 0.07393320184201002, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.07393320184201002, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 45.0, + "completions/mean_length": 98.25, + "completions/mean_terminated_length": 39.142860412597656, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.7180831506848335, + "epoch": 0.21115017579105977, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.055514097213745, + "learning_rate": 3.633333333333334e-06, + "loss": -0.1172, + "num_tokens": 3780782.0, + "reward": 0.35340049862861633, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 0.875, + "reward_hard_gate_std": 0.3535533845424652, + "reward_judge_quality_mean": 0.1862500011920929, + "reward_judge_quality_std": 0.11475905776023865, + "reward_meter_mean": 0.44329869747161865, + "reward_meter_std": 0.40356191992759705, + "reward_repeat_soft_mean": 0.9842722415924072, + "reward_repeat_soft_std": 0.024804506450891495, + "reward_std": 0.14779430627822876, + "reward_total_composite_mean": 0.35340049862861633, + "reward_total_composite_std": 0.14779430627822876, + "reward_total_mean": 0.35340049862861633, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 0.875, + "rewards/hard_gate/std": 0.3535533845424652, + "rewards/judge_quality/mean": 0.1862500011920929, + "rewards/judge_quality/std": 0.11475905776023865, + "rewards/meter/mean": 0.44329869747161865, + "rewards/meter/std": 0.40356191992759705, + "rewards/repeat_soft/mean": 0.9842722415924072, + "rewards/repeat_soft/std": 0.024804506450891495, + "rewards/total_composite/mean": 0.35340049862861633, + "rewards/total_composite/std": 0.14779430627822876, + "sampling/importance_sampling_ratio/max": 1.880364179611206, + "sampling/importance_sampling_ratio/mean": 1.0372201204299927, + "sampling/importance_sampling_ratio/min": 0.33279985189437866, + "sampling/sampling_logp_difference/max": 1.1002140045166016, + "sampling/sampling_logp_difference/mean": 0.12127834558486938, + "step": 2102 + }, + { + "clip_ratio/high_max": 0.05287114903330803, + "clip_ratio/high_mean": 0.05287114903330803, + "clip_ratio/low_mean": 0.07524633966386318, + "clip_ratio/low_min": 0.07524633966386318, + "clip_ratio/region_mean": 0.1281174886971712, + "completions/clipped_ratio": 0.0, + "completions/max_length": 51.0, + "completions/max_terminated_length": 51.0, + "completions/mean_length": 48.75, + "completions/mean_terminated_length": 48.75, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.807975247502327, + "epoch": 0.21125062782521345, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.103949546813965, + "learning_rate": 3.6303030303030307e-06, + "loss": 0.0086, + "num_tokens": 3782484.0, + "reward": 0.39005687832832336, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.14625000953674316, + "reward_judge_quality_std": 0.010606604628264904, + "reward_meter_mean": 0.442565381526947, + "reward_meter_std": 0.30177053809165955, + "reward_repeat_soft_mean": 0.9805001020431519, + "reward_repeat_soft_std": 0.017646679654717445, + "reward_std": 0.02893988974392414, + "reward_total_composite_mean": 0.39005687832832336, + "reward_total_composite_std": 0.02893988788127899, + "reward_total_mean": 0.39005687832832336, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.14625000953674316, + "rewards/judge_quality/std": 0.010606604628264904, + "rewards/meter/mean": 0.442565381526947, + "rewards/meter/std": 0.30177053809165955, + "rewards/repeat_soft/mean": 0.9805001020431519, + "rewards/repeat_soft/std": 0.017646679654717445, + "rewards/total_composite/mean": 0.39005687832832336, + "rewards/total_composite/std": 0.02893988788127899, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.005758285522461, + "sampling/importance_sampling_ratio/min": 0.08134415000677109, + "sampling/sampling_logp_difference/max": 2.509066343307495, + "sampling/sampling_logp_difference/mean": 0.1269722580909729, + "step": 2103 + }, + { + "clip_ratio/high_max": 0.07308703940361738, + "clip_ratio/high_mean": 0.07308703940361738, + "clip_ratio/low_mean": 0.03905245568603277, + "clip_ratio/low_min": 0.03905245568603277, + "clip_ratio/region_mean": 0.11213949508965015, + "completions/clipped_ratio": 0.0, + "completions/max_length": 105.0, + "completions/max_terminated_length": 105.0, + "completions/mean_length": 88.125, + "completions/mean_terminated_length": 88.125, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "entropy": 0.43947426602244377, + "epoch": 0.21135107985936716, + "frac_reward_zero_std": 0.0, + "grad_norm": 10.681594848632812, + "learning_rate": 3.6272727272727275e-06, + "loss": -0.0374, + "num_tokens": 3784613.0, + "reward": 0.4134945273399353, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.16875001788139343, + "reward_judge_quality_std": 0.015526476316154003, + "reward_meter_mean": 0.6759259700775146, + "reward_meter_std": 0.307078093290329, + "reward_repeat_soft_mean": 0.9187784194946289, + "reward_repeat_soft_std": 0.04656059294939041, + "reward_std": 0.0365760400891304, + "reward_total_composite_mean": 0.4134945273399353, + "reward_total_composite_std": 0.036576032638549805, + "reward_total_mean": 0.4134945273399353, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.16875001788139343, + "rewards/judge_quality/std": 0.015526476316154003, + "rewards/meter/mean": 0.6759259700775146, + "rewards/meter/std": 0.307078093290329, + "rewards/repeat_soft/mean": 0.9187784194946289, + "rewards/repeat_soft/std": 0.04656059294939041, + "rewards/total_composite/mean": 0.4134945273399353, + "rewards/total_composite/std": 0.036576032638549805, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9974398612976074, + "sampling/importance_sampling_ratio/min": 0.10594313591718674, + "sampling/sampling_logp_difference/max": 2.2448527812957764, + "sampling/sampling_logp_difference/mean": 0.11971750110387802, + "step": 2104 + }, + { + "clip_ratio/high_max": 0.08387564588338137, + "clip_ratio/high_mean": 0.08387564588338137, + "clip_ratio/low_mean": 0.03388928063213825, + "clip_ratio/low_min": 0.03388928063213825, + "clip_ratio/region_mean": 0.11776492651551962, + "completions/clipped_ratio": 0.0, + "completions/max_length": 55.0, + "completions/max_terminated_length": 55.0, + "completions/mean_length": 48.125, + "completions/mean_terminated_length": 48.125, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "entropy": 0.8007223606109619, + "epoch": 0.21145153189352084, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.617268562316895, + "learning_rate": 3.6242424242424248e-06, + "loss": 0.0628, + "num_tokens": 3786334.0, + "reward": 0.4121430814266205, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.6768026947975159, + "reward_meter_std": 0.35109129548072815, + "reward_repeat_soft_mean": 0.9743654727935791, + "reward_repeat_soft_std": 0.02952939085662365, + "reward_std": 0.034643322229385376, + "reward_total_composite_mean": 0.4121430814266205, + "reward_total_composite_std": 0.034643322229385376, + "reward_total_mean": 0.4121430814266205, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.6768026947975159, + "rewards/meter/std": 0.35109129548072815, + "rewards/repeat_soft/mean": 0.9743654727935791, + "rewards/repeat_soft/std": 0.02952939085662365, + "rewards/total_composite/mean": 0.4121430814266205, + "rewards/total_composite/std": 0.034643322229385376, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0105738639831543, + "sampling/importance_sampling_ratio/min": 0.0959840640425682, + "sampling/sampling_logp_difference/max": 2.3435730934143066, + "sampling/sampling_logp_difference/mean": 0.12021853029727936, + "step": 2105 + }, + { + "clip_ratio/high_max": 0.05873551592230797, + "clip_ratio/high_mean": 0.05873551592230797, + "clip_ratio/low_mean": 0.0763331362977624, + "clip_ratio/low_min": 0.0763331362977624, + "clip_ratio/region_mean": 0.13506865222007036, + "completions/clipped_ratio": 0.0, + "completions/max_length": 96.0, + "completions/max_terminated_length": 96.0, + "completions/mean_length": 84.5, + "completions/mean_terminated_length": 84.5, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "entropy": 0.7003876119852066, + "epoch": 0.21155198392767455, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.996867179870605, + "learning_rate": 3.6212121212121216e-06, + "loss": 0.0452, + "num_tokens": 3788346.0, + "reward": 0.40567585825920105, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.1612500101327896, + "reward_judge_quality_std": 0.022320717573165894, + "reward_meter_mean": 0.6127023100852966, + "reward_meter_std": 0.33788344264030457, + "reward_repeat_soft_mean": 0.950439453125, + "reward_repeat_soft_std": 0.04181034490466118, + "reward_std": 0.03350795805454254, + "reward_total_composite_mean": 0.40567585825920105, + "reward_total_composite_std": 0.03350796550512314, + "reward_total_mean": 0.40567585825920105, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.1612500101327896, + "rewards/judge_quality/std": 0.022320717573165894, + "rewards/meter/mean": 0.6127023100852966, + "rewards/meter/std": 0.33788344264030457, + "rewards/repeat_soft/mean": 0.950439453125, + "rewards/repeat_soft/std": 0.04181034490466118, + "rewards/total_composite/mean": 0.40567585825920105, + "rewards/total_composite/std": 0.03350796550512314, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0094692707061768, + "sampling/importance_sampling_ratio/min": 0.1115092858672142, + "sampling/sampling_logp_difference/max": 2.1936473846435547, + "sampling/sampling_logp_difference/mean": 0.11898557841777802, + "step": 2106 + }, + { + "clip_ratio/high_max": 0.03724791551940143, + "clip_ratio/high_mean": 0.03724791551940143, + "clip_ratio/low_mean": 0.027148242574185133, + "clip_ratio/low_min": 0.027148242574185133, + "clip_ratio/region_mean": 0.06439615809358656, + "completions/clipped_ratio": 0.0, + "completions/max_length": 46.0, + "completions/max_terminated_length": 46.0, + "completions/mean_length": 41.25, + "completions/mean_terminated_length": 41.25, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.3964330926537514, + "epoch": 0.21165243596182823, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.233802795410156, + "learning_rate": 3.6181818181818184e-06, + "loss": 0.0247, + "num_tokens": 3789996.0, + "reward": 0.40734606981277466, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.13875000178813934, + "reward_judge_quality_std": 0.022320719435811043, + "reward_meter_mean": 0.9781842231750488, + "reward_meter_std": 0.015315087512135506, + "reward_repeat_soft_mean": 0.7953553199768066, + "reward_repeat_soft_std": 0.11163556575775146, + "reward_std": 0.027200007811188698, + "reward_total_composite_mean": 0.40734606981277466, + "reward_total_composite_std": 0.02720000222325325, + "reward_total_mean": 0.40734606981277466, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.13875000178813934, + "rewards/judge_quality/std": 0.022320719435811043, + "rewards/meter/mean": 0.9781842231750488, + "rewards/meter/std": 0.015315087512135506, + "rewards/repeat_soft/mean": 0.7953553199768066, + "rewards/repeat_soft/std": 0.11163556575775146, + "rewards/total_composite/mean": 0.40734606981277466, + "rewards/total_composite/std": 0.02720000222325325, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0048346519470215, + "sampling/importance_sampling_ratio/min": 0.15594787895679474, + "sampling/sampling_logp_difference/max": 1.8582334518432617, + "sampling/sampling_logp_difference/mean": 0.09198253601789474, + "step": 2107 + }, + { + "clip_ratio/high_max": 0.016964286100119352, + "clip_ratio/high_mean": 0.016964286100119352, + "clip_ratio/low_mean": 0.054619363974779844, + "clip_ratio/low_min": 0.054619363974779844, + "clip_ratio/region_mean": 0.0715836500748992, + "completions/clipped_ratio": 0.0, + "completions/max_length": 42.0, + "completions/max_terminated_length": 42.0, + "completions/mean_length": 38.625, + "completions/mean_terminated_length": 38.625, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "entropy": 0.41267629340291023, + "epoch": 0.2117528879959819, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.737648963928223, + "learning_rate": 3.6151515151515153e-06, + "loss": 0.0295, + "num_tokens": 3791689.0, + "reward": 0.38604602217674255, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.16124999523162842, + "reward_judge_quality_std": 0.015526476316154003, + "reward_meter_mean": 0.46569669246673584, + "reward_meter_std": 0.4034918248653412, + "reward_repeat_soft_mean": 0.9199074506759644, + "reward_repeat_soft_std": 0.04545706883072853, + "reward_std": 0.0423961766064167, + "reward_total_composite_mean": 0.38604602217674255, + "reward_total_composite_std": 0.042396180331707, + "reward_total_mean": 0.38604602217674255, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.16124999523162842, + "rewards/judge_quality/std": 0.015526476316154003, + "rewards/meter/mean": 0.46569669246673584, + "rewards/meter/std": 0.4034918248653412, + "rewards/repeat_soft/mean": 0.9199074506759644, + "rewards/repeat_soft/std": 0.04545706883072853, + "rewards/total_composite/mean": 0.38604602217674255, + "rewards/total_composite/std": 0.042396180331707, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0085159540176392, + "sampling/importance_sampling_ratio/min": 0.08646226674318314, + "sampling/sampling_logp_difference/max": 2.448047161102295, + "sampling/sampling_logp_difference/mean": 0.08647025376558304, + "step": 2108 + }, + { + "clip_ratio/high_max": 0.03729429328814149, + "clip_ratio/high_mean": 0.03729429328814149, + "clip_ratio/low_mean": 0.05624600313603878, + "clip_ratio/low_min": 0.05624600313603878, + "clip_ratio/region_mean": 0.09354029642418027, + "completions/clipped_ratio": 0.0, + "completions/max_length": 47.0, + "completions/max_terminated_length": 47.0, + "completions/mean_length": 38.5, + "completions/mean_terminated_length": 38.5, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "entropy": 0.5103457905352116, + "epoch": 0.21185334003013562, + "frac_reward_zero_std": 0.0, + "grad_norm": 13.237957000732422, + "learning_rate": 3.6121212121212125e-06, + "loss": 0.1557, + "num_tokens": 3793245.0, + "reward": 0.3842517137527466, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.19500000774860382, + "reward_judge_quality_std": 0.10392305254936218, + "reward_meter_mean": 0.33292731642723083, + "reward_meter_std": 0.2673882842063904, + "reward_repeat_soft_mean": 0.9947908520698547, + "reward_repeat_soft_std": 0.007985309697687626, + "reward_std": 0.02546876110136509, + "reward_total_composite_mean": 0.3842517137527466, + "reward_total_composite_std": 0.025468753650784492, + "reward_total_mean": 0.3842517137527466, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.19500000774860382, + "rewards/judge_quality/std": 0.10392305254936218, + "rewards/meter/mean": 0.33292731642723083, + "rewards/meter/std": 0.2673882842063904, + "rewards/repeat_soft/mean": 0.9947908520698547, + "rewards/repeat_soft/std": 0.007985309697687626, + "rewards/total_composite/mean": 0.3842517137527466, + "rewards/total_composite/std": 0.025468753650784492, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0069389343261719, + "sampling/importance_sampling_ratio/min": 0.11226688325405121, + "sampling/sampling_logp_difference/max": 2.1868762969970703, + "sampling/sampling_logp_difference/mean": 0.11784003674983978, + "step": 2109 + }, + { + "clip_ratio/high_max": 0.06695579458028078, + "clip_ratio/high_mean": 0.06695579458028078, + "clip_ratio/low_mean": 0.02714580879546702, + "clip_ratio/low_min": 0.02714580879546702, + "clip_ratio/region_mean": 0.0941016033757478, + "completions/clipped_ratio": 0.0, + "completions/max_length": 49.0, + "completions/max_terminated_length": 49.0, + "completions/mean_length": 41.875, + "completions/mean_terminated_length": 41.875, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "entropy": 0.5558187812566757, + "epoch": 0.2119537920642893, + "frac_reward_zero_std": 0.0, + "grad_norm": 13.800068855285645, + "learning_rate": 3.6090909090909093e-06, + "loss": 0.0113, + "num_tokens": 3794772.0, + "reward": 0.4227389991283417, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.8712973594665527, + "reward_meter_std": 0.18867942690849304, + "reward_repeat_soft_mean": 0.918583333492279, + "reward_repeat_soft_std": 0.05381178855895996, + "reward_std": 0.022761141881346703, + "reward_total_composite_mean": 0.4227389991283417, + "reward_total_composite_std": 0.022761140018701553, + "reward_total_mean": 0.4227389991283417, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.8712973594665527, + "rewards/meter/std": 0.18867942690849304, + "rewards/repeat_soft/mean": 0.918583333492279, + "rewards/repeat_soft/std": 0.05381178855895996, + "rewards/total_composite/mean": 0.4227389991283417, + "rewards/total_composite/std": 0.022761140018701553, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0151242017745972, + "sampling/importance_sampling_ratio/min": 0.20714989304542542, + "sampling/sampling_logp_difference/max": 1.574312686920166, + "sampling/sampling_logp_difference/mean": 0.11791300028562546, + "step": 2110 + }, + { + "clip_ratio/high_max": 0.036213971907272935, + "clip_ratio/high_mean": 0.036213971907272935, + "clip_ratio/low_mean": 0.0406727041117847, + "clip_ratio/low_min": 0.0406727041117847, + "clip_ratio/region_mean": 0.07688667601905763, + "completions/clipped_ratio": 0.0, + "completions/max_length": 109.0, + "completions/max_terminated_length": 109.0, + "completions/mean_length": 89.125, + "completions/mean_terminated_length": 89.125, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "entropy": 0.40958814322948456, + "epoch": 0.212054244098443, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.104913711547852, + "learning_rate": 3.606060606060606e-06, + "loss": 0.0434, + "num_tokens": 3796749.0, + "reward": 0.33592337369918823, + "reward_count_adherence_mean": 0.8999999761581421, + "reward_count_adherence_std": 0.10690449178218842, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.13875000178813934, + "reward_judge_quality_std": 0.022320719435811043, + "reward_meter_mean": 0.6411469578742981, + "reward_meter_std": 0.302879273891449, + "reward_repeat_soft_mean": 0.6518181562423706, + "reward_repeat_soft_std": 0.11911094933748245, + "reward_std": 0.020304612815380096, + "reward_total_composite_mean": 0.33592337369918823, + "reward_total_composite_std": 0.020304610952734947, + "reward_total_mean": 0.33592337369918823, + "rewards/count_adherence/mean": 0.8999999761581421, + "rewards/count_adherence/std": 0.10690449178218842, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.13875000178813934, + "rewards/judge_quality/std": 0.022320719435811043, + "rewards/meter/mean": 0.6411469578742981, + "rewards/meter/std": 0.302879273891449, + "rewards/repeat_soft/mean": 0.6518181562423706, + "rewards/repeat_soft/std": 0.11911094933748245, + "rewards/total_composite/mean": 0.33592337369918823, + "rewards/total_composite/std": 0.020304610952734947, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0048784017562866, + "sampling/importance_sampling_ratio/min": 0.13876067101955414, + "sampling/sampling_logp_difference/max": 1.9750046730041504, + "sampling/sampling_logp_difference/mean": 0.07979433983564377, + "step": 2111 + }, + { + "clip_ratio/high_max": 0.0698704281821847, + "clip_ratio/high_mean": 0.0698704281821847, + "clip_ratio/low_mean": 0.0528691983781755, + "clip_ratio/low_min": 0.0528691983781755, + "clip_ratio/region_mean": 0.1227396265603602, + "completions/clipped_ratio": 0.0, + "completions/max_length": 58.0, + "completions/max_terminated_length": 58.0, + "completions/mean_length": 42.5, + "completions/mean_terminated_length": 42.5, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 0.6444509625434875, + "epoch": 0.2121546961325967, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.300381660461426, + "learning_rate": 3.603030303030303e-06, + "loss": -0.0532, + "num_tokens": 3798369.0, + "reward": 0.3744850158691406, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.4879136383533478, + "reward_meter_std": 0.4658832550048828, + "reward_repeat_soft_mean": 0.8460896015167236, + "reward_repeat_soft_std": 0.058859921991825104, + "reward_std": 0.04525057598948479, + "reward_total_composite_mean": 0.3744850158691406, + "reward_total_composite_std": 0.045250579714775085, + "reward_total_mean": 0.3744850158691406, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.4879136383533478, + "rewards/meter/std": 0.4658832550048828, + "rewards/repeat_soft/mean": 0.8460896015167236, + "rewards/repeat_soft/std": 0.058859921991825104, + "rewards/total_composite/mean": 0.3744850158691406, + "rewards/total_composite/std": 0.045250579714775085, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0110188722610474, + "sampling/importance_sampling_ratio/min": 0.3320287764072418, + "sampling/sampling_logp_difference/max": 1.1025335788726807, + "sampling/sampling_logp_difference/mean": 0.11138882488012314, + "step": 2112 + }, + { + "clip_ratio/high_max": 0.09421969391405582, + "clip_ratio/high_mean": 0.09421969391405582, + "clip_ratio/low_mean": 0.047727273777127266, + "clip_ratio/low_min": 0.047727273777127266, + "clip_ratio/region_mean": 0.1419469676911831, + "completions/clipped_ratio": 0.0, + "completions/max_length": 32.0, + "completions/max_terminated_length": 32.0, + "completions/mean_length": 22.0, + "completions/mean_terminated_length": 22.0, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "entropy": 0.7180051393806934, + "epoch": 0.21225514816675037, + "frac_reward_zero_std": 0.0, + "grad_norm": 16.208444595336914, + "learning_rate": 3.6000000000000003e-06, + "loss": 0.0482, + "num_tokens": 3799785.0, + "reward": 0.4071803689002991, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.6580546498298645, + "reward_meter_std": 0.41127723455429077, + "reward_repeat_soft_mean": 0.9534668326377869, + "reward_repeat_soft_std": 0.014675324782729149, + "reward_std": 0.03894574195146561, + "reward_total_composite_mean": 0.4071803689002991, + "reward_total_composite_std": 0.038945745676755905, + "reward_total_mean": 0.4071803689002991, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.6580546498298645, + "rewards/meter/std": 0.41127723455429077, + "rewards/repeat_soft/mean": 0.9534668326377869, + "rewards/repeat_soft/std": 0.014675324782729149, + "rewards/total_composite/mean": 0.4071803689002991, + "rewards/total_composite/std": 0.038945745676755905, + "sampling/importance_sampling_ratio/max": 1.6984217166900635, + "sampling/importance_sampling_ratio/mean": 0.9922757744789124, + "sampling/importance_sampling_ratio/min": 0.15824668109416962, + "sampling/sampling_logp_difference/max": 1.8436002731323242, + "sampling/sampling_logp_difference/mean": 0.13734646141529083, + "step": 2113 + }, + { + "clip_ratio/high_max": 0.08157390914857388, + "clip_ratio/high_mean": 0.08157390914857388, + "clip_ratio/low_mean": 0.01024590153247118, + "clip_ratio/low_min": 0.01024590153247118, + "clip_ratio/region_mean": 0.09181981068104506, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 70.0, + "completions/mean_length": 115.75, + "completions/mean_terminated_length": 59.142860412597656, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.47304895520210266, + "epoch": 0.21235560020090408, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8867636919021606, + "learning_rate": 3.596969696969697e-06, + "loss": -0.1491, + "num_tokens": 3801567.0, + "reward": 0.36274904012680054, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 0.875, + "reward_hard_gate_std": 0.3535533845424652, + "reward_judge_quality_mean": 0.1525000035762787, + "reward_judge_quality_std": 0.04399675503373146, + "reward_meter_mean": 0.5922025442123413, + "reward_meter_std": 0.3228558599948883, + "reward_repeat_soft_mean": 0.971898078918457, + "reward_repeat_soft_std": 0.02474227547645569, + "reward_std": 0.1502305418252945, + "reward_total_composite_mean": 0.36274904012680054, + "reward_total_composite_std": 0.1502305567264557, + "reward_total_mean": 0.36274904012680054, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 0.875, + "rewards/hard_gate/std": 0.3535533845424652, + "rewards/judge_quality/mean": 0.1525000035762787, + "rewards/judge_quality/std": 0.04399675503373146, + "rewards/meter/mean": 0.5922025442123413, + "rewards/meter/std": 0.3228558599948883, + "rewards/repeat_soft/mean": 0.971898078918457, + "rewards/repeat_soft/std": 0.02474227547645569, + "rewards/total_composite/mean": 0.36274904012680054, + "rewards/total_composite/std": 0.1502305567264557, + "sampling/importance_sampling_ratio/max": 1.9030864238739014, + "sampling/importance_sampling_ratio/mean": 0.9857916235923767, + "sampling/importance_sampling_ratio/min": 0.04539196565747261, + "sampling/sampling_logp_difference/max": 3.0924201011657715, + "sampling/sampling_logp_difference/mean": 0.11633001267910004, + "step": 2114 + }, + { + "clip_ratio/high_max": 0.1053868168964982, + "clip_ratio/high_mean": 0.1053868168964982, + "clip_ratio/low_mean": 0.011627906933426857, + "clip_ratio/low_min": 0.011627906933426857, + "clip_ratio/region_mean": 0.11701472382992506, + "completions/clipped_ratio": 0.0, + "completions/max_length": 46.0, + "completions/max_terminated_length": 46.0, + "completions/mean_length": 43.875, + "completions/mean_terminated_length": 43.875, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "entropy": 0.4199068732559681, + "epoch": 0.21245605223505776, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.018208503723145, + "learning_rate": 3.593939393939394e-06, + "loss": 0.0127, + "num_tokens": 3803102.0, + "reward": 0.4282057285308838, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.1537500023841858, + "reward_judge_quality_std": 0.010606602765619755, + "reward_meter_mean": 0.8543543219566345, + "reward_meter_std": 0.28948941826820374, + "reward_repeat_soft_mean": 0.9505391120910645, + "reward_repeat_soft_std": 0.01934647373855114, + "reward_std": 0.03167115896940231, + "reward_total_composite_mean": 0.4282057285308838, + "reward_total_composite_std": 0.03167116641998291, + "reward_total_mean": 0.4282057285308838, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.1537500023841858, + "rewards/judge_quality/std": 0.010606602765619755, + "rewards/meter/mean": 0.8543543219566345, + "rewards/meter/std": 0.28948941826820374, + "rewards/repeat_soft/mean": 0.9505391120910645, + "rewards/repeat_soft/std": 0.01934647373855114, + "rewards/total_composite/mean": 0.4282057285308838, + "rewards/total_composite/std": 0.03167116641998291, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9898253679275513, + "sampling/importance_sampling_ratio/min": 0.19986747205257416, + "sampling/sampling_logp_difference/max": 1.6101007461547852, + "sampling/sampling_logp_difference/mean": 0.10174911469221115, + "step": 2115 + }, + { + "clip_ratio/high_max": 0.009433962404727936, + "clip_ratio/high_mean": 0.009433962404727936, + "clip_ratio/low_mean": 0.08550144638866186, + "clip_ratio/low_min": 0.08550144638866186, + "clip_ratio/region_mean": 0.0949354087933898, + "completions/clipped_ratio": 0.0, + "completions/max_length": 56.0, + "completions/max_terminated_length": 56.0, + "completions/mean_length": 51.375, + "completions/mean_terminated_length": 51.375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.5200207754969597, + "epoch": 0.21255650426921144, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.96137809753418, + "learning_rate": 3.590909090909091e-06, + "loss": 0.0131, + "num_tokens": 3804769.0, + "reward": 0.46495306491851807, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.25, + "reward_judge_quality_std": 0.2709243595600128, + "reward_meter_mean": 0.610297441482544, + "reward_meter_std": 0.37020719051361084, + "reward_repeat_soft_mean": 0.9516139030456543, + "reward_repeat_soft_std": 0.03949163854122162, + "reward_std": 0.1915210336446762, + "reward_total_composite_mean": 0.46495306491851807, + "reward_total_composite_std": 0.1915210336446762, + "reward_total_mean": 0.46495306491851807, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.25, + "rewards/judge_quality/std": 0.2709243595600128, + "rewards/meter/mean": 0.610297441482544, + "rewards/meter/std": 0.37020719051361084, + "rewards/repeat_soft/mean": 0.9516139030456543, + "rewards/repeat_soft/std": 0.03949163854122162, + "rewards/total_composite/mean": 0.46495306491851807, + "rewards/total_composite/std": 0.1915210336446762, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0204806327819824, + "sampling/importance_sampling_ratio/min": 0.20287548005580902, + "sampling/sampling_logp_difference/max": 1.5951628684997559, + "sampling/sampling_logp_difference/mean": 0.0902131050825119, + "step": 2116 + }, + { + "clip_ratio/high_max": 0.04374784370884299, + "clip_ratio/high_mean": 0.04374784370884299, + "clip_ratio/low_mean": 0.05163145810365677, + "clip_ratio/low_min": 0.05163145810365677, + "clip_ratio/region_mean": 0.09537930181249976, + "completions/clipped_ratio": 0.0, + "completions/max_length": 68.0, + "completions/max_terminated_length": 68.0, + "completions/mean_length": 56.0, + "completions/mean_terminated_length": 56.0, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.6553157195448875, + "epoch": 0.21265695630336515, + "frac_reward_zero_std": 0.0, + "grad_norm": 10.987751960754395, + "learning_rate": 3.587878787878788e-06, + "loss": -0.0345, + "num_tokens": 3806689.0, + "reward": 0.388683557510376, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.14625000953674316, + "reward_judge_quality_std": 0.010606604628264904, + "reward_meter_mean": 0.48459476232528687, + "reward_meter_std": 0.3567312955856323, + "reward_repeat_soft_mean": 0.9572969079017639, + "reward_repeat_soft_std": 0.03782852366566658, + "reward_std": 0.03286158666014671, + "reward_total_composite_mean": 0.388683557510376, + "reward_total_composite_std": 0.03286159411072731, + "reward_total_mean": 0.388683557510376, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.14625000953674316, + "rewards/judge_quality/std": 0.010606604628264904, + "rewards/meter/mean": 0.48459476232528687, + "rewards/meter/std": 0.3567312955856323, + "rewards/repeat_soft/mean": 0.9572969079017639, + "rewards/repeat_soft/std": 0.03782852366566658, + "rewards/total_composite/mean": 0.388683557510376, + "rewards/total_composite/std": 0.03286159411072731, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0060267448425293, + "sampling/importance_sampling_ratio/min": 0.037821173667907715, + "sampling/sampling_logp_difference/max": 3.274886131286621, + "sampling/sampling_logp_difference/mean": 0.11895419657230377, + "step": 2117 + }, + { + "clip_ratio/high_max": 0.11489598639309406, + "clip_ratio/high_mean": 0.11489598639309406, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.11489598639309406, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 57.0, + "completions/mean_length": 106.875, + "completions/mean_terminated_length": 49.000003814697266, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "entropy": 0.5105771720409393, + "epoch": 0.21275740833751883, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8776116371154785, + "learning_rate": 3.584848484848485e-06, + "loss": -0.1357, + "num_tokens": 3808480.0, + "reward": 0.384857714176178, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 0.875, + "reward_hard_gate_std": 0.3535533845424652, + "reward_judge_quality_mean": 0.13375000655651093, + "reward_judge_quality_std": 0.03543102368712425, + "reward_meter_mean": 0.9100183248519897, + "reward_meter_std": 0.22264239192008972, + "reward_repeat_soft_mean": 0.9692996740341187, + "reward_repeat_soft_std": 0.016032908111810684, + "reward_std": 0.15563073754310608, + "reward_total_composite_mean": 0.384857714176178, + "reward_total_composite_std": 0.15563072264194489, + "reward_total_mean": 0.384857714176178, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 0.875, + "rewards/hard_gate/std": 0.3535533845424652, + "rewards/judge_quality/mean": 0.13375000655651093, + "rewards/judge_quality/std": 0.03543102368712425, + "rewards/meter/mean": 0.9100183248519897, + "rewards/meter/std": 0.22264239192008972, + "rewards/repeat_soft/mean": 0.9692996740341187, + "rewards/repeat_soft/std": 0.016032908111810684, + "rewards/total_composite/mean": 0.384857714176178, + "rewards/total_composite/std": 0.15563072264194489, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9984962344169617, + "sampling/importance_sampling_ratio/min": 0.013491343706846237, + "sampling/sampling_logp_difference/max": 4.305706977844238, + "sampling/sampling_logp_difference/mean": 0.13287843763828278, + "step": 2118 + }, + { + "clip_ratio/high_max": 0.04883924592286348, + "clip_ratio/high_mean": 0.04883924592286348, + "clip_ratio/low_mean": 0.03219923097640276, + "clip_ratio/low_min": 0.03219923097640276, + "clip_ratio/region_mean": 0.08103847689926624, + "completions/clipped_ratio": 0.0, + "completions/max_length": 92.0, + "completions/max_terminated_length": 92.0, + "completions/mean_length": 85.25, + "completions/mean_terminated_length": 85.25, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "entropy": 0.3776538223028183, + "epoch": 0.21285786037167254, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.920814514160156, + "learning_rate": 3.5818181818181817e-06, + "loss": 0.0006, + "num_tokens": 3810570.0, + "reward": 0.43916055560112, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.18000000715255737, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.9678657054901123, + "reward_meter_std": 0.02926114946603775, + "reward_repeat_soft_mean": 0.8394684195518494, + "reward_repeat_soft_std": 0.03335115313529968, + "reward_std": 0.0045523978769779205, + "reward_total_composite_mean": 0.43916055560112, + "reward_total_composite_std": 0.004552391357719898, + "reward_total_mean": 0.43916055560112, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.18000000715255737, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.9678657054901123, + "rewards/meter/std": 0.02926114946603775, + "rewards/repeat_soft/mean": 0.8394684195518494, + "rewards/repeat_soft/std": 0.03335115313529968, + "rewards/total_composite/mean": 0.43916055560112, + "rewards/total_composite/std": 0.004552391357719898, + "sampling/importance_sampling_ratio/max": 1.83241605758667, + "sampling/importance_sampling_ratio/mean": 1.002211093902588, + "sampling/importance_sampling_ratio/min": 0.2845260798931122, + "sampling/sampling_logp_difference/max": 1.2569303512573242, + "sampling/sampling_logp_difference/mean": 0.07062681764364243, + "step": 2119 + }, + { + "clip_ratio/high_max": 0.0718057572375983, + "clip_ratio/high_mean": 0.0718057572375983, + "clip_ratio/low_mean": 0.019736841320991516, + "clip_ratio/low_min": 0.019736841320991516, + "clip_ratio/region_mean": 0.09154259855858982, + "completions/clipped_ratio": 0.0, + "completions/max_length": 40.0, + "completions/max_terminated_length": 40.0, + "completions/mean_length": 37.25, + "completions/mean_terminated_length": 37.25, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "entropy": 0.375054020434618, + "epoch": 0.21295831240582622, + "frac_reward_zero_std": 0.0, + "grad_norm": 18.88593101501465, + "learning_rate": 3.578787878787879e-06, + "loss": 0.0594, + "num_tokens": 3812212.0, + "reward": 0.4270309805870056, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.8623172044754028, + "reward_meter_std": 0.2601478397846222, + "reward_repeat_soft_mean": 0.9530335664749146, + "reward_repeat_soft_std": 0.05007192865014076, + "reward_std": 0.023408222943544388, + "reward_total_composite_mean": 0.4270309805870056, + "reward_total_composite_std": 0.02340822108089924, + "reward_total_mean": 0.4270309805870056, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.8623172044754028, + "rewards/meter/std": 0.2601478397846222, + "rewards/repeat_soft/mean": 0.9530335664749146, + "rewards/repeat_soft/std": 0.05007192865014076, + "rewards/total_composite/mean": 0.4270309805870056, + "rewards/total_composite/std": 0.02340822108089924, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0077043771743774, + "sampling/importance_sampling_ratio/min": 0.13091398775577545, + "sampling/sampling_logp_difference/max": 2.033214807510376, + "sampling/sampling_logp_difference/mean": 0.09455607831478119, + "step": 2120 + }, + { + "clip_ratio/high_max": 0.02470588218420744, + "clip_ratio/high_mean": 0.02470588218420744, + "clip_ratio/low_mean": 0.03363671526312828, + "clip_ratio/low_min": 0.03363671526312828, + "clip_ratio/region_mean": 0.05834259744733572, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.0, + "completions/max_terminated_length": 34.0, + "completions/mean_length": 27.625, + "completions/mean_terminated_length": 27.625, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "entropy": 0.3139124680310488, + "epoch": 0.2130587644399799, + "frac_reward_zero_std": 0.0, + "grad_norm": 16.1114444732666, + "learning_rate": 3.575757575757576e-06, + "loss": -0.1005, + "num_tokens": 3813721.0, + "reward": 0.3796011209487915, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.34160304069519043, + "reward_meter_std": 0.3914759159088135, + "reward_repeat_soft_mean": 0.9752987623214722, + "reward_repeat_soft_std": 0.004002916160970926, + "reward_std": 0.03823775053024292, + "reward_total_composite_mean": 0.3796011209487915, + "reward_total_composite_std": 0.03823774307966232, + "reward_total_mean": 0.3796011209487915, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.34160304069519043, + "rewards/meter/std": 0.3914759159088135, + "rewards/repeat_soft/mean": 0.9752987623214722, + "rewards/repeat_soft/std": 0.004002916160970926, + "rewards/total_composite/mean": 0.3796011209487915, + "rewards/total_composite/std": 0.03823774307966232, + "sampling/importance_sampling_ratio/max": 1.8110864162445068, + "sampling/importance_sampling_ratio/mean": 0.9941287636756897, + "sampling/importance_sampling_ratio/min": 0.1346319317817688, + "sampling/sampling_logp_difference/max": 2.0052106380462646, + "sampling/sampling_logp_difference/mean": 0.08197850733995438, + "step": 2121 + }, + { + "clip_ratio/high_max": 0.12481671385467052, + "clip_ratio/high_mean": 0.12481671385467052, + "clip_ratio/low_mean": 0.06414473708719015, + "clip_ratio/low_min": 0.06414473708719015, + "clip_ratio/region_mean": 0.18896145094186068, + "completions/clipped_ratio": 0.0, + "completions/max_length": 38.0, + "completions/max_terminated_length": 38.0, + "completions/mean_length": 22.875, + "completions/mean_terminated_length": 22.875, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "entropy": 0.9253654107451439, + "epoch": 0.2131592164741336, + "frac_reward_zero_std": 0.0, + "grad_norm": 20.8836727142334, + "learning_rate": 3.5727272727272734e-06, + "loss": -0.0917, + "num_tokens": 3815112.0, + "reward": 0.4096284508705139, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.6766870021820068, + "reward_meter_std": 0.4345723092556, + "reward_repeat_soft_mean": 0.9576764702796936, + "reward_repeat_soft_std": 0.013642913661897182, + "reward_std": 0.04190043359994888, + "reward_total_composite_mean": 0.4096284508705139, + "reward_total_composite_std": 0.04190043732523918, + "reward_total_mean": 0.4096284508705139, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.6766870021820068, + "rewards/meter/std": 0.4345723092556, + "rewards/repeat_soft/mean": 0.9576764702796936, + "rewards/repeat_soft/std": 0.013642913661897182, + "rewards/total_composite/mean": 0.4096284508705139, + "rewards/total_composite/std": 0.04190043732523918, + "sampling/importance_sampling_ratio/max": 1.772874116897583, + "sampling/importance_sampling_ratio/mean": 0.9993340969085693, + "sampling/importance_sampling_ratio/min": 0.2402452528476715, + "sampling/sampling_logp_difference/max": 1.4260950088500977, + "sampling/sampling_logp_difference/mean": 0.16140659153461456, + "step": 2122 + }, + { + "clip_ratio/high_max": 0.046594975516200066, + "clip_ratio/high_mean": 0.046594975516200066, + "clip_ratio/low_mean": 0.0252380957826972, + "clip_ratio/low_min": 0.0252380957826972, + "clip_ratio/region_mean": 0.07183307129889727, + "completions/clipped_ratio": 0.0, + "completions/max_length": 60.0, + "completions/max_terminated_length": 60.0, + "completions/mean_length": 47.5, + "completions/mean_terminated_length": 47.5, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "entropy": 0.4308505542576313, + "epoch": 0.2132596685082873, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.174962043762207, + "learning_rate": 3.5696969696969703e-06, + "loss": 0.0178, + "num_tokens": 3816764.0, + "reward": 0.4190818667411804, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.8043650984764099, + "reward_meter_std": 0.3080444037914276, + "reward_repeat_soft_mean": 0.9377086758613586, + "reward_repeat_soft_std": 0.02444099448621273, + "reward_std": 0.02932891622185707, + "reward_total_composite_mean": 0.4190818667411804, + "reward_total_composite_std": 0.029328929260373116, + "reward_total_mean": 0.4190818667411804, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.8043650984764099, + "rewards/meter/std": 0.3080444037914276, + "rewards/repeat_soft/mean": 0.9377086758613586, + "rewards/repeat_soft/std": 0.02444099448621273, + "rewards/total_composite/mean": 0.4190818667411804, + "rewards/total_composite/std": 0.029328929260373116, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9955849647521973, + "sampling/importance_sampling_ratio/min": 0.16263586282730103, + "sampling/sampling_logp_difference/max": 1.8162415027618408, + "sampling/sampling_logp_difference/mean": 0.08279678970575333, + "step": 2123 + }, + { + "clip_ratio/high_max": 0.030169055331498384, + "clip_ratio/high_mean": 0.030169055331498384, + "clip_ratio/low_mean": 0.02626256551593542, + "clip_ratio/low_min": 0.02626256551593542, + "clip_ratio/region_mean": 0.056431620847433805, + "completions/clipped_ratio": 0.0, + "completions/max_length": 54.0, + "completions/max_terminated_length": 54.0, + "completions/mean_length": 51.125, + "completions/mean_terminated_length": 51.125, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.2968391440808773, + "epoch": 0.213360120542441, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.952363014221191, + "learning_rate": 3.566666666666667e-06, + "loss": 0.0328, + "num_tokens": 3818445.0, + "reward": 0.4213140606880188, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.14249999821186066, + "reward_judge_quality_std": 0.02121320739388466, + "reward_meter_mean": 0.9706482887268066, + "reward_meter_std": 0.039042141288518906, + "reward_repeat_soft_mean": 0.877032995223999, + "reward_repeat_soft_std": 0.07919418066740036, + "reward_std": 0.020526403561234474, + "reward_total_composite_mean": 0.4213140606880188, + "reward_total_composite_std": 0.02052641287446022, + "reward_total_mean": 0.4213140606880188, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.14249999821186066, + "rewards/judge_quality/std": 0.02121320739388466, + "rewards/meter/mean": 0.9706482887268066, + "rewards/meter/std": 0.039042141288518906, + "rewards/repeat_soft/mean": 0.877032995223999, + "rewards/repeat_soft/std": 0.07919418066740036, + "rewards/total_composite/mean": 0.4213140606880188, + "rewards/total_composite/std": 0.02052641287446022, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9943525791168213, + "sampling/importance_sampling_ratio/min": 0.3058168292045593, + "sampling/sampling_logp_difference/max": 1.197680950164795, + "sampling/sampling_logp_difference/mean": 0.07824968546628952, + "step": 2124 + }, + { + "clip_ratio/high_max": 0.07545540388673544, + "clip_ratio/high_mean": 0.07545540388673544, + "clip_ratio/low_mean": 0.022395833395421505, + "clip_ratio/low_min": 0.022395833395421505, + "clip_ratio/region_mean": 0.09785123728215694, + "completions/clipped_ratio": 0.0, + "completions/max_length": 128.0, + "completions/max_terminated_length": 128.0, + "completions/mean_length": 115.75, + "completions/mean_terminated_length": 115.75, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "entropy": 0.46189267933368683, + "epoch": 0.21346057257659468, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.96971321105957, + "learning_rate": 3.563636363636364e-06, + "loss": -0.0465, + "num_tokens": 3820811.0, + "reward": 0.3969489634037018, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.1237500011920929, + "reward_judge_quality_std": 0.010606604628264904, + "reward_meter_mean": 0.8100929260253906, + "reward_meter_std": 0.3253575265407562, + "reward_repeat_soft_mean": 0.8903893232345581, + "reward_repeat_soft_std": 0.017612675204873085, + "reward_std": 0.023812511935830116, + "reward_total_composite_mean": 0.3969489634037018, + "reward_total_composite_std": 0.023812510073184967, + "reward_total_mean": 0.3969489634037018, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.1237500011920929, + "rewards/judge_quality/std": 0.010606604628264904, + "rewards/meter/mean": 0.8100929260253906, + "rewards/meter/std": 0.3253575265407562, + "rewards/repeat_soft/mean": 0.8903893232345581, + "rewards/repeat_soft/std": 0.017612675204873085, + "rewards/total_composite/mean": 0.3969489634037018, + "rewards/total_composite/std": 0.023812510073184967, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0107030868530273, + "sampling/importance_sampling_ratio/min": 0.1517883539199829, + "sampling/sampling_logp_difference/max": 1.885268211364746, + "sampling/sampling_logp_difference/mean": 0.09560476988554001, + "step": 2125 + }, + { + "clip_ratio/high_max": 0.07084880443289876, + "clip_ratio/high_mean": 0.07084880443289876, + "clip_ratio/low_mean": 0.012500000186264515, + "clip_ratio/low_min": 0.012500000186264515, + "clip_ratio/region_mean": 0.08334880461916327, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 62.0, + "completions/mean_length": 110.25, + "completions/mean_terminated_length": 52.85714340209961, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "entropy": 0.5782650634646416, + "epoch": 0.21356102461074836, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.2829623222351074, + "learning_rate": 3.560606060606061e-06, + "loss": -0.1275, + "num_tokens": 3822445.0, + "reward": 0.39845943450927734, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 0.875, + "reward_hard_gate_std": 0.3535533845424652, + "reward_judge_quality_mean": 0.17875000834465027, + "reward_judge_quality_std": 0.11605878919363022, + "reward_meter_mean": 0.7273818254470825, + "reward_meter_std": 0.42327576875686646, + "reward_repeat_soft_mean": 0.9806386232376099, + "reward_repeat_soft_std": 0.020722202956676483, + "reward_std": 0.1762647181749344, + "reward_total_composite_mean": 0.39845943450927734, + "reward_total_composite_std": 0.17626473307609558, + "reward_total_mean": 0.39845943450927734, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 0.875, + "rewards/hard_gate/std": 0.3535533845424652, + "rewards/judge_quality/mean": 0.17875000834465027, + "rewards/judge_quality/std": 0.11605878919363022, + "rewards/meter/mean": 0.7273818254470825, + "rewards/meter/std": 0.42327576875686646, + "rewards/repeat_soft/mean": 0.9806386232376099, + "rewards/repeat_soft/std": 0.020722202956676483, + "rewards/total_composite/mean": 0.39845943450927734, + "rewards/total_composite/std": 0.17626473307609558, + "sampling/importance_sampling_ratio/max": 1.9448227882385254, + "sampling/importance_sampling_ratio/mean": 1.0091761350631714, + "sampling/importance_sampling_ratio/min": 0.029697038233280182, + "sampling/sampling_logp_difference/max": 3.5167078971862793, + "sampling/sampling_logp_difference/mean": 0.10056637227535248, + "step": 2126 + }, + { + "clip_ratio/high_max": 0.029656318947672844, + "clip_ratio/high_mean": 0.029656318947672844, + "clip_ratio/low_mean": 0.03404927044175565, + "clip_ratio/low_min": 0.03404927044175565, + "clip_ratio/region_mean": 0.0637055893894285, + "completions/clipped_ratio": 0.0, + "completions/max_length": 52.0, + "completions/max_terminated_length": 52.0, + "completions/mean_length": 41.75, + "completions/mean_terminated_length": 41.75, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "entropy": 0.5424296446144581, + "epoch": 0.21366147664490207, + "frac_reward_zero_std": 0.0, + "grad_norm": 10.888141632080078, + "learning_rate": 3.557575757575758e-06, + "loss": 0.0744, + "num_tokens": 3824123.0, + "reward": 0.48145878314971924, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.22500000894069672, + "reward_judge_quality_std": 0.13887301087379456, + "reward_meter_mean": 0.961769700050354, + "reward_meter_std": 0.04579383134841919, + "reward_repeat_soft_mean": 0.9323242902755737, + "reward_repeat_soft_std": 0.029701024293899536, + "reward_std": 0.08942936360836029, + "reward_total_composite_mean": 0.48145878314971924, + "reward_total_composite_std": 0.08942937850952148, + "reward_total_mean": 0.48145878314971924, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.22500000894069672, + "rewards/judge_quality/std": 0.13887301087379456, + "rewards/meter/mean": 0.961769700050354, + "rewards/meter/std": 0.04579383134841919, + "rewards/repeat_soft/mean": 0.9323242902755737, + "rewards/repeat_soft/std": 0.029701024293899536, + "rewards/total_composite/mean": 0.48145878314971924, + "rewards/total_composite/std": 0.08942937850952148, + "sampling/importance_sampling_ratio/max": 1.674881935119629, + "sampling/importance_sampling_ratio/mean": 0.9984186887741089, + "sampling/importance_sampling_ratio/min": 0.2478538304567337, + "sampling/sampling_logp_difference/max": 1.39491605758667, + "sampling/sampling_logp_difference/mean": 0.08970857411623001, + "step": 2127 + }, + { + "clip_ratio/high_max": 0.07017199601978064, + "clip_ratio/high_mean": 0.07017199601978064, + "clip_ratio/low_mean": 0.014423076994717121, + "clip_ratio/low_min": 0.014423076994717121, + "clip_ratio/region_mean": 0.08459507301449776, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 130.0, + "completions/mean_length": 161.875, + "completions/mean_terminated_length": 111.85714721679688, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.4321543797850609, + "epoch": 0.21376192867905575, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7553354501724243, + "learning_rate": 3.554545454545455e-06, + "loss": -0.1926, + "num_tokens": 3826618.0, + "reward": 0.3597673177719116, + "reward_count_adherence_mean": 0.96875, + "reward_count_adherence_std": 0.0883883461356163, + "reward_hard_gate_mean": 0.875, + "reward_hard_gate_std": 0.3535533845424652, + "reward_judge_quality_mean": 0.14875000715255737, + "reward_judge_quality_std": 0.048236772418022156, + "reward_meter_mean": 0.8127691745758057, + "reward_meter_std": 0.2406369149684906, + "reward_repeat_soft_mean": 0.8884314298629761, + "reward_repeat_soft_std": 0.041326019912958145, + "reward_std": 0.15010634064674377, + "reward_total_composite_mean": 0.3597673177719116, + "reward_total_composite_std": 0.15010634064674377, + "reward_total_mean": 0.3597673177719116, + "rewards/count_adherence/mean": 0.96875, + "rewards/count_adherence/std": 0.0883883461356163, + "rewards/hard_gate/mean": 0.875, + "rewards/hard_gate/std": 0.3535533845424652, + "rewards/judge_quality/mean": 0.14875000715255737, + "rewards/judge_quality/std": 0.048236772418022156, + "rewards/meter/mean": 0.8127691745758057, + "rewards/meter/std": 0.2406369149684906, + "rewards/repeat_soft/mean": 0.8884314298629761, + "rewards/repeat_soft/std": 0.041326019912958145, + "rewards/total_composite/mean": 0.3597673177719116, + "rewards/total_composite/std": 0.15010634064674377, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.004586100578308, + "sampling/importance_sampling_ratio/min": 0.2390352040529251, + "sampling/sampling_logp_difference/max": 1.4311444759368896, + "sampling/sampling_logp_difference/mean": 0.09040025621652603, + "step": 2128 + }, + { + "clip_ratio/high_max": 0.04454277083277702, + "clip_ratio/high_mean": 0.04454277083277702, + "clip_ratio/low_mean": 0.02442577015608549, + "clip_ratio/low_min": 0.02442577015608549, + "clip_ratio/region_mean": 0.06896854098886251, + "completions/clipped_ratio": 0.0, + "completions/max_length": 114.0, + "completions/max_terminated_length": 114.0, + "completions/mean_length": 98.25, + "completions/mean_terminated_length": 98.25, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.3356841690838337, + "epoch": 0.21386238071320945, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.841037273406982, + "learning_rate": 3.551515151515152e-06, + "loss": 0.065, + "num_tokens": 3828796.0, + "reward": 0.44421523809432983, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.17625001072883606, + "reward_judge_quality_std": 0.010606602765619755, + "reward_meter_mean": 0.9749183654785156, + "reward_meter_std": 0.01414470374584198, + "reward_repeat_soft_mean": 0.8836276531219482, + "reward_repeat_soft_std": 0.0258223544806242, + "reward_std": 0.007170422002673149, + "reward_total_composite_mean": 0.44421523809432983, + "reward_total_composite_std": 0.0071704210713505745, + "reward_total_mean": 0.44421523809432983, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.17625001072883606, + "rewards/judge_quality/std": 0.010606602765619755, + "rewards/meter/mean": 0.9749183654785156, + "rewards/meter/std": 0.01414470374584198, + "rewards/repeat_soft/mean": 0.8836276531219482, + "rewards/repeat_soft/std": 0.0258223544806242, + "rewards/total_composite/mean": 0.44421523809432983, + "rewards/total_composite/std": 0.0071704210713505745, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9967421293258667, + "sampling/importance_sampling_ratio/min": 0.02282397635281086, + "sampling/sampling_logp_difference/max": 3.7799437046051025, + "sampling/sampling_logp_difference/mean": 0.09535162150859833, + "step": 2129 + }, + { + "clip_ratio/high_max": 0.040749015752226114, + "clip_ratio/high_mean": 0.040749015752226114, + "clip_ratio/low_mean": 0.02232967270538211, + "clip_ratio/low_min": 0.02232967270538211, + "clip_ratio/region_mean": 0.06307868845760822, + "completions/clipped_ratio": 0.0, + "completions/max_length": 115.0, + "completions/max_terminated_length": 115.0, + "completions/mean_length": 102.625, + "completions/mean_terminated_length": 102.625, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.3542307838797569, + "epoch": 0.21396283274736314, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.538710594177246, + "learning_rate": 3.548484848484849e-06, + "loss": -0.0544, + "num_tokens": 3831009.0, + "reward": 0.39185476303100586, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.1274999976158142, + "reward_judge_quality_std": 0.02121320739388466, + "reward_meter_mean": 0.9733519554138184, + "reward_meter_std": 0.02541358955204487, + "reward_repeat_soft_mean": 0.740585446357727, + "reward_repeat_soft_std": 0.12306283414363861, + "reward_std": 0.02539207972586155, + "reward_total_composite_mean": 0.39185476303100586, + "reward_total_composite_std": 0.0253920815885067, + "reward_total_mean": 0.39185476303100586, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.1274999976158142, + "rewards/judge_quality/std": 0.02121320739388466, + "rewards/meter/mean": 0.9733519554138184, + "rewards/meter/std": 0.02541358955204487, + "rewards/repeat_soft/mean": 0.740585446357727, + "rewards/repeat_soft/std": 0.12306283414363861, + "rewards/total_composite/mean": 0.39185476303100586, + "rewards/total_composite/std": 0.0253920815885067, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9985268712043762, + "sampling/importance_sampling_ratio/min": 0.16990265250205994, + "sampling/sampling_logp_difference/max": 1.7725296020507812, + "sampling/sampling_logp_difference/mean": 0.07328380644321442, + "step": 2130 + }, + { + "clip_ratio/high_max": 0.04470324795693159, + "clip_ratio/high_mean": 0.04470324795693159, + "clip_ratio/low_mean": 0.05468147713690996, + "clip_ratio/low_min": 0.05468147713690996, + "clip_ratio/region_mean": 0.09938472509384155, + "completions/clipped_ratio": 0.0, + "completions/max_length": 153.0, + "completions/max_terminated_length": 153.0, + "completions/mean_length": 143.125, + "completions/mean_terminated_length": 143.125, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.5583962202072144, + "epoch": 0.21406328478151682, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.729617595672607, + "learning_rate": 3.5454545454545458e-06, + "loss": -0.0066, + "num_tokens": 3833642.0, + "reward": 0.42175352573394775, + "reward_count_adherence_mean": 0.949999988079071, + "reward_count_adherence_std": 0.09258200973272324, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.1537500023841858, + "reward_judge_quality_std": 0.029730942100286484, + "reward_meter_mean": 0.9710224270820618, + "reward_meter_std": 0.024256430566310883, + "reward_repeat_soft_mean": 0.8977535963058472, + "reward_repeat_soft_std": 0.03404621779918671, + "reward_std": 0.020088035613298416, + "reward_total_composite_mean": 0.42175352573394775, + "reward_total_composite_std": 0.02008803002536297, + "reward_total_mean": 0.42175352573394775, + "rewards/count_adherence/mean": 0.949999988079071, + "rewards/count_adherence/std": 0.09258200973272324, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.1537500023841858, + "rewards/judge_quality/std": 0.029730942100286484, + "rewards/meter/mean": 0.9710224270820618, + "rewards/meter/std": 0.024256430566310883, + "rewards/repeat_soft/mean": 0.8977535963058472, + "rewards/repeat_soft/std": 0.03404621779918671, + "rewards/total_composite/mean": 0.42175352573394775, + "rewards/total_composite/std": 0.02008803002536297, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9980050325393677, + "sampling/importance_sampling_ratio/min": 0.10387929528951645, + "sampling/sampling_logp_difference/max": 2.2645256519317627, + "sampling/sampling_logp_difference/mean": 0.11049627512693405, + "step": 2131 + }, + { + "clip_ratio/high_max": 0.060073030181229115, + "clip_ratio/high_mean": 0.060073030181229115, + "clip_ratio/low_mean": 0.02175925951451063, + "clip_ratio/low_min": 0.02175925951451063, + "clip_ratio/region_mean": 0.08183228969573975, + "completions/clipped_ratio": 0.0, + "completions/max_length": 28.0, + "completions/max_terminated_length": 28.0, + "completions/mean_length": 25.125, + "completions/mean_terminated_length": 25.125, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "entropy": 0.6499927863478661, + "epoch": 0.21416373681567052, + "frac_reward_zero_std": 0.0, + "grad_norm": 19.79250717163086, + "learning_rate": 3.5424242424242426e-06, + "loss": -0.04, + "num_tokens": 3835091.0, + "reward": 0.4367702603340149, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.9905524253845215, + "reward_meter_std": 0.008373869583010674, + "reward_repeat_soft_mean": 0.9346094131469727, + "reward_repeat_soft_std": 0.04883209988474846, + "reward_std": 0.007294599432498217, + "reward_total_composite_mean": 0.4367702603340149, + "reward_total_composite_std": 0.007294591516256332, + "reward_total_mean": 0.4367702603340149, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.9905524253845215, + "rewards/meter/std": 0.008373869583010674, + "rewards/repeat_soft/mean": 0.9346094131469727, + "rewards/repeat_soft/std": 0.04883209988474846, + "rewards/total_composite/mean": 0.4367702603340149, + "rewards/total_composite/std": 0.007294591516256332, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0061790943145752, + "sampling/importance_sampling_ratio/min": 0.3049562871456146, + "sampling/sampling_logp_difference/max": 1.2098863124847412, + "sampling/sampling_logp_difference/mean": 0.08623015135526657, + "step": 2132 + }, + { + "clip_ratio/high_max": 0.044679089449346066, + "clip_ratio/high_mean": 0.044679089449346066, + "clip_ratio/low_mean": 0.02003205195069313, + "clip_ratio/low_min": 0.02003205195069313, + "clip_ratio/region_mean": 0.0647111414000392, + "completions/clipped_ratio": 0.0, + "completions/max_length": 26.0, + "completions/max_terminated_length": 26.0, + "completions/mean_length": 23.125, + "completions/mean_terminated_length": 23.125, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "entropy": 0.3871112950146198, + "epoch": 0.2142641888498242, + "frac_reward_zero_std": 0.0, + "grad_norm": 10.58881950378418, + "learning_rate": 3.53939393939394e-06, + "loss": 0.0394, + "num_tokens": 3836604.0, + "reward": 0.4227369427680969, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.8037121295928955, + "reward_meter_std": 0.3496001064777374, + "reward_repeat_soft_mean": 0.9624999761581421, + "reward_repeat_soft_std": 0.0, + "reward_std": 0.03408601135015488, + "reward_total_composite_mean": 0.4227369427680969, + "reward_total_composite_std": 0.03408600762486458, + "reward_total_mean": 0.4227369427680969, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.8037121295928955, + "rewards/meter/std": 0.3496001064777374, + "rewards/repeat_soft/mean": 0.9624999761581421, + "rewards/repeat_soft/std": 0.0, + "rewards/total_composite/mean": 0.4227369427680969, + "rewards/total_composite/std": 0.03408600762486458, + "sampling/importance_sampling_ratio/max": 1.919601321220398, + "sampling/importance_sampling_ratio/mean": 1.0104715824127197, + "sampling/importance_sampling_ratio/min": 0.22084252536296844, + "sampling/sampling_logp_difference/max": 1.510305404663086, + "sampling/sampling_logp_difference/mean": 0.09222669899463654, + "step": 2133 + }, + { + "clip_ratio/high_max": 0.07275812653824687, + "clip_ratio/high_mean": 0.07275812653824687, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.07275812653824687, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 127.0, + "completions/mean_length": 163.25, + "completions/mean_terminated_length": 113.42857360839844, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "entropy": 0.4335934594273567, + "epoch": 0.2143646408839779, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5877206325531006, + "learning_rate": 3.5363636363636367e-06, + "loss": -0.2088, + "num_tokens": 3838974.0, + "reward": 0.39359205961227417, + "reward_count_adherence_mean": 0.90625, + "reward_count_adherence_std": 0.2651650309562683, + "reward_hard_gate_mean": 0.875, + "reward_hard_gate_std": 0.3535533845424652, + "reward_judge_quality_mean": 0.1600000113248825, + "reward_judge_quality_std": 0.04566962271928787, + "reward_meter_mean": 0.9713046550750732, + "reward_meter_std": 0.03033020719885826, + "reward_repeat_soft_mean": 0.9271901845932007, + "reward_repeat_soft_std": 0.020566994324326515, + "reward_std": 0.159275084733963, + "reward_total_composite_mean": 0.39359205961227417, + "reward_total_composite_std": 0.159275084733963, + "reward_total_mean": 0.39359205961227417, + "rewards/count_adherence/mean": 0.90625, + "rewards/count_adherence/std": 0.2651650309562683, + "rewards/hard_gate/mean": 0.875, + "rewards/hard_gate/std": 0.3535533845424652, + "rewards/judge_quality/mean": 0.1600000113248825, + "rewards/judge_quality/std": 0.04566962271928787, + "rewards/meter/mean": 0.9713046550750732, + "rewards/meter/std": 0.03033020719885826, + "rewards/repeat_soft/mean": 0.9271901845932007, + "rewards/repeat_soft/std": 0.020566994324326515, + "rewards/total_composite/mean": 0.39359205961227417, + "rewards/total_composite/std": 0.159275084733963, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9960200786590576, + "sampling/importance_sampling_ratio/min": 0.12460698932409286, + "sampling/sampling_logp_difference/max": 2.0825905799865723, + "sampling/sampling_logp_difference/mean": 0.08693575859069824, + "step": 2134 + }, + { + "clip_ratio/high_max": 0.08523908769711852, + "clip_ratio/high_mean": 0.08523908769711852, + "clip_ratio/low_mean": 0.007575757801532745, + "clip_ratio/low_min": 0.007575757801532745, + "clip_ratio/region_mean": 0.09281484549865127, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 39.0, + "completions/mean_length": 95.375, + "completions/mean_terminated_length": 35.85714340209961, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "entropy": 0.30892448872327805, + "epoch": 0.2144650929181316, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.3888208866119385, + "learning_rate": 3.5333333333333335e-06, + "loss": -0.1097, + "num_tokens": 3840585.0, + "reward": 0.39515113830566406, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 0.875, + "reward_hard_gate_std": 0.3535533845424652, + "reward_judge_quality_mean": 0.17500001192092896, + "reward_judge_quality_std": 0.11649646610021591, + "reward_meter_mean": 0.7480518817901611, + "reward_meter_std": 0.29896217584609985, + "reward_repeat_soft_mean": 0.9610674381256104, + "reward_repeat_soft_std": 0.01786000281572342, + "reward_std": 0.17731983959674835, + "reward_total_composite_mean": 0.39515113830566406, + "reward_total_composite_std": 0.17731983959674835, + "reward_total_mean": 0.39515113830566406, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 0.875, + "rewards/hard_gate/std": 0.3535533845424652, + "rewards/judge_quality/mean": 0.17500001192092896, + "rewards/judge_quality/std": 0.11649646610021591, + "rewards/meter/mean": 0.7480518817901611, + "rewards/meter/std": 0.29896217584609985, + "rewards/repeat_soft/mean": 0.9610674381256104, + "rewards/repeat_soft/std": 0.01786000281572342, + "rewards/total_composite/mean": 0.39515113830566406, + "rewards/total_composite/std": 0.17731983959674835, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.978323221206665, + "sampling/importance_sampling_ratio/min": 0.35283517837524414, + "sampling/sampling_logp_difference/max": 1.0417542457580566, + "sampling/sampling_logp_difference/mean": 0.08909184485673904, + "step": 2135 + }, + { + "clip_ratio/high_max": 0.10606145393103361, + "clip_ratio/high_mean": 0.10606145393103361, + "clip_ratio/low_mean": 0.018518518656492233, + "clip_ratio/low_min": 0.018518518656492233, + "clip_ratio/region_mean": 0.12457997258752584, + "completions/clipped_ratio": 0.0, + "completions/max_length": 36.0, + "completions/max_terminated_length": 36.0, + "completions/mean_length": 29.0, + "completions/mean_terminated_length": 29.0, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "entropy": 0.6100064590573311, + "epoch": 0.21456554495228528, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.88821792602539, + "learning_rate": 3.5303030303030304e-06, + "loss": 0.0455, + "num_tokens": 3842041.0, + "reward": 0.42245009541511536, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.8173210024833679, + "reward_meter_std": 0.33915388584136963, + "reward_repeat_soft_mean": 0.9517419338226318, + "reward_repeat_soft_std": 0.020060529932379723, + "reward_std": 0.032370127737522125, + "reward_total_composite_mean": 0.42245009541511536, + "reward_total_composite_std": 0.032370131462812424, + "reward_total_mean": 0.42245009541511536, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.8173210024833679, + "rewards/meter/std": 0.33915388584136963, + "rewards/repeat_soft/mean": 0.9517419338226318, + "rewards/repeat_soft/std": 0.020060529932379723, + "rewards/total_composite/mean": 0.42245009541511536, + "rewards/total_composite/std": 0.032370131462812424, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9857088923454285, + "sampling/importance_sampling_ratio/min": 0.094399593770504, + "sampling/sampling_logp_difference/max": 2.3602185249328613, + "sampling/sampling_logp_difference/mean": 0.13327038288116455, + "step": 2136 + }, + { + "clip_ratio/high_max": 0.10881273122504354, + "clip_ratio/high_mean": 0.10881273122504354, + "clip_ratio/low_mean": 0.03321759309619665, + "clip_ratio/low_min": 0.03321759309619665, + "clip_ratio/region_mean": 0.1420303243212402, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.0, + "completions/max_terminated_length": 27.0, + "completions/mean_length": 21.5, + "completions/mean_terminated_length": 21.5, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "entropy": 0.930234007537365, + "epoch": 0.21466599698643898, + "frac_reward_zero_std": 0.0, + "grad_norm": 24.174345016479492, + "learning_rate": 3.5272727272727276e-06, + "loss": 0.0787, + "num_tokens": 3843421.0, + "reward": 0.40339311957359314, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.24625001847743988, + "reward_judge_quality_std": 0.2722361087799072, + "reward_meter_mean": 0.6113148927688599, + "reward_meter_std": 0.4718313217163086, + "reward_repeat_soft_mean": 0.9569318294525146, + "reward_repeat_soft_std": 0.015749182552099228, + "reward_std": 0.0453665629029274, + "reward_total_composite_mean": 0.40339311957359314, + "reward_total_composite_std": 0.0453665629029274, + "reward_total_mean": 0.40339311957359314, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.24625001847743988, + "rewards/judge_quality/std": 0.2722361087799072, + "rewards/meter/mean": 0.6113148927688599, + "rewards/meter/std": 0.4718313217163086, + "rewards/repeat_soft/mean": 0.9569318294525146, + "rewards/repeat_soft/std": 0.015749182552099228, + "rewards/total_composite/mean": 0.40339311957359314, + "rewards/total_composite/std": 0.0453665629029274, + "sampling/importance_sampling_ratio/max": 1.6885017156600952, + "sampling/importance_sampling_ratio/mean": 0.9756884574890137, + "sampling/importance_sampling_ratio/min": 0.32185009121894836, + "sampling/sampling_logp_difference/max": 1.133669376373291, + "sampling/sampling_logp_difference/mean": 0.14792943000793457, + "step": 2137 + }, + { + "clip_ratio/high_max": 0.057091874070465565, + "clip_ratio/high_mean": 0.057091874070465565, + "clip_ratio/low_mean": 0.03886368125677109, + "clip_ratio/low_min": 0.03886368125677109, + "clip_ratio/region_mean": 0.09595555532723665, + "completions/clipped_ratio": 0.0, + "completions/max_length": 134.0, + "completions/max_terminated_length": 134.0, + "completions/mean_length": 118.5, + "completions/mean_terminated_length": 118.5, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "entropy": 0.43669046834111214, + "epoch": 0.21476644902059266, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.568856716156006, + "learning_rate": 3.5242424242424244e-06, + "loss": 0.009, + "num_tokens": 3845705.0, + "reward": 0.43214356899261475, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.17250001430511475, + "reward_judge_quality_std": 0.013887302950024605, + "reward_meter_mean": 0.8722131848335266, + "reward_meter_std": 0.14797870814800262, + "reward_repeat_soft_mean": 0.8938058614730835, + "reward_repeat_soft_std": 0.03232758864760399, + "reward_std": 0.019955281168222427, + "reward_total_composite_mean": 0.43214356899261475, + "reward_total_composite_std": 0.019955281168222427, + "reward_total_mean": 0.43214356899261475, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.17250001430511475, + "rewards/judge_quality/std": 0.013887302950024605, + "rewards/meter/mean": 0.8722131848335266, + "rewards/meter/std": 0.14797870814800262, + "rewards/repeat_soft/mean": 0.8938058614730835, + "rewards/repeat_soft/std": 0.03232758864760399, + "rewards/total_composite/mean": 0.43214356899261475, + "rewards/total_composite/std": 0.019955281168222427, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0000040531158447, + "sampling/importance_sampling_ratio/min": 0.08528824895620346, + "sampling/sampling_logp_difference/max": 2.4617185592651367, + "sampling/sampling_logp_difference/mean": 0.10581151396036148, + "step": 2138 + }, + { + "clip_ratio/high_max": 0.016964286100119352, + "clip_ratio/high_mean": 0.016964286100119352, + "clip_ratio/low_mean": 0.05222701234742999, + "clip_ratio/low_min": 0.05222701234742999, + "clip_ratio/region_mean": 0.06919129844754934, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.0, + "completions/max_terminated_length": 30.0, + "completions/mean_length": 27.75, + "completions/mean_terminated_length": 27.75, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "entropy": 0.5129394680261612, + "epoch": 0.21486690105474635, + "frac_reward_zero_std": 0.0, + "grad_norm": 13.603361129760742, + "learning_rate": 3.5212121212121213e-06, + "loss": -0.0343, + "num_tokens": 3847031.0, + "reward": 0.4358510673046112, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.9820600748062134, + "reward_meter_std": 0.023328015580773354, + "reward_repeat_soft_mean": 0.9340013265609741, + "reward_repeat_soft_std": 0.030499953776597977, + "reward_std": 0.005299215670675039, + "reward_total_composite_mean": 0.4358510673046112, + "reward_total_composite_std": 0.0052992128767073154, + "reward_total_mean": 0.4358510673046112, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.9820600748062134, + "rewards/meter/std": 0.023328015580773354, + "rewards/repeat_soft/mean": 0.9340013265609741, + "rewards/repeat_soft/std": 0.030499953776597977, + "rewards/total_composite/mean": 0.4358510673046112, + "rewards/total_composite/std": 0.0052992128767073154, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0212786197662354, + "sampling/importance_sampling_ratio/min": 0.36647212505340576, + "sampling/sampling_logp_difference/max": 1.0038328170776367, + "sampling/sampling_logp_difference/mean": 0.0838530957698822, + "step": 2139 + }, + { + "clip_ratio/high_max": 0.061627304181456566, + "clip_ratio/high_mean": 0.061627304181456566, + "clip_ratio/low_mean": 0.028292838484048843, + "clip_ratio/low_min": 0.028292838484048843, + "clip_ratio/region_mean": 0.08992014266550541, + "completions/clipped_ratio": 0.0, + "completions/max_length": 46.0, + "completions/max_terminated_length": 46.0, + "completions/mean_length": 34.25, + "completions/mean_terminated_length": 34.25, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "entropy": 0.3702515698969364, + "epoch": 0.21496735308890005, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.767311096191406, + "learning_rate": 3.5181818181818185e-06, + "loss": 0.0841, + "num_tokens": 3848601.0, + "reward": 0.4237838089466095, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.0, + "reward_meter_mean": 0.8089805841445923, + "reward_meter_std": 0.216323122382164, + "reward_repeat_soft_mean": 0.9660546183586121, + "reward_repeat_soft_std": 0.015586726367473602, + "reward_std": 0.020835043862462044, + "reward_total_composite_mean": 0.4237838089466095, + "reward_total_composite_std": 0.02083503268659115, + "reward_total_mean": 0.4237838089466095, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.0, + "rewards/meter/mean": 0.8089805841445923, + "rewards/meter/std": 0.216323122382164, + "rewards/repeat_soft/mean": 0.9660546183586121, + "rewards/repeat_soft/std": 0.015586726367473602, + "rewards/total_composite/mean": 0.4237838089466095, + "rewards/total_composite/std": 0.02083503268659115, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0170397758483887, + "sampling/importance_sampling_ratio/min": 0.13060487806797028, + "sampling/sampling_logp_difference/max": 2.035578727722168, + "sampling/sampling_logp_difference/mean": 0.08979345858097076, + "step": 2140 + }, + { + "clip_ratio/high_max": 0.021226415410637856, + "clip_ratio/high_mean": 0.021226415410637856, + "clip_ratio/low_mean": 0.09280007239431143, + "clip_ratio/low_min": 0.09280007239431143, + "clip_ratio/region_mean": 0.11402648780494928, + "completions/clipped_ratio": 0.0, + "completions/max_length": 58.0, + "completions/max_terminated_length": 58.0, + "completions/mean_length": 52.875, + "completions/mean_terminated_length": 52.875, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "entropy": 0.5642917603254318, + "epoch": 0.21506780512305373, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.464240550994873, + "learning_rate": 3.5151515151515154e-06, + "loss": 0.0321, + "num_tokens": 3850272.0, + "reward": 0.43157362937927246, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.1537500023841858, + "reward_judge_quality_std": 0.010606602765619755, + "reward_meter_mean": 0.9091874361038208, + "reward_meter_std": 0.1122494712471962, + "reward_repeat_soft_mean": 0.9380810260772705, + "reward_repeat_soft_std": 0.0403120182454586, + "reward_std": 0.010991537012159824, + "reward_total_composite_mean": 0.43157362937927246, + "reward_total_composite_std": 0.010991547256708145, + "reward_total_mean": 0.43157362937927246, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.1537500023841858, + "rewards/judge_quality/std": 0.010606602765619755, + "rewards/meter/mean": 0.9091874361038208, + "rewards/meter/std": 0.1122494712471962, + "rewards/repeat_soft/mean": 0.9380810260772705, + "rewards/repeat_soft/std": 0.0403120182454586, + "rewards/total_composite/mean": 0.43157362937927246, + "rewards/total_composite/std": 0.010991547256708145, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0083507299423218, + "sampling/importance_sampling_ratio/min": 0.23833072185516357, + "sampling/sampling_logp_difference/max": 1.4340959787368774, + "sampling/sampling_logp_difference/mean": 0.09349511563777924, + "step": 2141 + }, + { + "clip_ratio/high_max": 0.058138023130595684, + "clip_ratio/high_mean": 0.058138023130595684, + "clip_ratio/low_mean": 0.053842433262616396, + "clip_ratio/low_min": 0.053842433262616396, + "clip_ratio/region_mean": 0.11198045639321208, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/max_terminated_length": 145.0, + "completions/mean_length": 100.75, + "completions/mean_terminated_length": 100.75, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "entropy": 0.5177117586135864, + "epoch": 0.21516825715720744, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.290336608886719, + "learning_rate": 3.512121212121212e-06, + "loss": -0.0837, + "num_tokens": 3852342.0, + "reward": 0.42709919810295105, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.21000000834465027, + "reward_judge_quality_std": 0.08485280722379684, + "reward_meter_mean": 0.6486868858337402, + "reward_meter_std": 0.2888045907020569, + "reward_repeat_soft_mean": 0.9150484800338745, + "reward_repeat_soft_std": 0.02585035189986229, + "reward_std": 0.05602974444627762, + "reward_total_composite_mean": 0.42709919810295105, + "reward_total_composite_std": 0.05602974444627762, + "reward_total_mean": 0.42709919810295105, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.21000000834465027, + "rewards/judge_quality/std": 0.08485280722379684, + "rewards/meter/mean": 0.6486868858337402, + "rewards/meter/std": 0.2888045907020569, + "rewards/repeat_soft/mean": 0.9150484800338745, + "rewards/repeat_soft/std": 0.02585035189986229, + "rewards/total_composite/mean": 0.42709919810295105, + "rewards/total_composite/std": 0.05602974444627762, + "sampling/importance_sampling_ratio/max": 1.9052740335464478, + "sampling/importance_sampling_ratio/mean": 1.0121235847473145, + "sampling/importance_sampling_ratio/min": 0.14424732327461243, + "sampling/sampling_logp_difference/max": 1.9362258911132812, + "sampling/sampling_logp_difference/mean": 0.0966973677277565, + "step": 2142 + }, + { + "clip_ratio/high_max": 0.046734149334952235, + "clip_ratio/high_mean": 0.046734149334952235, + "clip_ratio/low_mean": 0.04185404907912016, + "clip_ratio/low_min": 0.04185404907912016, + "clip_ratio/region_mean": 0.0885881984140724, + "completions/clipped_ratio": 0.0, + "completions/max_length": 62.0, + "completions/max_terminated_length": 62.0, + "completions/mean_length": 53.0, + "completions/mean_terminated_length": 53.0, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "entropy": 0.44840671494603157, + "epoch": 0.21526870919136112, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.681342124938965, + "learning_rate": 3.509090909090909e-06, + "loss": 0.0574, + "num_tokens": 3854286.0, + "reward": 0.4429652690887451, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.16500000655651093, + "reward_judge_quality_std": 0.01603567600250244, + "reward_meter_mean": 0.9523389339447021, + "reward_meter_std": 0.03752657771110535, + "reward_repeat_soft_mean": 0.9398510456085205, + "reward_repeat_soft_std": 0.030083410441875458, + "reward_std": 0.008120759390294552, + "reward_total_composite_mean": 0.4429652690887451, + "reward_total_composite_std": 0.008120769634842873, + "reward_total_mean": 0.4429652690887451, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.16500000655651093, + "rewards/judge_quality/std": 0.01603567600250244, + "rewards/meter/mean": 0.9523389339447021, + "rewards/meter/std": 0.03752657771110535, + "rewards/repeat_soft/mean": 0.9398510456085205, + "rewards/repeat_soft/std": 0.030083410441875458, + "rewards/total_composite/mean": 0.4429652690887451, + "rewards/total_composite/std": 0.008120769634842873, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0037641525268555, + "sampling/importance_sampling_ratio/min": 0.0958370566368103, + "sampling/sampling_logp_difference/max": 2.3451058864593506, + "sampling/sampling_logp_difference/mean": 0.09259400516748428, + "step": 2143 + }, + { + "clip_ratio/high_max": 0.08712796587496996, + "clip_ratio/high_mean": 0.08712796587496996, + "clip_ratio/low_mean": 0.023078530095517635, + "clip_ratio/low_min": 0.023078530095517635, + "clip_ratio/region_mean": 0.1102064959704876, + "completions/clipped_ratio": 0.0, + "completions/max_length": 88.0, + "completions/max_terminated_length": 88.0, + "completions/mean_length": 78.625, + "completions/mean_terminated_length": 78.625, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "entropy": 0.5412853918969631, + "epoch": 0.2153691612255148, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.980925559997559, + "learning_rate": 3.5060606060606063e-06, + "loss": -0.0546, + "num_tokens": 3856427.0, + "reward": 0.4235439896583557, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.16124999523162842, + "reward_judge_quality_std": 0.015526476316154003, + "reward_meter_mean": 0.8787567019462585, + "reward_meter_std": 0.20320703089237213, + "reward_repeat_soft_mean": 0.8748688697814941, + "reward_repeat_soft_std": 0.019511040300130844, + "reward_std": 0.0233326256275177, + "reward_total_composite_mean": 0.4235439896583557, + "reward_total_composite_std": 0.02333262749016285, + "reward_total_mean": 0.4235439896583557, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.16124999523162842, + "rewards/judge_quality/std": 0.015526476316154003, + "rewards/meter/mean": 0.8787567019462585, + "rewards/meter/std": 0.20320703089237213, + "rewards/repeat_soft/mean": 0.8748688697814941, + "rewards/repeat_soft/std": 0.019511040300130844, + "rewards/total_composite/mean": 0.4235439896583557, + "rewards/total_composite/std": 0.02333262749016285, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9930413365364075, + "sampling/importance_sampling_ratio/min": 0.1639026403427124, + "sampling/sampling_logp_difference/max": 1.8084826469421387, + "sampling/sampling_logp_difference/mean": 0.12014502286911011, + "step": 2144 + }, + { + "clip_ratio/high_max": 0.076064292807132, + "clip_ratio/high_mean": 0.076064292807132, + "clip_ratio/low_mean": 0.021739130839705467, + "clip_ratio/low_min": 0.021739130839705467, + "clip_ratio/region_mean": 0.09780342364683747, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/max_terminated_length": 31.0, + "completions/mean_length": 86.375, + "completions/mean_terminated_length": 25.571430206298828, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "entropy": 0.8141542971134186, + "epoch": 0.2154696132596685, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.2406935691833496, + "learning_rate": 3.503030303030303e-06, + "loss": -0.0907, + "num_tokens": 3857950.0, + "reward": 0.36311087012290955, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 0.875, + "reward_hard_gate_std": 0.3535533845424652, + "reward_judge_quality_mean": 0.13750000298023224, + "reward_judge_quality_std": 0.0353553406894207, + "reward_meter_mean": 0.8375157713890076, + "reward_meter_std": 0.3385445475578308, + "reward_repeat_soft_mean": 0.8941828608512878, + "reward_repeat_soft_std": 0.11264170706272125, + "reward_std": 0.15466636419296265, + "reward_total_composite_mean": 0.36311087012290955, + "reward_total_composite_std": 0.15466636419296265, + "reward_total_mean": 0.36311087012290955, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 0.875, + "rewards/hard_gate/std": 0.3535533845424652, + "rewards/judge_quality/mean": 0.13750000298023224, + "rewards/judge_quality/std": 0.0353553406894207, + "rewards/meter/mean": 0.8375157713890076, + "rewards/meter/std": 0.3385445475578308, + "rewards/repeat_soft/mean": 0.8941828608512878, + "rewards/repeat_soft/std": 0.11264170706272125, + "rewards/total_composite/mean": 0.36311087012290955, + "rewards/total_composite/std": 0.15466636419296265, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.052787184715271, + "sampling/importance_sampling_ratio/min": 0.3893972635269165, + "sampling/sampling_logp_difference/max": 0.9431552886962891, + "sampling/sampling_logp_difference/mean": 0.11213835328817368, + "step": 2145 + }, + { + "clip_ratio/high_max": 0.048446863889694214, + "clip_ratio/high_mean": 0.048446863889694214, + "clip_ratio/low_mean": 0.03247932391241193, + "clip_ratio/low_min": 0.03247932391241193, + "clip_ratio/region_mean": 0.08092618780210614, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.0, + "completions/max_terminated_length": 229.0, + "completions/mean_length": 201.375, + "completions/mean_terminated_length": 201.375, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.4255397655069828, + "epoch": 0.2155700652938222, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.35801362991333, + "learning_rate": 3.5e-06, + "loss": 0.0478, + "num_tokens": 3860945.0, + "reward": 0.365647554397583, + "reward_count_adherence_mean": 0.8125, + "reward_count_adherence_std": 0.05892555043101311, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.13500000536441803, + "reward_judge_quality_std": 0.02777460776269436, + "reward_meter_mean": 0.9370740652084351, + "reward_meter_std": 0.06427296251058578, + "reward_repeat_soft_mean": 0.8033504486083984, + "reward_repeat_soft_std": 0.07749520242214203, + "reward_std": 0.03421282768249512, + "reward_total_composite_mean": 0.365647554397583, + "reward_total_composite_std": 0.034212831407785416, + "reward_total_mean": 0.365647554397583, + "rewards/count_adherence/mean": 0.8125, + "rewards/count_adherence/std": 0.05892555043101311, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.13500000536441803, + "rewards/judge_quality/std": 0.02777460776269436, + "rewards/meter/mean": 0.9370740652084351, + "rewards/meter/std": 0.06427296251058578, + "rewards/repeat_soft/mean": 0.8033504486083984, + "rewards/repeat_soft/std": 0.07749520242214203, + "rewards/total_composite/mean": 0.365647554397583, + "rewards/total_composite/std": 0.034212831407785416, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.001274585723877, + "sampling/importance_sampling_ratio/min": 0.03523445874452591, + "sampling/sampling_logp_difference/max": 3.345730781555176, + "sampling/sampling_logp_difference/mean": 0.08685857802629471, + "step": 2146 + }, + { + "clip_ratio/high_max": 0.08540754113346338, + "clip_ratio/high_mean": 0.08540754113346338, + "clip_ratio/low_mean": 0.014354674611240625, + "clip_ratio/low_min": 0.014354674611240625, + "clip_ratio/region_mean": 0.09976221574470401, + "completions/clipped_ratio": 0.0, + "completions/max_length": 51.0, + "completions/max_terminated_length": 51.0, + "completions/mean_length": 45.75, + "completions/mean_terminated_length": 45.75, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "entropy": 0.44552335888147354, + "epoch": 0.2156705173279759, + "frac_reward_zero_std": 0.0, + "grad_norm": 13.747391700744629, + "learning_rate": 3.496969696969697e-06, + "loss": -0.009, + "num_tokens": 3862703.0, + "reward": 0.4283779263496399, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.14625000953674316, + "reward_judge_quality_std": 0.010606604628264904, + "reward_meter_mean": 0.8913014531135559, + "reward_meter_std": 0.22273597121238708, + "reward_repeat_soft_mean": 0.9591418504714966, + "reward_repeat_soft_std": 0.035527247935533524, + "reward_std": 0.02429075725376606, + "reward_total_composite_mean": 0.4283779263496399, + "reward_total_composite_std": 0.024290764704346657, + "reward_total_mean": 0.4283779263496399, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.14625000953674316, + "rewards/judge_quality/std": 0.010606604628264904, + "rewards/meter/mean": 0.8913014531135559, + "rewards/meter/std": 0.22273597121238708, + "rewards/repeat_soft/mean": 0.9591418504714966, + "rewards/repeat_soft/std": 0.035527247935533524, + "rewards/total_composite/mean": 0.4283779263496399, + "rewards/total_composite/std": 0.024290764704346657, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0007129907608032, + "sampling/importance_sampling_ratio/min": 0.14243902266025543, + "sampling/sampling_logp_difference/max": 1.9488413333892822, + "sampling/sampling_logp_difference/mean": 0.09208957105875015, + "step": 2147 + }, + { + "clip_ratio/high_max": 0.06361279031261802, + "clip_ratio/high_mean": 0.06361279031261802, + "clip_ratio/low_mean": 0.018656716216355562, + "clip_ratio/low_min": 0.018656716216355562, + "clip_ratio/region_mean": 0.08226950652897358, + "completions/clipped_ratio": 0.0, + "completions/max_length": 88.0, + "completions/max_terminated_length": 88.0, + "completions/mean_length": 69.875, + "completions/mean_terminated_length": 69.875, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "entropy": 0.5813146941363811, + "epoch": 0.21577096936212958, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.051156997680664, + "learning_rate": 3.493939393939394e-06, + "loss": -0.0048, + "num_tokens": 3864702.0, + "reward": 0.37210574746131897, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 0.875, + "reward_hard_gate_std": 0.3535533845424652, + "reward_judge_quality_mean": 0.1537500023841858, + "reward_judge_quality_std": 0.01922610215842724, + "reward_meter_mean": 0.8386724591255188, + "reward_meter_std": 0.2629724144935608, + "reward_repeat_soft_mean": 0.9236710071563721, + "reward_repeat_soft_std": 0.02468961849808693, + "reward_std": 0.1534768044948578, + "reward_total_composite_mean": 0.37210574746131897, + "reward_total_composite_std": 0.15347681939601898, + "reward_total_mean": 0.37210574746131897, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 0.875, + "rewards/hard_gate/std": 0.3535533845424652, + "rewards/judge_quality/mean": 0.1537500023841858, + "rewards/judge_quality/std": 0.01922610215842724, + "rewards/meter/mean": 0.8386724591255188, + "rewards/meter/std": 0.2629724144935608, + "rewards/repeat_soft/mean": 0.9236710071563721, + "rewards/repeat_soft/std": 0.02468961849808693, + "rewards/total_composite/mean": 0.37210574746131897, + "rewards/total_composite/std": 0.15347681939601898, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0113861560821533, + "sampling/importance_sampling_ratio/min": 0.09408576041460037, + "sampling/sampling_logp_difference/max": 2.363548517227173, + "sampling/sampling_logp_difference/mean": 0.10458245873451233, + "step": 2148 + }, + { + "clip_ratio/high_max": 0.07705757208168507, + "clip_ratio/high_mean": 0.07705757208168507, + "clip_ratio/low_mean": 0.0335365841165185, + "clip_ratio/low_min": 0.0335365841165185, + "clip_ratio/region_mean": 0.11059415619820356, + "completions/clipped_ratio": 0.0, + "completions/max_length": 50.0, + "completions/max_terminated_length": 50.0, + "completions/mean_length": 42.625, + "completions/mean_terminated_length": 42.625, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "entropy": 0.5505771040916443, + "epoch": 0.21587142139628326, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.868727684020996, + "learning_rate": 3.4909090909090913e-06, + "loss": 0.0132, + "num_tokens": 3866315.0, + "reward": 0.4373164772987366, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.1537500023841858, + "reward_judge_quality_std": 0.010606602765619755, + "reward_meter_mean": 0.9211106896400452, + "reward_meter_std": 0.10339754819869995, + "reward_repeat_soft_mean": 0.9673133492469788, + "reward_repeat_soft_std": 0.019251564517617226, + "reward_std": 0.012059218250215054, + "reward_total_composite_mean": 0.4373164772987366, + "reward_total_composite_std": 0.012059218250215054, + "reward_total_mean": 0.4373164772987366, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.1537500023841858, + "rewards/judge_quality/std": 0.010606602765619755, + "rewards/meter/mean": 0.9211106896400452, + "rewards/meter/std": 0.10339754819869995, + "rewards/repeat_soft/mean": 0.9673133492469788, + "rewards/repeat_soft/std": 0.019251564517617226, + "rewards/total_composite/mean": 0.4373164772987366, + "rewards/total_composite/std": 0.012059218250215054, + "sampling/importance_sampling_ratio/max": 1.8737012147903442, + "sampling/importance_sampling_ratio/mean": 0.987741231918335, + "sampling/importance_sampling_ratio/min": 0.14948733150959015, + "sampling/sampling_logp_difference/max": 1.9005436897277832, + "sampling/sampling_logp_difference/mean": 0.11840193718671799, + "step": 2149 + }, + { + "clip_ratio/high_max": 0.05818670894950628, + "clip_ratio/high_mean": 0.05818670894950628, + "clip_ratio/low_mean": 0.03481538034975529, + "clip_ratio/low_min": 0.03481538034975529, + "clip_ratio/region_mean": 0.09300208929926157, + "completions/clipped_ratio": 0.0, + "completions/max_length": 57.0, + "completions/max_terminated_length": 57.0, + "completions/mean_length": 52.25, + "completions/mean_terminated_length": 52.25, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "entropy": 0.589105699211359, + "epoch": 0.21597187343043697, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.236807823181152, + "learning_rate": 3.4878787878787885e-06, + "loss": -0.0127, + "num_tokens": 3868253.0, + "reward": 0.4297049641609192, + "reward_count_adherence_mean": 1.0, + "reward_count_adherence_std": 0.0, + "reward_hard_gate_mean": 1.0, + "reward_hard_gate_std": 0.0, + "reward_judge_quality_mean": 0.15000000596046448, + "reward_judge_quality_std": 0.01603567786514759, + "reward_meter_mean": 0.8498669862747192, + "reward_meter_std": 0.14511364698410034, + "reward_repeat_soft_mean": 0.9805920124053955, + "reward_repeat_soft_std": 0.020208818838000298, + "reward_std": 0.013732440769672394, + "reward_total_composite_mean": 0.4297049641609192, + "reward_total_composite_std": 0.013732438907027245, + "reward_total_mean": 0.4297049641609192, + "rewards/count_adherence/mean": 1.0, + "rewards/count_adherence/std": 0.0, + "rewards/hard_gate/mean": 1.0, + "rewards/hard_gate/std": 0.0, + "rewards/judge_quality/mean": 0.15000000596046448, + "rewards/judge_quality/std": 0.01603567786514759, + "rewards/meter/mean": 0.8498669862747192, + "rewards/meter/std": 0.14511364698410034, + "rewards/repeat_soft/mean": 0.9805920124053955, + "rewards/repeat_soft/std": 0.020208818838000298, + "rewards/total_composite/mean": 0.4297049641609192, + "rewards/total_composite/std": 0.013732438907027245, + "sampling/importance_sampling_ratio/max": 1.9407857656478882, + "sampling/importance_sampling_ratio/mean": 0.9951035976409912, + "sampling/importance_sampling_ratio/min": 0.12261290848255157, + "sampling/sampling_logp_difference/max": 2.0987229347229004, + "sampling/sampling_logp_difference/mean": 0.11804606765508652, + "step": 2150 + }, + { + "epoch": 0.21597187343043697, + "eval_clip_ratio/high_max": 0.0, + "eval_clip_ratio/high_mean": 0.0, + "eval_clip_ratio/low_mean": 0.0, + "eval_clip_ratio/low_min": 0.0, + "eval_clip_ratio/region_mean": 0.0, + "eval_completions/clipped_ratio": 0.0125, + "eval_completions/max_length": 175.2, + "eval_completions/max_terminated_length": 139.9, + "eval_completions/mean_length": 84.7, + "eval_completions/mean_terminated_length": 79.49821472167969, + "eval_completions/min_length": 31.7, + "eval_completions/min_terminated_length": 31.7, + "eval_entropy": 0.46796957552433016, + "eval_frac_reward_zero_std": 0.0, + "eval_loss": NaN, + "eval_num_tokens": 3868253.0, + "eval_reward": 0.4048567622900009, + "eval_reward_count_adherence_mean": 0.9881250023841858, + "eval_reward_count_adherence_std": 0.03358757160604, + "eval_reward_hard_gate_mean": 0.975, + "eval_reward_hard_gate_std": 0.07071067690849304, + "eval_reward_judge_quality_mean": 0.15700000524520874, + "eval_reward_judge_quality_std": 0.02237090365961194, + "eval_reward_meter_mean": 0.7940246820449829, + "eval_reward_meter_std": 0.26315709128975867, + "eval_reward_repeat_soft_mean": 0.8970467984676361, + "eval_reward_repeat_soft_std": 0.07692538686096669, + "eval_reward_std": NaN, + "eval_reward_total_composite_mean": 0.4048567622900009, + "eval_reward_total_composite_std": 0.0588286342099309, + "eval_reward_total_mean": 0.4048567622900009, + "eval_rewards/count_adherence/mean": 0.9881250023841858, + "eval_rewards/count_adherence/std": 0.03358757160604, + "eval_rewards/hard_gate/mean": 0.975, + "eval_rewards/hard_gate/std": 0.07071067690849304, + "eval_rewards/judge_quality/mean": 0.15700000524520874, + "eval_rewards/judge_quality/std": 0.02237090365961194, + "eval_rewards/meter/mean": 0.7940246820449829, + "eval_rewards/meter/std": 0.26315709128975867, + "eval_rewards/repeat_soft/mean": 0.8970467984676361, + "eval_rewards/repeat_soft/std": 0.07692538686096669, + "eval_rewards/total_composite/mean": 0.4048567622900009, + "eval_rewards/total_composite/std": 0.0588286342099309, + "eval_runtime": 46.3808, + "eval_samples_per_second": 1.725, + "eval_sampling/importance_sampling_ratio/max": 1.4984214782714844, + "eval_sampling/importance_sampling_ratio/mean": 1.0083821773529054, + "eval_sampling/importance_sampling_ratio/min": 0.418349090218544, + "eval_sampling/sampling_logp_difference/max": 0.909369945526123, + "eval_sampling/sampling_logp_difference/mean": 0.046076252683997156, + "eval_steps_per_second": 0.216, + "step": 2150 } ], "logging_steps": 1, "max_steps": 3300, - "num_input_tokens_seen": 3777336, + "num_input_tokens_seen": 3868253, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": {