{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.08, "eval_steps": 500, "global_step": 300, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0002666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.00017316703451797366, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17440.0, "reward": 6.223865509033203, "reward_std": 0.3913865387439728, "rewards/CLIPTextReward/mean": 1.062528371810913, "rewards/CLIPTextReward/std": 0.18956568092107773, "rewards/HPSV21TextReward/mean": 0.8446283340454102, "rewards/HPSV21TextReward/std": 0.13295899704098701, "rewards/MANIQAReward/mean": 0.9095516800880432, "rewards/MANIQAReward/std": 0.18011662364006042, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1484375, "rewards/QwenTextReward/std": 0.03503827005624771, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0005333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.00028873729752376676, "learning_rate": 9.999866666666667e-06, "loss": 0.0, "num_tokens": 34768.0, "reward": 6.316446304321289, "reward_std": 0.43581461906433105, "rewards/CLIPTextReward/mean": 1.1428767442703247, "rewards/CLIPTextReward/std": 0.2098335102200508, "rewards/HPSV21TextReward/mean": 0.8283510208129883, "rewards/HPSV21TextReward/std": 0.0815248042345047, "rewards/MANIQAReward/mean": 0.9150067567825317, "rewards/MANIQAReward/std": 0.1554507613182068, "rewards/QwenFakeDiscrimReward/mean": 0.1171875, "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, "rewards/QwenTextReward/mean": 1.154296875, "rewards/QwenTextReward/std": 0.03688107803463936, "rewards/QwenWeirdDiscrimReward/mean": 0.1875, "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0008, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.00028006985667161644, "learning_rate": 9.999733333333335e-06, "loss": 0.0, "num_tokens": 52128.0, "reward": 6.023024559020996, "reward_std": 0.35070011019706726, "rewards/CLIPTextReward/mean": 0.9945527911186218, "rewards/CLIPTextReward/std": 0.16822098195552826, "rewards/HPSV21TextReward/mean": 0.7806148529052734, "rewards/HPSV21TextReward/std": 0.08261141180992126, "rewards/MANIQAReward/mean": 0.9121421575546265, "rewards/MANIQAReward/std": 0.21801920235157013, "rewards/QwenFakeDiscrimReward/mean": 0.1875, "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, "rewards/QwenTextReward/mean": 1.146484375, "rewards/QwenTextReward/std": 0.06738707423210144, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0010666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.00030690443236380816, "learning_rate": 9.9996e-06, "loss": 0.0, "num_tokens": 69472.0, "reward": 6.15365743637085, "reward_std": 0.38576382398605347, "rewards/CLIPTextReward/mean": 1.0728851556777954, "rewards/CLIPTextReward/std": 0.21109368652105331, "rewards/HPSV21TextReward/mean": 0.8231821060180664, "rewards/HPSV21TextReward/std": 0.07272331416606903, "rewards/MANIQAReward/mean": 0.8876948952674866, "rewards/MANIQAReward/std": 0.17049363255500793, "rewards/QwenFakeDiscrimReward/mean": 0.12890625, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.1613280773162842, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0013333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0003715873172041029, "learning_rate": 9.999466666666668e-06, "loss": 0.0, "num_tokens": 86816.0, "reward": 6.4580397605896, "reward_std": 0.512778103351593, "rewards/CLIPTextReward/mean": 1.1607547998428345, "rewards/CLIPTextReward/std": 0.1884431168437004, "rewards/HPSV21TextReward/mean": 0.844273567199707, "rewards/HPSV21TextReward/std": 0.09659288078546524, "rewards/MANIQAReward/mean": 0.9534515142440796, "rewards/MANIQAReward/std": 0.1852920651435852, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1507811546325684, "rewards/QwenTextReward/std": 0.03590354323387146, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0016, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.0003576505114324391, "learning_rate": 9.999333333333334e-06, "loss": 0.0, "num_tokens": 104080.0, "reward": 6.372293472290039, "reward_std": 0.38334107398986816, "rewards/CLIPTextReward/mean": 1.0605176091194153, "rewards/CLIPTextReward/std": 0.19939535111188889, "rewards/HPSV21TextReward/mean": 0.8251571655273438, "rewards/HPSV21TextReward/std": 0.11583757400512695, "rewards/MANIQAReward/mean": 1.0298501253128052, "rewards/MANIQAReward/std": 0.20007139444351196, "rewards/QwenFakeDiscrimReward/mean": 0.1875, "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, "rewards/QwenTextReward/mean": 1.1453125476837158, "rewards/QwenTextReward/std": 0.08473525196313858, "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0018666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0004226515593472868, "learning_rate": 9.9992e-06, "loss": 0.0, "num_tokens": 121456.0, "reward": 6.498950481414795, "reward_std": 0.4029247760772705, "rewards/CLIPTextReward/mean": 1.1567931771278381, "rewards/CLIPTextReward/std": 0.18052731454372406, "rewards/HPSV21TextReward/mean": 0.8444890975952148, "rewards/HPSV21TextReward/std": 0.1347436159849167, "rewards/MANIQAReward/mean": 0.9831047058105469, "rewards/MANIQAReward/std": 0.15593110024929047, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.146093726158142, "rewards/QwenTextReward/std": 0.03398728370666504, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0021333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.296875, "kl": 0.00035783814382739365, "learning_rate": 9.999066666666669e-06, "loss": 0.0, "num_tokens": 138880.0, "reward": 6.2124223709106445, "reward_std": 0.4332669973373413, "rewards/CLIPTextReward/mean": 1.1610087156295776, "rewards/CLIPTextReward/std": 0.21405618637800217, "rewards/HPSV21TextReward/mean": 0.8069028854370117, "rewards/HPSV21TextReward/std": 0.08297182619571686, "rewards/MANIQAReward/mean": 0.8281617164611816, "rewards/MANIQAReward/std": 0.15055784583091736, "rewards/QwenFakeDiscrimReward/mean": 0.12109375, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.1437500715255737, "rewards/QwenTextReward/std": 0.03779647499322891, "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0024, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.00041289368527941406, "learning_rate": 9.998933333333334e-06, "loss": 0.0, "num_tokens": 156200.0, "reward": 6.300518989562988, "reward_std": 0.4562382400035858, "rewards/CLIPTextReward/mean": 1.0610955953598022, "rewards/CLIPTextReward/std": 0.20977036654949188, "rewards/HPSV21TextReward/mean": 0.8444728851318359, "rewards/HPSV21TextReward/std": 0.08883989602327347, "rewards/MANIQAReward/mean": 0.9604760408401489, "rewards/MANIQAReward/std": 0.20047466456890106, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1304688453674316, "rewards/QwenTextReward/std": 0.0844530537724495, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0026666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0004245891759637743, "learning_rate": 9.998800000000002e-06, "loss": 0.0, "num_tokens": 173536.0, "reward": 6.316299915313721, "reward_std": 0.441649466753006, "rewards/CLIPTextReward/mean": 1.1014129519462585, "rewards/CLIPTextReward/std": 0.20749090611934662, "rewards/HPSV21TextReward/mean": 0.823883056640625, "rewards/HPSV21TextReward/std": 0.07232779264450073, "rewards/MANIQAReward/mean": 0.9332858324050903, "rewards/MANIQAReward/std": 0.19034986197948456, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1417968273162842, "rewards/QwenTextReward/std": 0.08393294364213943, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0029333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0004424244980327785, "learning_rate": 9.998666666666668e-06, "loss": 0.0, "num_tokens": 190824.0, "reward": 6.211538314819336, "reward_std": 0.2898920178413391, "rewards/CLIPTextReward/mean": 1.0182899236679077, "rewards/CLIPTextReward/std": 0.20580250024795532, "rewards/HPSV21TextReward/mean": 0.7981195449829102, "rewards/HPSV21TextReward/std": 0.08515609055757523, "rewards/MANIQAReward/mean": 1.0154380798339844, "rewards/MANIQAReward/std": 0.1735469251871109, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.1531250476837158, "rewards/QwenTextReward/std": 0.03659627586603165, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0032, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.0004862526257056743, "learning_rate": 9.998533333333334e-06, "loss": 0.0, "num_tokens": 208072.0, "reward": 6.214302062988281, "reward_std": 0.36605674028396606, "rewards/CLIPTextReward/mean": 1.0899392366409302, "rewards/CLIPTextReward/std": 0.18800289183855057, "rewards/HPSV21TextReward/mean": 0.8150472640991211, "rewards/HPSV21TextReward/std": 0.0666610449552536, "rewards/MANIQAReward/mean": 0.9449541568756104, "rewards/MANIQAReward/std": 0.21699047088623047, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1429686546325684, "rewards/QwenTextReward/std": 0.06776899099349976, "rewards/QwenWeirdDiscrimReward/mean": 0.16796875, "rewards/QwenWeirdDiscrimReward/std": 0.1183105930685997, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0034666666666666665, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.00047596223885193467, "learning_rate": 9.9984e-06, "loss": 0.0, "num_tokens": 225400.0, "reward": 6.290804862976074, "reward_std": 0.370810329914093, "rewards/CLIPTextReward/mean": 1.0436859726905823, "rewards/CLIPTextReward/std": 0.18754911422729492, "rewards/HPSV21TextReward/mean": 0.8343992233276367, "rewards/HPSV21TextReward/std": 0.13207274675369263, "rewards/MANIQAReward/mean": 1.0338529348373413, "rewards/MANIQAReward/std": 0.1687883734703064, "rewards/QwenFakeDiscrimReward/mean": 0.1640625, "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, "rewards/QwenTextReward/mean": 1.1257812976837158, "rewards/QwenTextReward/std": 0.12182222306728363, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 13 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0037333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.0004692480724770576, "learning_rate": 9.998266666666667e-06, "loss": 0.0, "num_tokens": 242768.0, "reward": 6.429315567016602, "reward_std": 0.3880622386932373, "rewards/CLIPTextReward/mean": 1.1203209161758423, "rewards/CLIPTextReward/std": 0.19741670042276382, "rewards/HPSV21TextReward/mean": 0.8249845504760742, "rewards/HPSV21TextReward/std": 0.05856947973370552, "rewards/MANIQAReward/mean": 0.9805015921592712, "rewards/MANIQAReward/std": 0.16263115406036377, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.148046851158142, "rewards/QwenTextReward/std": 0.07859187573194504, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.004, "frac_reward_zero_std": 0.0, "grad_norm": 1.265625, "kl": 0.00043514688150025904, "learning_rate": 9.998133333333333e-06, "loss": 0.0, "num_tokens": 260160.0, "reward": 6.392142295837402, "reward_std": 0.3614084720611572, "rewards/CLIPTextReward/mean": 1.1251448392868042, "rewards/CLIPTextReward/std": 0.18251177296042442, "rewards/HPSV21TextReward/mean": 0.8510913848876953, "rewards/HPSV21TextReward/std": 0.07994198799133301, "rewards/MANIQAReward/mean": 0.9330289363861084, "rewards/MANIQAReward/std": 0.15623073279857635, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1511719226837158, "rewards/QwenTextReward/std": 0.06834637373685837, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.004266666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "kl": 0.0004994600312784314, "learning_rate": 9.998000000000002e-06, "loss": 0.0, "num_tokens": 277400.0, "reward": 6.754908561706543, "reward_std": 0.37522512674331665, "rewards/CLIPTextReward/mean": 1.207650899887085, "rewards/CLIPTextReward/std": 0.16807805746793747, "rewards/HPSV21TextReward/mean": 0.8673477172851562, "rewards/HPSV21TextReward/std": 0.0916677750647068, "rewards/MANIQAReward/mean": 1.0424113273620605, "rewards/MANIQAReward/std": 0.22193863987922668, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.171875, "rewards/QwenTextReward/std": 0.03659627586603165, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.004533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.53125, "kl": 0.0008303101640194654, "learning_rate": 9.997866666666668e-06, "loss": 0.0001, "num_tokens": 294712.0, "reward": 6.324787139892578, "reward_std": 0.4522237181663513, "rewards/CLIPTextReward/mean": 1.104324460029602, "rewards/CLIPTextReward/std": 0.19701702892780304, "rewards/HPSV21TextReward/mean": 0.8078851699829102, "rewards/HPSV21TextReward/std": 0.07371688634157181, "rewards/MANIQAReward/mean": 0.9988051652908325, "rewards/MANIQAReward/std": 0.1549089401960373, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03750002756714821, "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, "step": 17 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0048, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0005571045912802219, "learning_rate": 9.997733333333334e-06, "loss": 0.0001, "num_tokens": 312136.0, "reward": 6.244279384613037, "reward_std": 0.4300357699394226, "rewards/CLIPTextReward/mean": 1.0606692433357239, "rewards/CLIPTextReward/std": 0.19169732928276062, "rewards/HPSV21TextReward/mean": 0.8216180801391602, "rewards/HPSV21TextReward/std": 0.09397384524345398, "rewards/MANIQAReward/mean": 0.9781421422958374, "rewards/MANIQAReward/std": 0.19963569939136505, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1500000953674316, "rewards/QwenTextReward/std": 0.06813853234052658, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.005066666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0005525641608983278, "learning_rate": 9.9976e-06, "loss": 0.0001, "num_tokens": 329464.0, "reward": 6.356129169464111, "reward_std": 0.3178771734237671, "rewards/CLIPTextReward/mean": 1.0747361183166504, "rewards/CLIPTextReward/std": 0.2152412012219429, "rewards/HPSV21TextReward/mean": 0.8245840072631836, "rewards/HPSV21TextReward/std": 0.07334348559379578, "rewards/MANIQAReward/mean": 1.0098326206207275, "rewards/MANIQAReward/std": 0.17456312477588654, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.1453125476837158, "rewards/QwenTextReward/std": 0.0670931488275528, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 19 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.005333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "kl": 0.000582196400500834, "learning_rate": 9.997466666666667e-06, "loss": 0.0001, "num_tokens": 346704.0, "reward": 6.182271957397461, "reward_std": 0.347652792930603, "rewards/CLIPTextReward/mean": 1.0524665117263794, "rewards/CLIPTextReward/std": 0.19090761989355087, "rewards/HPSV21TextReward/mean": 0.8114709854125977, "rewards/HPSV21TextReward/std": 0.06487897783517838, "rewards/MANIQAReward/mean": 0.91963130235672, "rewards/MANIQAReward/std": 0.1376141756772995, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1519532203674316, "rewards/QwenTextReward/std": 0.03627080097794533, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0056, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0006274660117924213, "learning_rate": 9.997333333333335e-06, "loss": 0.0001, "num_tokens": 364112.0, "reward": 6.596420764923096, "reward_std": 0.455806165933609, "rewards/CLIPTextReward/mean": 1.1512434482574463, "rewards/CLIPTextReward/std": 0.20681702345609665, "rewards/HPSV21TextReward/mean": 0.8719406127929688, "rewards/HPSV21TextReward/std": 0.11565934866666794, "rewards/MANIQAReward/mean": 1.0477088689804077, "rewards/MANIQAReward/std": 0.13334345817565918, "rewards/QwenFakeDiscrimReward/mean": 0.125, "rewards/QwenFakeDiscrimReward/std": 0.1259881556034088, "rewards/QwenTextReward/mean": 1.150781273841858, "rewards/QwenTextReward/std": 0.03590353950858116, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 21 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.005866666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "kl": 0.0005800112849101424, "learning_rate": 9.997200000000001e-06, "loss": 0.0001, "num_tokens": 381584.0, "reward": 6.364524841308594, "reward_std": 0.4369554817676544, "rewards/CLIPTextReward/mean": 1.150652289390564, "rewards/CLIPTextReward/std": 0.2127503752708435, "rewards/HPSV21TextReward/mean": 0.8205938339233398, "rewards/HPSV21TextReward/std": 0.10877228528261185, "rewards/MANIQAReward/mean": 0.9673448801040649, "rewards/MANIQAReward/std": 0.144505575299263, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.146093726158142, "rewards/QwenTextReward/std": 0.03398728370666504, "rewards/QwenWeirdDiscrimReward/mean": 0.1640625, "rewards/QwenWeirdDiscrimReward/std": 0.11967839300632477, "step": 22 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0061333333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.000612370204180479, "learning_rate": 9.997066666666668e-06, "loss": 0.0001, "num_tokens": 398896.0, "reward": 6.500385761260986, "reward_std": 0.37200304865837097, "rewards/CLIPTextReward/mean": 1.1138979196548462, "rewards/CLIPTextReward/std": 0.18904101103544235, "rewards/HPSV21TextReward/mean": 0.8669729232788086, "rewards/HPSV21TextReward/std": 0.1344263069331646, "rewards/MANIQAReward/mean": 0.9835654497146606, "rewards/MANIQAReward/std": 0.1934371143579483, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.156640648841858, "rewards/QwenTextReward/std": 0.037332236766815186, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 23 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0064, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0006386711029335856, "learning_rate": 9.996933333333334e-06, "loss": 0.0001, "num_tokens": 416392.0, "reward": 6.323739051818848, "reward_std": 0.4089418649673462, "rewards/CLIPTextReward/mean": 1.0975658893585205, "rewards/CLIPTextReward/std": 0.1965111717581749, "rewards/HPSV21TextReward/mean": 0.8068838119506836, "rewards/HPSV21TextReward/std": 0.08229941129684448, "rewards/MANIQAReward/mean": 0.9601527452468872, "rewards/MANIQAReward/std": 0.1424037218093872, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.040019553154706955, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 24 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.006666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.265625, "kl": 0.0006111857946962118, "learning_rate": 9.9968e-06, "loss": 0.0001, "num_tokens": 433824.0, "reward": 6.292370796203613, "reward_std": 0.3929201662540436, "rewards/CLIPTextReward/mean": 1.1571096181869507, "rewards/CLIPTextReward/std": 0.2031644508242607, "rewards/HPSV21TextReward/mean": 0.7992210388183594, "rewards/HPSV21TextReward/std": 0.06519654393196106, "rewards/MANIQAReward/mean": 0.9660371541976929, "rewards/MANIQAReward/std": 0.16348601877689362, "rewards/QwenFakeDiscrimReward/mean": 0.11328125, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.140234351158142, "rewards/QwenTextReward/std": 0.08037824183702469, "rewards/QwenWeirdDiscrimReward/mean": 0.16015625, "rewards/QwenWeirdDiscrimReward/std": 0.12090259045362473, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.006933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.000643837614916265, "learning_rate": 9.996666666666669e-06, "loss": 0.0001, "num_tokens": 451288.0, "reward": 6.366451740264893, "reward_std": 0.44354701042175293, "rewards/CLIPTextReward/mean": 1.1308492422103882, "rewards/CLIPTextReward/std": 0.19625339657068253, "rewards/HPSV21TextReward/mean": 0.8046188354492188, "rewards/HPSV21TextReward/std": 0.06655003875494003, "rewards/MANIQAReward/mean": 0.9287184476852417, "rewards/MANIQAReward/std": 0.16072098910808563, "rewards/QwenFakeDiscrimReward/mean": 0.19140625, "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, "rewards/QwenTextReward/mean": 1.168359398841858, "rewards/QwenTextReward/std": 0.03965182974934578, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 26 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0072, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.0006543556228280067, "learning_rate": 9.996533333333333e-06, "loss": 0.0001, "num_tokens": 468712.0, "reward": 6.34525728225708, "reward_std": 0.44636788964271545, "rewards/CLIPTextReward/mean": 1.1022841930389404, "rewards/CLIPTextReward/std": 0.20151617377996445, "rewards/HPSV21TextReward/mean": 0.8236732482910156, "rewards/HPSV21TextReward/std": 0.0780109316110611, "rewards/MANIQAReward/mean": 0.9542800784111023, "rewards/MANIQAReward/std": 0.15946204960346222, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 27 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.007466666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.2734375, "kl": 0.0005756819737143815, "learning_rate": 9.996400000000001e-06, "loss": 0.0001, "num_tokens": 485992.0, "reward": 6.045117378234863, "reward_std": 0.29585862159729004, "rewards/CLIPTextReward/mean": 1.0063416957855225, "rewards/CLIPTextReward/std": 0.1854785531759262, "rewards/HPSV21TextReward/mean": 0.7990350723266602, "rewards/HPSV21TextReward/std": 0.08045350760221481, "rewards/MANIQAReward/mean": 0.9308480024337769, "rewards/MANIQAReward/std": 0.13604064285755157, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1480469703674316, "rewards/QwenTextReward/std": 0.0706130787730217, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 28 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.007733333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.000711756176315248, "learning_rate": 9.996266666666668e-06, "loss": 0.0001, "num_tokens": 503344.0, "reward": 6.625212669372559, "reward_std": 0.37315335869789124, "rewards/CLIPTextReward/mean": 1.1993611454963684, "rewards/CLIPTextReward/std": 0.18592771887779236, "rewards/HPSV21TextReward/mean": 0.8432188034057617, "rewards/HPSV21TextReward/std": 0.0620620995759964, "rewards/MANIQAReward/mean": 1.0455212593078613, "rewards/MANIQAReward/std": 0.16021911799907684, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.150781273841858, "rewards/QwenTextReward/std": 0.03590353950858116, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 29 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.008, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.000612220203038305, "learning_rate": 9.996133333333334e-06, "loss": 0.0001, "num_tokens": 520648.0, "reward": 6.518412113189697, "reward_std": 0.40200844407081604, "rewards/CLIPTextReward/mean": 1.1843986511230469, "rewards/CLIPTextReward/std": 0.19694440811872482, "rewards/HPSV21TextReward/mean": 0.8349838256835938, "rewards/HPSV21TextReward/std": 0.0758972018957138, "rewards/MANIQAReward/mean": 0.9241786003112793, "rewards/MANIQAReward/std": 0.14468914270401, "rewards/QwenFakeDiscrimReward/mean": 0.1875, "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, "rewards/QwenTextReward/mean": 1.1531250476837158, "rewards/QwenTextReward/std": 0.08183173090219498, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.008266666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0005961424903944135, "learning_rate": 9.996e-06, "loss": 0.0001, "num_tokens": 538040.0, "reward": 6.067929267883301, "reward_std": 0.3891369700431824, "rewards/CLIPTextReward/mean": 1.0164735317230225, "rewards/CLIPTextReward/std": 0.19765233993530273, "rewards/HPSV21TextReward/mean": 0.7834997177124023, "rewards/HPSV21TextReward/std": 0.07358982414007187, "rewards/MANIQAReward/mean": 0.9195454716682434, "rewards/MANIQAReward/std": 0.1801491528749466, "rewards/QwenFakeDiscrimReward/mean": 0.19921875, "rewards/QwenFakeDiscrimReward/std": 0.10137632489204407, "rewards/QwenTextReward/mean": 1.1343750953674316, "rewards/QwenTextReward/std": 0.11456441134214401, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 31 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.008533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.265625, "kl": 0.0006076592253521085, "learning_rate": 9.995866666666667e-06, "loss": 0.0001, "num_tokens": 555432.0, "reward": 6.158683776855469, "reward_std": 0.37738487124443054, "rewards/CLIPTextReward/mean": 1.07703697681427, "rewards/CLIPTextReward/std": 0.18969328701496124, "rewards/HPSV21TextReward/mean": 0.8205890655517578, "rewards/HPSV21TextReward/std": 0.10732360929250717, "rewards/MANIQAReward/mean": 0.8689001798629761, "rewards/MANIQAReward/std": 0.15288065373897552, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1429686546325684, "rewards/QwenTextReward/std": 0.06643842905759811, "rewards/QwenWeirdDiscrimReward/mean": 0.1796875, "rewards/QwenWeirdDiscrimReward/std": 0.11329087615013123, "step": 32 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0088, "frac_reward_zero_std": 0.0, "grad_norm": 1.4453125, "kl": 0.0007134638144634664, "learning_rate": 9.995733333333333e-06, "loss": 0.0001, "num_tokens": 572752.0, "reward": 6.217061996459961, "reward_std": 0.3514450490474701, "rewards/CLIPTextReward/mean": 1.031137228012085, "rewards/CLIPTextReward/std": 0.20571595430374146, "rewards/HPSV21TextReward/mean": 0.8295192718505859, "rewards/HPSV21TextReward/std": 0.09005855023860931, "rewards/MANIQAReward/mean": 0.9406708478927612, "rewards/MANIQAReward/std": 0.15536098182201385, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.156640648841858, "rewards/QwenTextReward/std": 0.037332236766815186, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 33 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.009066666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0006718998192809522, "learning_rate": 9.995600000000001e-06, "loss": 0.0001, "num_tokens": 590176.0, "reward": 6.396887302398682, "reward_std": 0.35195744037628174, "rewards/CLIPTextReward/mean": 1.087852418422699, "rewards/CLIPTextReward/std": 0.2179098054766655, "rewards/HPSV21TextReward/mean": 0.8623418807983398, "rewards/HPSV21TextReward/std": 0.12201350927352905, "rewards/MANIQAReward/mean": 0.9789204001426697, "rewards/MANIQAReward/std": 0.1579286903142929, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.158203125, "rewards/QwenTextReward/std": 0.04250461608171463, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 34 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.009333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0006477828719653189, "learning_rate": 9.995466666666668e-06, "loss": 0.0001, "num_tokens": 607512.0, "reward": 6.4933390617370605, "reward_std": 0.4455246329307556, "rewards/CLIPTextReward/mean": 1.1464911699295044, "rewards/CLIPTextReward/std": 0.20508115738630295, "rewards/HPSV21TextReward/mean": 0.8466615676879883, "rewards/HPSV21TextReward/std": 0.06445857137441635, "rewards/MANIQAReward/mean": 0.9734399318695068, "rewards/MANIQAReward/std": 0.18179957568645477, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.162500023841858, "rewards/QwenTextReward/std": 0.040089212357997894, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0096, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.0006656671175733209, "learning_rate": 9.995333333333334e-06, "loss": 0.0001, "num_tokens": 624928.0, "reward": 6.222779750823975, "reward_std": 0.513312578201294, "rewards/CLIPTextReward/mean": 1.0915876626968384, "rewards/CLIPTextReward/std": 0.21901841461658478, "rewards/HPSV21TextReward/mean": 0.8346834182739258, "rewards/HPSV21TextReward/std": 0.07725655287504196, "rewards/MANIQAReward/mean": 0.8718000054359436, "rewards/MANIQAReward/std": 0.19936029613018036, "rewards/QwenFakeDiscrimReward/mean": 0.1328125, "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, "rewards/QwenTextReward/mean": 1.154687523841858, "rewards/QwenTextReward/std": 0.06664808094501495, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 36 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.009866666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 2.515625, "kl": 0.0007665607845410705, "learning_rate": 9.9952e-06, "loss": 0.0001, "num_tokens": 642304.0, "reward": 6.087282180786133, "reward_std": 0.4493650197982788, "rewards/CLIPTextReward/mean": 1.0269891023635864, "rewards/CLIPTextReward/std": 0.19465582817792892, "rewards/HPSV21TextReward/mean": 0.8145656585693359, "rewards/HPSV21TextReward/std": 0.06945210695266724, "rewards/MANIQAReward/mean": 0.908078670501709, "rewards/MANIQAReward/std": 0.15028269588947296, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.15234375, "rewards/QwenTextReward/std": 0.06853322684764862, "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, "step": 37 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.010133333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.0006748391315340996, "learning_rate": 9.995066666666667e-06, "loss": 0.0001, "num_tokens": 659616.0, "reward": 6.521594047546387, "reward_std": 0.28268447518348694, "rewards/CLIPTextReward/mean": 1.1239572763442993, "rewards/CLIPTextReward/std": 0.19386181235313416, "rewards/HPSV21TextReward/mean": 0.8642807006835938, "rewards/HPSV21TextReward/std": 0.16283436119556427, "rewards/MANIQAReward/mean": 0.9767587184906006, "rewards/MANIQAReward/std": 0.13164694607257843, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.154296875, "rewards/QwenTextReward/std": 0.03688107803463936, "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, "step": 38 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0104, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.0006359356339089572, "learning_rate": 9.994933333333335e-06, "loss": 0.0001, "num_tokens": 677008.0, "reward": 6.220743179321289, "reward_std": 0.41407012939453125, "rewards/CLIPTextReward/mean": 1.0410704612731934, "rewards/CLIPTextReward/std": 0.16566266119480133, "rewards/HPSV21TextReward/mean": 0.8238363265991211, "rewards/HPSV21TextReward/std": 0.10602444410324097, "rewards/MANIQAReward/mean": 0.9217893481254578, "rewards/MANIQAReward/std": 0.1595626324415207, "rewards/QwenFakeDiscrimReward/mean": 0.1953125, "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, "rewards/QwenTextReward/mean": 1.1472656726837158, "rewards/QwenTextReward/std": 0.03453698009252548, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 39 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.010666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.000720483367331326, "learning_rate": 9.994800000000001e-06, "loss": 0.0001, "num_tokens": 694360.0, "reward": 6.090251922607422, "reward_std": 0.3836488425731659, "rewards/CLIPTextReward/mean": 1.0377899408340454, "rewards/CLIPTextReward/std": 0.20379655063152313, "rewards/HPSV21TextReward/mean": 0.8056297302246094, "rewards/HPSV21TextReward/std": 0.0683428943157196, "rewards/MANIQAReward/mean": 0.9120062589645386, "rewards/MANIQAReward/std": 0.1517135202884674, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1554687023162842, "rewards/QwenTextReward/std": 0.03945784643292427, "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.010933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.25, "kl": 0.0007212897762656212, "learning_rate": 9.994666666666668e-06, "loss": 0.0001, "num_tokens": 711672.0, "reward": 6.369787216186523, "reward_std": 0.3154414892196655, "rewards/CLIPTextReward/mean": 1.116544485092163, "rewards/CLIPTextReward/std": 0.2094419151544571, "rewards/HPSV21TextReward/mean": 0.8303442001342773, "rewards/HPSV21TextReward/std": 0.06440813094377518, "rewards/MANIQAReward/mean": 0.969369113445282, "rewards/MANIQAReward/std": 0.19423282146453857, "rewards/QwenFakeDiscrimReward/mean": 0.13671875, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.1707031726837158, "rewards/QwenTextReward/std": 0.03688107430934906, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 41 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0112, "frac_reward_zero_std": 0.0, "grad_norm": 1.4375, "kl": 0.0007738057756796479, "learning_rate": 9.994533333333334e-06, "loss": 0.0001, "num_tokens": 729000.0, "reward": 6.430874347686768, "reward_std": 0.43917739391326904, "rewards/CLIPTextReward/mean": 1.0832951068878174, "rewards/CLIPTextReward/std": 0.19060203433036804, "rewards/HPSV21TextReward/mean": 0.8397102355957031, "rewards/HPSV21TextReward/std": 0.11080950498580933, "rewards/MANIQAReward/mean": 1.0235352516174316, "rewards/MANIQAReward/std": 0.15490847826004028, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1511719226837158, "rewards/QwenTextReward/std": 0.06834637373685837, "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, "step": 42 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.011466666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.000737136579118669, "learning_rate": 9.9944e-06, "loss": 0.0001, "num_tokens": 746304.0, "reward": 6.281833648681641, "reward_std": 0.3190336525440216, "rewards/CLIPTextReward/mean": 1.044514775276184, "rewards/CLIPTextReward/std": 0.18819903582334518, "rewards/HPSV21TextReward/mean": 0.8453836441040039, "rewards/HPSV21TextReward/std": 0.05973060056567192, "rewards/MANIQAReward/mean": 0.9825053215026855, "rewards/MANIQAReward/std": 0.16364580392837524, "rewards/QwenFakeDiscrimReward/mean": 0.1640625, "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, "rewards/QwenTextReward/mean": 1.1484375, "rewards/QwenTextReward/std": 0.03503827005624771, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 43 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.011733333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0007885091472417116, "learning_rate": 9.994266666666668e-06, "loss": 0.0001, "num_tokens": 763560.0, "reward": 6.076322555541992, "reward_std": 0.45350897312164307, "rewards/CLIPTextReward/mean": 1.0211127996444702, "rewards/CLIPTextReward/std": 0.19371271133422852, "rewards/HPSV21TextReward/mean": 0.8039369583129883, "rewards/HPSV21TextReward/std": 0.07984976470470428, "rewards/MANIQAReward/mean": 0.9441919326782227, "rewards/MANIQAReward/std": 0.15260986983776093, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1500000953674316, "rewards/QwenTextReward/std": 0.040824852883815765, "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, "step": 44 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.012, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0007469256524927914, "learning_rate": 9.994133333333335e-06, "loss": 0.0001, "num_tokens": 780912.0, "reward": 6.200069427490234, "reward_std": 0.4303773045539856, "rewards/CLIPTextReward/mean": 1.0523357391357422, "rewards/CLIPTextReward/std": 0.20511925965547562, "rewards/HPSV21TextReward/mean": 0.850214958190918, "rewards/HPSV21TextReward/std": 0.10265272110700607, "rewards/MANIQAReward/mean": 0.9160617589950562, "rewards/MANIQAReward/std": 0.16560041904449463, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.150781273841858, "rewards/QwenTextReward/std": 0.06298917531967163, "rewards/QwenWeirdDiscrimReward/mean": 0.1875, "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, "step": 45 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.012266666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.4296875, "kl": 0.0008448648732155561, "learning_rate": 9.994000000000001e-06, "loss": 0.0001, "num_tokens": 798256.0, "reward": 6.494973182678223, "reward_std": 0.36463749408721924, "rewards/CLIPTextReward/mean": 1.0624587535858154, "rewards/CLIPTextReward/std": 0.17267398536205292, "rewards/HPSV21TextReward/mean": 0.8356428146362305, "rewards/HPSV21TextReward/std": 0.13331294804811478, "rewards/MANIQAReward/mean": 1.0733790397644043, "rewards/MANIQAReward/std": 0.17399130761623383, "rewards/QwenFakeDiscrimReward/mean": 0.22265625, "rewards/QwenFakeDiscrimReward/std": 0.07864411175251007, "rewards/QwenTextReward/mean": 1.156640648841858, "rewards/QwenTextReward/std": 0.037332240492105484, "rewards/QwenWeirdDiscrimReward/mean": 0.24609375, "rewards/QwenWeirdDiscrimReward/std": 0.03125, "step": 46 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.012533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.0007328381179831922, "learning_rate": 9.993866666666668e-06, "loss": 0.0001, "num_tokens": 815576.0, "reward": 6.38761043548584, "reward_std": 0.3840087056159973, "rewards/CLIPTextReward/mean": 1.13616943359375, "rewards/CLIPTextReward/std": 0.19060511887073517, "rewards/HPSV21TextReward/mean": 0.8550443649291992, "rewards/HPSV21TextReward/std": 0.07190331816673279, "rewards/MANIQAReward/mean": 0.8887766599655151, "rewards/MANIQAReward/std": 0.17912423610687256, "rewards/QwenFakeDiscrimReward/mean": 0.1640625, "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, "rewards/QwenTextReward/mean": 1.1453125476837158, "rewards/QwenTextReward/std": 0.0648372694849968, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 47 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0128, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0007853477727621794, "learning_rate": 9.993733333333334e-06, "loss": 0.0001, "num_tokens": 832856.0, "reward": 6.501211643218994, "reward_std": 0.43350669741630554, "rewards/CLIPTextReward/mean": 1.1371588110923767, "rewards/CLIPTextReward/std": 0.20060991495847702, "rewards/HPSV21TextReward/mean": 0.8673725128173828, "rewards/HPSV21TextReward/std": 0.12160860747098923, "rewards/MANIQAReward/mean": 0.9319926500320435, "rewards/MANIQAReward/std": 0.18476037681102753, "rewards/QwenFakeDiscrimReward/mean": 0.19140625, "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, "rewards/QwenTextReward/mean": 1.161718726158142, "rewards/QwenTextReward/std": 0.04796842113137245, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 48 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.013066666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0008283444913104177, "learning_rate": 9.9936e-06, "loss": 0.0001, "num_tokens": 850200.0, "reward": 6.140354156494141, "reward_std": 0.4246932864189148, "rewards/CLIPTextReward/mean": 1.0640084743499756, "rewards/CLIPTextReward/std": 0.19425738602876663, "rewards/HPSV21TextReward/mean": 0.8056106567382812, "rewards/HPSV21TextReward/std": 0.07409681379795074, "rewards/MANIQAReward/mean": 0.9229907989501953, "rewards/MANIQAReward/std": 0.19340187311172485, "rewards/QwenFakeDiscrimReward/mean": 0.1328125, "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, "rewards/QwenTextReward/mean": 1.157812476158142, "rewards/QwenTextReward/std": 0.03980984166264534, "rewards/QwenWeirdDiscrimReward/mean": 0.1875, "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, "step": 49 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.013333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0007959783542901278, "learning_rate": 9.993466666666668e-06, "loss": 0.0001, "num_tokens": 867576.0, "reward": 6.259000778198242, "reward_std": 0.5205031037330627, "rewards/CLIPTextReward/mean": 1.073885202407837, "rewards/CLIPTextReward/std": 0.19922605156898499, "rewards/HPSV21TextReward/mean": 0.8068361282348633, "rewards/HPSV21TextReward/std": 0.07939974218606949, "rewards/MANIQAReward/mean": 0.9553705453872681, "rewards/MANIQAReward/std": 0.16773752868175507, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1476562023162842, "rewards/QwenTextReward/std": 0.04425728693604469, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0136, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.0007928041159175336, "learning_rate": 9.993333333333333e-06, "loss": 0.0001, "num_tokens": 885056.0, "reward": 6.2416276931762695, "reward_std": 0.33849644660949707, "rewards/CLIPTextReward/mean": 1.0327467322349548, "rewards/CLIPTextReward/std": 0.21467745304107666, "rewards/HPSV21TextReward/mean": 0.8171548843383789, "rewards/HPSV21TextReward/std": 0.07690542936325073, "rewards/MANIQAReward/mean": 1.025027871131897, "rewards/MANIQAReward/std": 0.1482948362827301, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1339843273162842, "rewards/QwenTextReward/std": 0.10013555735349655, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 51 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.013866666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.0008756698807701468, "learning_rate": 9.993200000000001e-06, "loss": 0.0001, "num_tokens": 902304.0, "reward": 6.167211532592773, "reward_std": 0.5379205346107483, "rewards/CLIPTextReward/mean": 1.0588380098342896, "rewards/CLIPTextReward/std": 0.19965583831071854, "rewards/HPSV21TextReward/mean": 0.7852449417114258, "rewards/HPSV21TextReward/std": 0.08360420167446136, "rewards/MANIQAReward/mean": 0.9634205102920532, "rewards/MANIQAReward/std": 0.16673992574214935, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 52 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.014133333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.4296875, "kl": 0.0009323436534032226, "learning_rate": 9.993066666666668e-06, "loss": 0.0001, "num_tokens": 919728.0, "reward": 6.469545364379883, "reward_std": 0.3471949100494385, "rewards/CLIPTextReward/mean": 1.1538089513778687, "rewards/CLIPTextReward/std": 0.20266949385404587, "rewards/HPSV21TextReward/mean": 0.7827653884887695, "rewards/HPSV21TextReward/std": 0.08379530906677246, "rewards/MANIQAReward/mean": 1.067490816116333, "rewards/MANIQAReward/std": 0.17146986722946167, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.146093726158142, "rewards/QwenTextReward/std": 0.03398728370666504, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 53 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0144, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0008969584014266729, "learning_rate": 9.992933333333334e-06, "loss": 0.0001, "num_tokens": 937080.0, "reward": 6.236813545227051, "reward_std": 0.3845016658306122, "rewards/CLIPTextReward/mean": 1.0523546934127808, "rewards/CLIPTextReward/std": 0.19251258671283722, "rewards/HPSV21TextReward/mean": 0.8053016662597656, "rewards/HPSV21TextReward/std": 0.1164521649479866, "rewards/MANIQAReward/mean": 0.9812659025192261, "rewards/MANIQAReward/std": 0.1439070701599121, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1613280773162842, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 54 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.014666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0008943352149799466, "learning_rate": 9.9928e-06, "loss": 0.0001, "num_tokens": 954624.0, "reward": 6.408685684204102, "reward_std": 0.36431431770324707, "rewards/CLIPTextReward/mean": 1.1066354513168335, "rewards/CLIPTextReward/std": 0.1999390497803688, "rewards/HPSV21TextReward/mean": 0.832585334777832, "rewards/HPSV21TextReward/std": 0.08066749572753906, "rewards/MANIQAReward/mean": 1.012665867805481, "rewards/MANIQAReward/std": 0.17072820663452148, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.142578125, "rewards/QwenTextReward/std": 0.032021742314100266, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.014933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0008820436196401715, "learning_rate": 9.992666666666667e-06, "loss": 0.0001, "num_tokens": 971936.0, "reward": 6.167948246002197, "reward_std": 0.2834055423736572, "rewards/CLIPTextReward/mean": 0.9877440929412842, "rewards/CLIPTextReward/std": 0.1475164331495762, "rewards/HPSV21TextReward/mean": 0.8200302124023438, "rewards/HPSV21TextReward/std": 0.08542144298553467, "rewards/MANIQAReward/mean": 0.9781813621520996, "rewards/MANIQAReward/std": 0.1652689427137375, "rewards/QwenFakeDiscrimReward/mean": 0.1953125, "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, "rewards/QwenTextReward/mean": 1.15234375, "rewards/QwenTextReward/std": 0.07672860473394394, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 56 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0152, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.0008737280732020736, "learning_rate": 9.992533333333335e-06, "loss": 0.0001, "num_tokens": 989296.0, "reward": 6.1255717277526855, "reward_std": 0.47202640771865845, "rewards/CLIPTextReward/mean": 1.0381349325180054, "rewards/CLIPTextReward/std": 0.1850999891757965, "rewards/HPSV21TextReward/mean": 0.810758113861084, "rewards/HPSV21TextReward/std": 0.07670347392559052, "rewards/MANIQAReward/mean": 0.9480981826782227, "rewards/MANIQAReward/std": 0.17108558118343353, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1398437023162842, "rewards/QwenTextReward/std": 0.08748229593038559, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 57 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.015466666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.0010484395315870643, "learning_rate": 9.992400000000001e-06, "loss": 0.0001, "num_tokens": 1006584.0, "reward": 6.327513217926025, "reward_std": 0.4302184581756592, "rewards/CLIPTextReward/mean": 1.0683895349502563, "rewards/CLIPTextReward/std": 0.20896943658590317, "rewards/HPSV21TextReward/mean": 0.8449268341064453, "rewards/HPSV21TextReward/std": 0.18004687875509262, "rewards/MANIQAReward/mean": 0.9700211882591248, "rewards/MANIQAReward/std": 0.19353777170181274, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1636719703674316, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 58 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.015733333333333332, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.0008922148845158517, "learning_rate": 9.992266666666667e-06, "loss": 0.0001, "num_tokens": 1024008.0, "reward": 6.32124662399292, "reward_std": 0.3739131689071655, "rewards/CLIPTextReward/mean": 1.0668448209762573, "rewards/CLIPTextReward/std": 0.19243191927671432, "rewards/HPSV21TextReward/mean": 0.8356704711914062, "rewards/HPSV21TextReward/std": 0.07387026399374008, "rewards/MANIQAReward/mean": 0.9986376166343689, "rewards/MANIQAReward/std": 0.1391114592552185, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.138671875, "rewards/QwenTextReward/std": 0.09469351917505264, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 59 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.016, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.0008508785394951701, "learning_rate": 9.992133333333334e-06, "loss": 0.0001, "num_tokens": 1041288.0, "reward": 6.385502815246582, "reward_std": 0.37456315755844116, "rewards/CLIPTextReward/mean": 1.0719799399375916, "rewards/CLIPTextReward/std": 0.19807685166597366, "rewards/HPSV21TextReward/mean": 0.8345270156860352, "rewards/HPSV21TextReward/std": 0.10260890424251556, "rewards/MANIQAReward/mean": 1.0138952732086182, "rewards/MANIQAReward/std": 0.16401827335357666, "rewards/QwenFakeDiscrimReward/mean": 0.20703125, "rewards/QwenFakeDiscrimReward/std": 0.09506355226039886, "rewards/QwenTextReward/mean": 1.1484375, "rewards/QwenTextReward/std": 0.03503827005624771, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.016266666666666665, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.000956534524448216, "learning_rate": 9.992e-06, "loss": 0.0001, "num_tokens": 1058704.0, "reward": 6.24791145324707, "reward_std": 0.3534240126609802, "rewards/CLIPTextReward/mean": 1.087317705154419, "rewards/CLIPTextReward/std": 0.2023688331246376, "rewards/HPSV21TextReward/mean": 0.7986822128295898, "rewards/HPSV21TextReward/std": 0.06513593345880508, "rewards/MANIQAReward/mean": 0.89153653383255, "rewards/MANIQAReward/std": 0.1612655520439148, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.174218773841858, "rewards/QwenTextReward/std": 0.03590353950858116, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 61 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.016533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0009708399884402752, "learning_rate": 9.991866666666668e-06, "loss": 0.0001, "num_tokens": 1076168.0, "reward": 6.217422962188721, "reward_std": 0.3819130063056946, "rewards/CLIPTextReward/mean": 1.085562825202942, "rewards/CLIPTextReward/std": 0.1930897980928421, "rewards/HPSV21TextReward/mean": 0.7957496643066406, "rewards/HPSV21TextReward/std": 0.08908992260694504, "rewards/MANIQAReward/mean": 0.9255013465881348, "rewards/MANIQAReward/std": 0.15410436689853668, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.166015625, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 62 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0168, "frac_reward_zero_std": 0.0, "grad_norm": 1.296875, "kl": 0.0009530046954751015, "learning_rate": 9.991733333333335e-06, "loss": 0.0001, "num_tokens": 1093560.0, "reward": 6.324612140655518, "reward_std": 0.32763224840164185, "rewards/CLIPTextReward/mean": 1.0497437715530396, "rewards/CLIPTextReward/std": 0.19730062782764435, "rewards/HPSV21TextReward/mean": 0.8403587341308594, "rewards/HPSV21TextReward/std": 0.11551825702190399, "rewards/MANIQAReward/mean": 0.9768288731575012, "rewards/MANIQAReward/std": 0.18326574563980103, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1652344465255737, "rewards/QwenTextReward/std": 0.06923867762088776, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 63 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.017066666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.390625, "kl": 0.0010437159799039364, "learning_rate": 9.991600000000001e-06, "loss": 0.0001, "num_tokens": 1110864.0, "reward": 6.267642021179199, "reward_std": 0.3985981345176697, "rewards/CLIPTextReward/mean": 1.0354081988334656, "rewards/CLIPTextReward/std": 0.196964293718338, "rewards/HPSV21TextReward/mean": 0.816436767578125, "rewards/HPSV21TextReward/std": 0.1313062310218811, "rewards/MANIQAReward/mean": 1.0315299034118652, "rewards/MANIQAReward/std": 0.18300341069698334, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1496093273162842, "rewards/QwenTextReward/std": 0.06275618821382523, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 64 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.017333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.0010261985007673502, "learning_rate": 9.991466666666667e-06, "loss": 0.0001, "num_tokens": 1128240.0, "reward": 6.297903060913086, "reward_std": 0.3682844340801239, "rewards/CLIPTextReward/mean": 1.05167156457901, "rewards/CLIPTextReward/std": 0.1987253725528717, "rewards/HPSV21TextReward/mean": 0.8304023742675781, "rewards/HPSV21TextReward/std": 0.11583104729652405, "rewards/MANIQAReward/mean": 0.9892240762710571, "rewards/MANIQAReward/std": 0.1848040372133255, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1500000953674316, "rewards/QwenTextReward/std": 0.06813853234052658, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 65 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0176, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0009466060437262058, "learning_rate": 9.991333333333334e-06, "loss": 0.0001, "num_tokens": 1145576.0, "reward": 6.463624000549316, "reward_std": 0.436023086309433, "rewards/CLIPTextReward/mean": 1.1064223051071167, "rewards/CLIPTextReward/std": 0.21644117683172226, "rewards/HPSV21TextReward/mean": 0.8701066970825195, "rewards/HPSV21TextReward/std": 0.1395508013665676, "rewards/MANIQAReward/mean": 1.0152537822723389, "rewards/MANIQAReward/std": 0.16907808184623718, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1554687023162842, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, "step": 66 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.017866666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.0010095324832946062, "learning_rate": 9.9912e-06, "loss": 0.0001, "num_tokens": 1163008.0, "reward": 6.304980278015137, "reward_std": 0.4456925094127655, "rewards/CLIPTextReward/mean": 1.058637797832489, "rewards/CLIPTextReward/std": 0.19116178900003433, "rewards/HPSV21TextReward/mean": 0.8249893188476562, "rewards/HPSV21TextReward/std": 0.07794276624917984, "rewards/MANIQAReward/mean": 1.0181949138641357, "rewards/MANIQAReward/std": 0.15045630931854248, "rewards/QwenFakeDiscrimReward/mean": 0.1640625, "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, "rewards/QwenTextReward/mean": 1.140625, "rewards/QwenTextReward/std": 0.08363043516874313, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 67 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.018133333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.001076919841580093, "learning_rate": 9.991066666666668e-06, "loss": 0.0001, "num_tokens": 1180448.0, "reward": 6.19140625, "reward_std": 0.33228224515914917, "rewards/CLIPTextReward/mean": 1.0292329788208008, "rewards/CLIPTextReward/std": 0.18924038857221603, "rewards/HPSV21TextReward/mean": 0.8369541168212891, "rewards/HPSV21TextReward/std": 0.11581806093454361, "rewards/MANIQAReward/mean": 0.9867665767669678, "rewards/MANIQAReward/std": 0.13873277604579926, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.124609351158142, "rewards/QwenTextReward/std": 0.09692619740962982, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 68 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0184, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0010598800145089626, "learning_rate": 9.990933333333335e-06, "loss": 0.0001, "num_tokens": 1197824.0, "reward": 6.298628807067871, "reward_std": 0.4064304828643799, "rewards/CLIPTextReward/mean": 1.0958648920059204, "rewards/CLIPTextReward/std": 0.1893087550997734, "rewards/HPSV21TextReward/mean": 0.846552848815918, "rewards/HPSV21TextReward/std": 0.09922610223293304, "rewards/MANIQAReward/mean": 0.9290274977684021, "rewards/MANIQAReward/std": 0.15360622107982635, "rewards/QwenFakeDiscrimReward/mean": 0.12890625, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.141015648841858, "rewards/QwenTextReward/std": 0.07694144546985626, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 69 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.018666666666666668, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.001305924030020833, "learning_rate": 9.990800000000001e-06, "loss": 0.0001, "num_tokens": 1215128.0, "reward": 6.0216965675354, "reward_std": 0.3705746829509735, "rewards/CLIPTextReward/mean": 0.9692765474319458, "rewards/CLIPTextReward/std": 0.17790019512176514, "rewards/HPSV21TextReward/mean": 0.8086051940917969, "rewards/HPSV21TextReward/std": 0.07346846163272858, "rewards/MANIQAReward/mean": 0.9409325122833252, "rewards/MANIQAReward/std": 0.156536266207695, "rewards/QwenFakeDiscrimReward/mean": 0.13671875, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.1695313453674316, "rewards/QwenTextReward/std": 0.03945784643292427, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.018933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0010778771247714758, "learning_rate": 9.990666666666667e-06, "loss": 0.0001, "num_tokens": 1232456.0, "reward": 6.8039398193359375, "reward_std": 0.30482178926467896, "rewards/CLIPTextReward/mean": 1.1879304647445679, "rewards/CLIPTextReward/std": 0.1860901564359665, "rewards/HPSV21TextReward/mean": 0.8902063369750977, "rewards/HPSV21TextReward/std": 0.06591881066560745, "rewards/MANIQAReward/mean": 1.088681697845459, "rewards/MANIQAReward/std": 0.15087370574474335, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.156640648841858, "rewards/QwenTextReward/std": 0.037332240492105484, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 71 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0192, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0010855778818950057, "learning_rate": 9.990533333333334e-06, "loss": 0.0001, "num_tokens": 1249840.0, "reward": 6.250958442687988, "reward_std": 0.3621181547641754, "rewards/CLIPTextReward/mean": 1.0959141254425049, "rewards/CLIPTextReward/std": 0.1913270279765129, "rewards/HPSV21TextReward/mean": 0.7908287048339844, "rewards/HPSV21TextReward/std": 0.0678899884223938, "rewards/MANIQAReward/mean": 0.9958319664001465, "rewards/MANIQAReward/std": 0.1409938782453537, "rewards/QwenFakeDiscrimReward/mean": 0.13671875, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.1417968273162842, "rewards/QwenTextReward/std": 0.06607697904109955, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 72 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.019466666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.001005525584332645, "learning_rate": 9.990400000000002e-06, "loss": 0.0001, "num_tokens": 1267272.0, "reward": 6.217660903930664, "reward_std": 0.4128098487854004, "rewards/CLIPTextReward/mean": 1.0573053359985352, "rewards/CLIPTextReward/std": 0.20548022538423538, "rewards/HPSV21TextReward/mean": 0.8063507080078125, "rewards/HPSV21TextReward/std": 0.1057361587882042, "rewards/MANIQAReward/mean": 0.9743335247039795, "rewards/MANIQAReward/std": 0.15329712629318237, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1566407680511475, "rewards/QwenTextReward/std": 0.037332236766815186, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 73 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.019733333333333332, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "kl": 0.0011021631071344018, "learning_rate": 9.990266666666666e-06, "loss": 0.0001, "num_tokens": 1284600.0, "reward": 6.18872594833374, "reward_std": 0.47515836358070374, "rewards/CLIPTextReward/mean": 1.0537782311439514, "rewards/CLIPTextReward/std": 0.19655564427375793, "rewards/HPSV21TextReward/mean": 0.8184757232666016, "rewards/HPSV21TextReward/std": 0.061357416212558746, "rewards/MANIQAReward/mean": 0.9172650575637817, "rewards/MANIQAReward/std": 0.14114581048488617, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1636719703674316, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 74 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.02, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0009673364111222327, "learning_rate": 9.990133333333335e-06, "loss": 0.0001, "num_tokens": 1302080.0, "reward": 6.431103706359863, "reward_std": 0.4042501449584961, "rewards/CLIPTextReward/mean": 1.1340290307998657, "rewards/CLIPTextReward/std": 0.18132489174604416, "rewards/HPSV21TextReward/mean": 0.812891960144043, "rewards/HPSV21TextReward/std": 0.06444458663463593, "rewards/MANIQAReward/mean": 0.9536682963371277, "rewards/MANIQAReward/std": 0.18433940410614014, "rewards/QwenFakeDiscrimReward/mean": 0.1875, "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, "rewards/QwenTextReward/mean": 1.157812476158142, "rewards/QwenTextReward/std": 0.03750002384185791, "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, "step": 75 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.020266666666666665, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0010889782570302486, "learning_rate": 9.990000000000001e-06, "loss": 0.0001, "num_tokens": 1319464.0, "reward": 6.319072246551514, "reward_std": 0.38241666555404663, "rewards/CLIPTextReward/mean": 1.1177687048912048, "rewards/CLIPTextReward/std": 0.20952477306127548, "rewards/HPSV21TextReward/mean": 0.8423194885253906, "rewards/HPSV21TextReward/std": 0.11822888255119324, "rewards/MANIQAReward/mean": 0.8778019547462463, "rewards/MANIQAReward/std": 0.1526302546262741, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03750002756714821, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 76 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.020533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.0011135638924315572, "learning_rate": 9.989866666666667e-06, "loss": 0.0001, "num_tokens": 1336768.0, "reward": 6.1349029541015625, "reward_std": 0.3507642149925232, "rewards/CLIPTextReward/mean": 1.0508588552474976, "rewards/CLIPTextReward/std": 0.19878961890935898, "rewards/HPSV21TextReward/mean": 0.8079948425292969, "rewards/HPSV21TextReward/std": 0.06525566428899765, "rewards/MANIQAReward/mean": 0.985555112361908, "rewards/MANIQAReward/std": 0.15456393361091614, "rewards/QwenFakeDiscrimReward/mean": 0.10546875, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.142578125, "rewards/QwenTextReward/std": 0.032021742314100266, "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, "step": 77 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0208, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0011158613488078117, "learning_rate": 9.989733333333335e-06, "loss": 0.0001, "num_tokens": 1354136.0, "reward": 6.195856094360352, "reward_std": 0.4714621305465698, "rewards/CLIPTextReward/mean": 1.021101474761963, "rewards/CLIPTextReward/std": 0.18839417397975922, "rewards/HPSV21TextReward/mean": 0.8526582717895508, "rewards/HPSV21TextReward/std": 0.13694702461361885, "rewards/MANIQAReward/mean": 0.9741174578666687, "rewards/MANIQAReward/std": 0.16313214600086212, "rewards/QwenFakeDiscrimReward/mean": 0.13671875, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.130468726158142, "rewards/QwenTextReward/std": 0.08550374954938889, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 78 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.021066666666666668, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.0010789756197482347, "learning_rate": 9.9896e-06, "loss": 0.0001, "num_tokens": 1371456.0, "reward": 6.297922134399414, "reward_std": 0.4766380190849304, "rewards/CLIPTextReward/mean": 1.0652101039886475, "rewards/CLIPTextReward/std": 0.1847854182124138, "rewards/HPSV21TextReward/mean": 0.8232393264770508, "rewards/HPSV21TextReward/std": 0.08456060290336609, "rewards/MANIQAReward/mean": 0.9893825650215149, "rewards/MANIQAReward/std": 0.17369645833969116, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.137109398841858, "rewards/QwenTextReward/std": 0.08261503279209137, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 79 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.021333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.001203256892040372, "learning_rate": 9.989466666666668e-06, "loss": 0.0001, "num_tokens": 1388880.0, "reward": 6.433945655822754, "reward_std": 0.40557098388671875, "rewards/CLIPTextReward/mean": 1.121677041053772, "rewards/CLIPTextReward/std": 0.21415946632623672, "rewards/HPSV21TextReward/mean": 0.8016910552978516, "rewards/HPSV21TextReward/std": 0.07152381539344788, "rewards/MANIQAReward/mean": 1.029787302017212, "rewards/MANIQAReward/std": 0.17881350219249725, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0216, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "kl": 0.0010657173115760088, "learning_rate": 9.989333333333334e-06, "loss": 0.0001, "num_tokens": 1406288.0, "reward": 6.374731063842773, "reward_std": 0.355613112449646, "rewards/CLIPTextReward/mean": 1.1012027859687805, "rewards/CLIPTextReward/std": 0.20229313522577286, "rewards/HPSV21TextReward/mean": 0.8576145172119141, "rewards/HPSV21TextReward/std": 0.07686951756477356, "rewards/MANIQAReward/mean": 0.9442054033279419, "rewards/MANIQAReward/std": 0.15360897779464722, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1496093273162842, "rewards/QwenTextReward/std": 0.06275618821382523, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 81 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.021866666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0011162076843902469, "learning_rate": 9.989200000000001e-06, "loss": 0.0001, "num_tokens": 1423504.0, "reward": 6.265508651733398, "reward_std": 0.3726772367954254, "rewards/CLIPTextReward/mean": 1.0828431844711304, "rewards/CLIPTextReward/std": 0.2197393849492073, "rewards/HPSV21TextReward/mean": 0.7947602272033691, "rewards/HPSV21TextReward/std": 0.096592977643013, "rewards/MANIQAReward/mean": 1.0126456022262573, "rewards/MANIQAReward/std": 0.17459847033023834, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.130468726158142, "rewards/QwenTextReward/std": 0.09904994070529938, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 82 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.022133333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.0012128560338169336, "learning_rate": 9.989066666666667e-06, "loss": 0.0001, "num_tokens": 1440856.0, "reward": 6.381606578826904, "reward_std": 0.46134722232818604, "rewards/CLIPTextReward/mean": 1.09823340177536, "rewards/CLIPTextReward/std": 0.21456897258758545, "rewards/HPSV21TextReward/mean": 0.8542604446411133, "rewards/HPSV21TextReward/std": 0.15725306794047356, "rewards/MANIQAReward/mean": 0.9957593083381653, "rewards/MANIQAReward/std": 0.210353285074234, "rewards/QwenFakeDiscrimReward/mean": 0.13671875, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.144921898841858, "rewards/QwenTextReward/std": 0.03338678926229477, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 83 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0224, "frac_reward_zero_std": 0.0, "grad_norm": 1.2734375, "kl": 0.0010785171762108803, "learning_rate": 9.988933333333334e-06, "loss": 0.0001, "num_tokens": 1458104.0, "reward": 6.2626566886901855, "reward_std": 0.36470773816108704, "rewards/CLIPTextReward/mean": 1.0574107766151428, "rewards/CLIPTextReward/std": 0.18804269284009933, "rewards/HPSV21TextReward/mean": 0.8381462097167969, "rewards/HPSV21TextReward/std": 0.10469352453947067, "rewards/MANIQAReward/mean": 0.9285739064216614, "rewards/MANIQAReward/std": 0.17206481099128723, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 84 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.02266666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0012498938012868166, "learning_rate": 9.9888e-06, "loss": 0.0001, "num_tokens": 1475472.0, "reward": 6.434211254119873, "reward_std": 0.44619888067245483, "rewards/CLIPTextReward/mean": 1.1103341579437256, "rewards/CLIPTextReward/std": 0.20621351152658463, "rewards/HPSV21TextReward/mean": 0.8277511596679688, "rewards/HPSV21TextReward/std": 0.12262411415576935, "rewards/MANIQAReward/mean": 1.024056315422058, "rewards/MANIQAReward/std": 0.1818513125181198, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 85 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.022933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0011960228439420462, "learning_rate": 9.988666666666668e-06, "loss": 0.0001, "num_tokens": 1492792.0, "reward": 6.389847755432129, "reward_std": 0.384074866771698, "rewards/CLIPTextReward/mean": 1.0775142908096313, "rewards/CLIPTextReward/std": 0.1948176771402359, "rewards/HPSV21TextReward/mean": 0.8495254516601562, "rewards/HPSV21TextReward/std": 0.15474874898791313, "rewards/MANIQAReward/mean": 0.9799091815948486, "rewards/MANIQAReward/std": 0.15445655584335327, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1496094465255737, "rewards/QwenTextReward/std": 0.03549320250749588, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 86 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0232, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0011886307038366795, "learning_rate": 9.988533333333334e-06, "loss": 0.0001, "num_tokens": 1510184.0, "reward": 6.298969268798828, "reward_std": 0.4068778455257416, "rewards/CLIPTextReward/mean": 1.0830622911453247, "rewards/CLIPTextReward/std": 0.1865195631980896, "rewards/HPSV21TextReward/mean": 0.8122434616088867, "rewards/HPSV21TextReward/std": 0.07124301791191101, "rewards/MANIQAReward/mean": 0.9450761675834656, "rewards/MANIQAReward/std": 0.1854465901851654, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1687500476837158, "rewards/QwenTextReward/std": 0.06900657713413239, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 87 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.023466666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.0013361312448978424, "learning_rate": 9.9884e-06, "loss": 0.0001, "num_tokens": 1527536.0, "reward": 6.399479866027832, "reward_std": 0.4067446291446686, "rewards/CLIPTextReward/mean": 1.0961484909057617, "rewards/CLIPTextReward/std": 0.18726787716150284, "rewards/HPSV21TextReward/mean": 0.8359432220458984, "rewards/HPSV21TextReward/std": 0.09672457724809647, "rewards/MANIQAReward/mean": 0.9806088209152222, "rewards/MANIQAReward/std": 0.15164178609848022, "rewards/QwenFakeDiscrimReward/mean": 0.19921875, "rewards/QwenFakeDiscrimReward/std": 0.10137632489204407, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 88 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.023733333333333332, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.0012429766356945038, "learning_rate": 9.988266666666667e-06, "loss": 0.0001, "num_tokens": 1544928.0, "reward": 6.399818420410156, "reward_std": 0.4257749319076538, "rewards/CLIPTextReward/mean": 1.0604230761528015, "rewards/CLIPTextReward/std": 0.18725504726171494, "rewards/HPSV21TextReward/mean": 0.8571786880493164, "rewards/HPSV21TextReward/std": 0.1451178975403309, "rewards/MANIQAReward/mean": 1.0099272727966309, "rewards/MANIQAReward/std": 0.17368283867835999, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.171875, "rewards/QwenTextReward/std": 0.036596279591321945, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 89 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.024, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.0012510295491665602, "learning_rate": 9.988133333333334e-06, "loss": 0.0001, "num_tokens": 1562248.0, "reward": 6.323746681213379, "reward_std": 0.4743574261665344, "rewards/CLIPTextReward/mean": 1.1159039735794067, "rewards/CLIPTextReward/std": 0.20765963941812515, "rewards/HPSV21TextReward/mean": 0.8270845413208008, "rewards/HPSV21TextReward/std": 0.13083595782518387, "rewards/MANIQAReward/mean": 0.9799572229385376, "rewards/MANIQAReward/std": 0.1780419647693634, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1335937976837158, "rewards/QwenTextReward/std": 0.11736347526311874, "rewards/QwenWeirdDiscrimReward/mean": 0.17578125, "rewards/QwenWeirdDiscrimReward/std": 0.1151231899857521, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.024266666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0012100131716579199, "learning_rate": 9.988000000000002e-06, "loss": 0.0001, "num_tokens": 1579496.0, "reward": 6.273241996765137, "reward_std": 0.5074683427810669, "rewards/CLIPTextReward/mean": 1.0791380405426025, "rewards/CLIPTextReward/std": 0.21002036333084106, "rewards/HPSV21TextReward/mean": 0.8155717849731445, "rewards/HPSV21TextReward/std": 0.08209662139415741, "rewards/MANIQAReward/mean": 0.9748379588127136, "rewards/MANIQAReward/std": 0.16974200308322906, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.1457030773162842, "rewards/QwenTextReward/std": 0.08881437033414841, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 91 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.024533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.0011587683111429214, "learning_rate": 9.987866666666668e-06, "loss": 0.0001, "num_tokens": 1596816.0, "reward": 5.989029407501221, "reward_std": 0.5233784914016724, "rewards/CLIPTextReward/mean": 1.0351091623306274, "rewards/CLIPTextReward/std": 0.1978161856532097, "rewards/HPSV21TextReward/mean": 0.7633829116821289, "rewards/HPSV21TextReward/std": 0.13681436330080032, "rewards/MANIQAReward/mean": 0.9248576164245605, "rewards/MANIQAReward/std": 0.23903001844882965, "rewards/QwenFakeDiscrimReward/mean": 0.1640625, "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, "rewards/QwenTextReward/mean": 1.100000023841858, "rewards/QwenTextReward/std": 0.15634721517562866, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 92 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0248, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0014097951352596283, "learning_rate": 9.987733333333334e-06, "loss": 0.0001, "num_tokens": 1614064.0, "reward": 6.371288299560547, "reward_std": 0.33220386505126953, "rewards/CLIPTextReward/mean": 1.060925841331482, "rewards/CLIPTextReward/std": 0.17794457077980042, "rewards/HPSV21TextReward/mean": 0.8472576141357422, "rewards/HPSV21TextReward/std": 0.07239008694887161, "rewards/MANIQAReward/mean": 0.9830461740493774, "rewards/MANIQAReward/std": 0.1399579495191574, "rewards/QwenFakeDiscrimReward/mean": 0.1953125, "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03750002756714821, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 93 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.025066666666666668, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.0013897167518734932, "learning_rate": 9.9876e-06, "loss": 0.0001, "num_tokens": 1631392.0, "reward": 6.293347358703613, "reward_std": 0.402121365070343, "rewards/CLIPTextReward/mean": 1.0947368144989014, "rewards/CLIPTextReward/std": 0.22335071116685867, "rewards/HPSV21TextReward/mean": 0.8451414108276367, "rewards/HPSV21TextReward/std": 0.09467275440692902, "rewards/MANIQAReward/mean": 0.9737468361854553, "rewards/MANIQAReward/std": 0.15930481255054474, "rewards/QwenFakeDiscrimReward/mean": 0.12890625, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.1156249046325684, "rewards/QwenTextReward/std": 0.11032959818840027, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 94 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.025333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.001365752425044775, "learning_rate": 9.987466666666667e-06, "loss": 0.0001, "num_tokens": 1648864.0, "reward": 6.3488569259643555, "reward_std": 0.37489140033721924, "rewards/CLIPTextReward/mean": 1.1043108701705933, "rewards/CLIPTextReward/std": 0.20488134771585464, "rewards/HPSV21TextReward/mean": 0.7973947525024414, "rewards/HPSV21TextReward/std": 0.057390403002500534, "rewards/MANIQAReward/mean": 1.0102896690368652, "rewards/MANIQAReward/std": 0.12302688509225845, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 95 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0256, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0014814387541264296, "learning_rate": 9.987333333333335e-06, "loss": 0.0001, "num_tokens": 1666120.0, "reward": 6.278581619262695, "reward_std": 0.3662630319595337, "rewards/CLIPTextReward/mean": 1.075215220451355, "rewards/CLIPTextReward/std": 0.20643755793571472, "rewards/HPSV21TextReward/mean": 0.8416833877563477, "rewards/HPSV21TextReward/std": 0.06616079062223434, "rewards/MANIQAReward/mean": 0.9537688493728638, "rewards/MANIQAReward/std": 0.1429898738861084, "rewards/QwenFakeDiscrimReward/mean": 0.12890625, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 96 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.025866666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.001286173122934997, "learning_rate": 9.987200000000002e-06, "loss": 0.0001, "num_tokens": 1683472.0, "reward": 6.097433567047119, "reward_std": 0.4108942747116089, "rewards/CLIPTextReward/mean": 1.0381823182106018, "rewards/CLIPTextReward/std": 0.1961667463183403, "rewards/HPSV21TextReward/mean": 0.8087921142578125, "rewards/HPSV21TextReward/std": 0.11398311704397202, "rewards/MANIQAReward/mean": 0.9116876125335693, "rewards/MANIQAReward/std": 0.1728503406047821, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.1441407203674316, "rewards/QwenTextReward/std": 0.06451010704040527, "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, "step": 97 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.026133333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.390625, "kl": 0.0014894043561071157, "learning_rate": 9.987066666666668e-06, "loss": 0.0001, "num_tokens": 1700776.0, "reward": 6.155697345733643, "reward_std": 0.43312111496925354, "rewards/CLIPTextReward/mean": 1.0117595791816711, "rewards/CLIPTextReward/std": 0.17692303657531738, "rewards/HPSV21TextReward/mean": 0.8317708969116211, "rewards/HPSV21TextReward/std": 0.10141152888536453, "rewards/MANIQAReward/mean": 0.9522302150726318, "rewards/MANIQAReward/std": 0.15004120767116547, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1453125476837158, "rewards/QwenTextReward/std": 0.0670931488275528, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 98 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0264, "frac_reward_zero_std": 0.0, "grad_norm": 1.390625, "kl": 0.0014492514310404658, "learning_rate": 9.986933333333334e-06, "loss": 0.0001, "num_tokens": 1718136.0, "reward": 6.55409049987793, "reward_std": 0.4162108302116394, "rewards/CLIPTextReward/mean": 1.1229727268218994, "rewards/CLIPTextReward/std": 0.18806544691324234, "rewards/HPSV21TextReward/mean": 0.8469762802124023, "rewards/HPSV21TextReward/std": 0.07917948812246323, "rewards/MANIQAReward/mean": 1.0454427003860474, "rewards/MANIQAReward/std": 0.1772715151309967, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1742188930511475, "rewards/QwenTextReward/std": 0.03590354323387146, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 99 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.02666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0017493412597104907, "learning_rate": 9.9868e-06, "loss": 0.0002, "num_tokens": 1735440.0, "reward": 6.363776206970215, "reward_std": 0.3973403573036194, "rewards/CLIPTextReward/mean": 1.0763663053512573, "rewards/CLIPTextReward/std": 0.19003498554229736, "rewards/HPSV21TextReward/mean": 0.8241357803344727, "rewards/HPSV21TextReward/std": 0.07310770452022552, "rewards/MANIQAReward/mean": 1.022928237915039, "rewards/MANIQAReward/std": 0.21806401014328003, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.1453124284744263, "rewards/QwenTextReward/std": 0.0670931413769722, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.026933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0013862024061381817, "learning_rate": 9.986666666666667e-06, "loss": 0.0001, "num_tokens": 1752792.0, "reward": 6.372419834136963, "reward_std": 0.43285655975341797, "rewards/CLIPTextReward/mean": 1.1106175184249878, "rewards/CLIPTextReward/std": 0.19487349689006805, "rewards/HPSV21TextReward/mean": 0.8137311935424805, "rewards/HPSV21TextReward/std": 0.08555622398853302, "rewards/MANIQAReward/mean": 0.944034993648529, "rewards/MANIQAReward/std": 0.1661282777786255, "rewards/QwenFakeDiscrimReward/mean": 0.203125, "rewards/QwenFakeDiscrimReward/std": 0.09834947437047958, "rewards/QwenTextReward/mean": 1.1539063453674316, "rewards/QwenTextReward/std": 0.04633788391947746, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0272, "frac_reward_zero_std": 0.0, "grad_norm": 1.390625, "kl": 0.0015219071647152305, "learning_rate": 9.986533333333333e-06, "loss": 0.0002, "num_tokens": 1770184.0, "reward": 6.147955894470215, "reward_std": 0.3923138380050659, "rewards/CLIPTextReward/mean": 1.0357251167297363, "rewards/CLIPTextReward/std": 0.18429216742515564, "rewards/HPSV21TextReward/mean": 0.8081808090209961, "rewards/HPSV21TextReward/std": 0.09258602559566498, "rewards/MANIQAReward/mean": 1.004284381866455, "rewards/MANIQAReward/std": 0.16934402287006378, "rewards/QwenFakeDiscrimReward/mean": 0.12109375, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.1316406726837158, "rewards/QwenTextReward/std": 0.11164733022451401, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.027466666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0014235489070415497, "learning_rate": 9.9864e-06, "loss": 0.0001, "num_tokens": 1787464.0, "reward": 6.333028793334961, "reward_std": 0.3288081884384155, "rewards/CLIPTextReward/mean": 1.0399454832077026, "rewards/CLIPTextReward/std": 0.20984552800655365, "rewards/HPSV21TextReward/mean": 0.8435735702514648, "rewards/HPSV21TextReward/std": 0.14140213280916214, "rewards/MANIQAReward/mean": 1.0234129428863525, "rewards/MANIQAReward/std": 0.15938599407672882, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1441407203674316, "rewards/QwenTextReward/std": 0.06677703559398651, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 103 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.027733333333333332, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0015638014301657677, "learning_rate": 9.986266666666668e-06, "loss": 0.0002, "num_tokens": 1804784.0, "reward": 6.247958183288574, "reward_std": 0.39279264211654663, "rewards/CLIPTextReward/mean": 1.0843626856803894, "rewards/CLIPTextReward/std": 0.19982705265283585, "rewards/HPSV21TextReward/mean": 0.7990350723266602, "rewards/HPSV21TextReward/std": 0.07866711914539337, "rewards/MANIQAReward/mean": 0.9483500719070435, "rewards/MANIQAReward/std": 0.1665145456790924, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1695313453674316, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.028, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.0015782412374392152, "learning_rate": 9.986133333333334e-06, "loss": 0.0002, "num_tokens": 1822224.0, "reward": 6.490713596343994, "reward_std": 0.43102598190307617, "rewards/CLIPTextReward/mean": 1.1477264761924744, "rewards/CLIPTextReward/std": 0.2057536318898201, "rewards/HPSV21TextReward/mean": 0.8155384063720703, "rewards/HPSV21TextReward/std": 0.05710719898343086, "rewards/MANIQAReward/mean": 1.0063711404800415, "rewards/MANIQAReward/std": 0.17023935914039612, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.1593749523162842, "rewards/QwenTextReward/std": 0.06922188401222229, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.028266666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.0014377952320501208, "learning_rate": 9.986e-06, "loss": 0.0001, "num_tokens": 1839608.0, "reward": 6.183026313781738, "reward_std": 0.32712695002555847, "rewards/CLIPTextReward/mean": 1.0003265738487244, "rewards/CLIPTextReward/std": 0.15935227647423744, "rewards/HPSV21TextReward/mean": 0.8352365493774414, "rewards/HPSV21TextReward/std": 0.0641993060708046, "rewards/MANIQAReward/mean": 0.9435409307479858, "rewards/MANIQAReward/std": 0.15548750758171082, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.166015625, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.028533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.390625, "kl": 0.0015673225279897451, "learning_rate": 9.985866666666667e-06, "loss": 0.0002, "num_tokens": 1857104.0, "reward": 6.220345973968506, "reward_std": 0.3758386969566345, "rewards/CLIPTextReward/mean": 1.0252848863601685, "rewards/CLIPTextReward/std": 0.1825021132826805, "rewards/HPSV21TextReward/mean": 0.8366527557373047, "rewards/HPSV21TextReward/std": 0.07109877467155457, "rewards/MANIQAReward/mean": 0.972642719745636, "rewards/MANIQAReward/std": 0.19886445999145508, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.156640648841858, "rewards/QwenTextReward/std": 0.037332236766815186, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0288, "frac_reward_zero_std": 0.0, "grad_norm": 1.3828125, "kl": 0.0015098307048901916, "learning_rate": 9.985733333333333e-06, "loss": 0.0002, "num_tokens": 1874456.0, "reward": 6.419452667236328, "reward_std": 0.4157279133796692, "rewards/CLIPTextReward/mean": 1.1197414994239807, "rewards/CLIPTextReward/std": 0.20846442133188248, "rewards/HPSV21TextReward/mean": 0.8084573745727539, "rewards/HPSV21TextReward/std": 0.08079753071069717, "rewards/MANIQAReward/mean": 1.0208675861358643, "rewards/MANIQAReward/std": 0.18414068222045898, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1554687023162842, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.029066666666666668, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.0014714575372636318, "learning_rate": 9.985600000000001e-06, "loss": 0.0001, "num_tokens": 1891752.0, "reward": 6.302655220031738, "reward_std": 0.4730131924152374, "rewards/CLIPTextReward/mean": 1.0720114707946777, "rewards/CLIPTextReward/std": 0.2158217504620552, "rewards/HPSV21TextReward/mean": 0.8254232406616211, "rewards/HPSV21TextReward/std": 0.0781259685754776, "rewards/MANIQAReward/mean": 0.9687229990959167, "rewards/MANIQAReward/std": 0.1913435310125351, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.029333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.21875, "kl": 0.0014287796802818775, "learning_rate": 9.985466666666668e-06, "loss": 0.0001, "num_tokens": 1909112.0, "reward": 6.507727146148682, "reward_std": 0.40447428822517395, "rewards/CLIPTextReward/mean": 1.1296402215957642, "rewards/CLIPTextReward/std": 0.199630968272686, "rewards/HPSV21TextReward/mean": 0.830596923828125, "rewards/HPSV21TextReward/std": 0.05249226465821266, "rewards/MANIQAReward/mean": 0.9966280460357666, "rewards/MANIQAReward/std": 0.14694969356060028, "rewards/QwenFakeDiscrimReward/mean": 0.20703125, "rewards/QwenFakeDiscrimReward/std": 0.09506355226039886, "rewards/QwenTextReward/mean": 1.1531250476837158, "rewards/QwenTextReward/std": 0.036596279591321945, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0296, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.0016530327266082168, "learning_rate": 9.985333333333334e-06, "loss": 0.0002, "num_tokens": 1926504.0, "reward": 6.363095760345459, "reward_std": 0.3255675733089447, "rewards/CLIPTextReward/mean": 1.056698203086853, "rewards/CLIPTextReward/std": 0.24591663479804993, "rewards/HPSV21TextReward/mean": 0.8341121673583984, "rewards/HPSV21TextReward/std": 0.13344483822584152, "rewards/MANIQAReward/mean": 1.0506157875061035, "rewards/MANIQAReward/std": 0.15129290521144867, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1519532203674316, "rewards/QwenTextReward/std": 0.03627080097794533, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.029866666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.2734375, "kl": 0.0014856918714940548, "learning_rate": 9.9852e-06, "loss": 0.0001, "num_tokens": 1943824.0, "reward": 6.13232421875, "reward_std": 0.3255869746208191, "rewards/CLIPTextReward/mean": 0.9885252714157104, "rewards/CLIPTextReward/std": 0.1493731364607811, "rewards/HPSV21TextReward/mean": 0.795954704284668, "rewards/HPSV21TextReward/std": 0.08864696323871613, "rewards/MANIQAReward/mean": 1.0371923446655273, "rewards/MANIQAReward/std": 0.1781325787305832, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.030133333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.0016064159572124481, "learning_rate": 9.985066666666667e-06, "loss": 0.0002, "num_tokens": 1961264.0, "reward": 6.490161895751953, "reward_std": 0.4232161045074463, "rewards/CLIPTextReward/mean": 1.1830998063087463, "rewards/CLIPTextReward/std": 0.19798733294010162, "rewards/HPSV21TextReward/mean": 0.8457889556884766, "rewards/HPSV21TextReward/std": 0.06092287600040436, "rewards/MANIQAReward/mean": 0.9694934487342834, "rewards/MANIQAReward/std": 0.16687992215156555, "rewards/QwenFakeDiscrimReward/mean": 0.13671875, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.154296875, "rewards/QwenTextReward/std": 0.03688107803463936, "rewards/QwenWeirdDiscrimReward/mean": 0.171875, "rewards/QwenWeirdDiscrimReward/std": 0.116794154047966, "step": 113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0304, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0017705290811136365, "learning_rate": 9.984933333333335e-06, "loss": 0.0002, "num_tokens": 1978768.0, "reward": 6.421212196350098, "reward_std": 0.4210098087787628, "rewards/CLIPTextReward/mean": 1.1120213866233826, "rewards/CLIPTextReward/std": 0.189268559217453, "rewards/HPSV21TextReward/mean": 0.8761920928955078, "rewards/HPSV21TextReward/std": 0.07564789801836014, "rewards/MANIQAReward/mean": 0.955332338809967, "rewards/MANIQAReward/std": 0.20470233261585236, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1339843273162842, "rewards/QwenTextReward/std": 0.08572373539209366, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.030666666666666665, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "kl": 0.0017124887090176344, "learning_rate": 9.984800000000001e-06, "loss": 0.0002, "num_tokens": 1996080.0, "reward": 6.347210884094238, "reward_std": 0.3971519470214844, "rewards/CLIPTextReward/mean": 1.0920844078063965, "rewards/CLIPTextReward/std": 0.18359429761767387, "rewards/HPSV21TextReward/mean": 0.8116426467895508, "rewards/HPSV21TextReward/std": 0.08479522168636322, "rewards/MANIQAReward/mean": 0.9760847687721252, "rewards/MANIQAReward/std": 0.1531572788953781, "rewards/QwenFakeDiscrimReward/mean": 0.1953125, "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, "rewards/QwenTextReward/mean": 1.1496094465255737, "rewards/QwenTextReward/std": 0.03792543709278107, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.030933333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.001727626658976078, "learning_rate": 9.984666666666668e-06, "loss": 0.0002, "num_tokens": 2013344.0, "reward": 6.798962593078613, "reward_std": 0.4133073687553406, "rewards/CLIPTextReward/mean": 1.1725159883499146, "rewards/CLIPTextReward/std": 0.17704249918460846, "rewards/HPSV21TextReward/mean": 0.8897771835327148, "rewards/HPSV21TextReward/std": 0.15622979402542114, "rewards/MANIQAReward/mean": 1.103672742843628, "rewards/MANIQAReward/std": 0.16740120947360992, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.156640648841858, "rewards/QwenTextReward/std": 0.037332240492105484, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0312, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.001998618245124817, "learning_rate": 9.984533333333334e-06, "loss": 0.0002, "num_tokens": 2030592.0, "reward": 6.627265930175781, "reward_std": 0.36036553978919983, "rewards/CLIPTextReward/mean": 1.1490726470947266, "rewards/CLIPTextReward/std": 0.19648005813360214, "rewards/HPSV21TextReward/mean": 0.8739986419677734, "rewards/HPSV21TextReward/std": 0.06020794063806534, "rewards/MANIQAReward/mean": 0.9924516677856445, "rewards/MANIQAReward/std": 0.18218012154102325, "rewards/QwenFakeDiscrimReward/mean": 0.1953125, "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.031466666666666664, "frac_reward_zero_std": 0.0, "grad_norm": 1.71875, "kl": 0.002251950791105628, "learning_rate": 9.9844e-06, "loss": 0.0002, "num_tokens": 2047896.0, "reward": 6.450213432312012, "reward_std": 0.3416060209274292, "rewards/CLIPTextReward/mean": 1.0652124285697937, "rewards/CLIPTextReward/std": 0.19462762027978897, "rewards/HPSV21TextReward/mean": 0.8843517303466797, "rewards/HPSV21TextReward/std": 0.08924156427383423, "rewards/MANIQAReward/mean": 1.0217883586883545, "rewards/MANIQAReward/std": 0.16342057287693024, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.154296875, "rewards/QwenTextReward/std": 0.03688107803463936, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 118 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.031733333333333336, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.0017585426103323698, "learning_rate": 9.984266666666667e-06, "loss": 0.0002, "num_tokens": 2065256.0, "reward": 6.567963600158691, "reward_std": 0.4625498354434967, "rewards/CLIPTextReward/mean": 1.1237746477127075, "rewards/CLIPTextReward/std": 0.22952871769666672, "rewards/HPSV21TextReward/mean": 0.8930902481079102, "rewards/HPSV21TextReward/std": 0.1436346434056759, "rewards/MANIQAReward/mean": 1.0518121719360352, "rewards/MANIQAReward/std": 0.17778155207633972, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.138671875, "rewards/QwenTextReward/std": 0.08368139714002609, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.032, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.001958254724740982, "learning_rate": 9.984133333333335e-06, "loss": 0.0002, "num_tokens": 2082568.0, "reward": 6.361819267272949, "reward_std": 0.39701226353645325, "rewards/CLIPTextReward/mean": 1.0916123390197754, "rewards/CLIPTextReward/std": 0.2180275395512581, "rewards/HPSV21TextReward/mean": 0.8297672271728516, "rewards/HPSV21TextReward/std": 0.08377574384212494, "rewards/MANIQAReward/mean": 0.9979667663574219, "rewards/MANIQAReward/std": 0.1789645552635193, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03980984166264534, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.032266666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0017855416517704725, "learning_rate": 9.984e-06, "loss": 0.0002, "num_tokens": 2099888.0, "reward": 6.2890801429748535, "reward_std": 0.41400352120399475, "rewards/CLIPTextReward/mean": 1.067662239074707, "rewards/CLIPTextReward/std": 0.2106454148888588, "rewards/HPSV21TextReward/mean": 0.8307352066040039, "rewards/HPSV21TextReward/std": 0.0845264345407486, "rewards/MANIQAReward/mean": 1.0333006381988525, "rewards/MANIQAReward/std": 0.18434347212314606, "rewards/QwenFakeDiscrimReward/mean": 0.12890625, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.123046875, "rewards/QwenTextReward/std": 0.1576303243637085, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 121 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.03253333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.0017559831030666828, "learning_rate": 9.983866666666668e-06, "loss": 0.0002, "num_tokens": 2117280.0, "reward": 6.306989669799805, "reward_std": 0.4608774781227112, "rewards/CLIPTextReward/mean": 1.083207905292511, "rewards/CLIPTextReward/std": 0.20290355384349823, "rewards/HPSV21TextReward/mean": 0.8382225036621094, "rewards/HPSV21TextReward/std": 0.10574927181005478, "rewards/MANIQAReward/mean": 0.9082695245742798, "rewards/MANIQAReward/std": 0.14249961078166962, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1613280773162842, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0328, "frac_reward_zero_std": 0.0, "grad_norm": 1.265625, "kl": 0.0017224764451384544, "learning_rate": 9.983733333333334e-06, "loss": 0.0002, "num_tokens": 2134640.0, "reward": 6.381166458129883, "reward_std": 0.34280797839164734, "rewards/CLIPTextReward/mean": 1.1224886775016785, "rewards/CLIPTextReward/std": 0.18830392882227898, "rewards/HPSV21TextReward/mean": 0.8438854217529297, "rewards/HPSV21TextReward/std": 0.14586665108799934, "rewards/MANIQAReward/mean": 0.9597461819648743, "rewards/MANIQAReward/std": 0.222927525639534, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.160546898841858, "rewards/QwenTextReward/std": 0.07179734110832214, "rewards/QwenWeirdDiscrimReward/mean": 0.16796875, "rewards/QwenWeirdDiscrimReward/std": 0.1183105930685997, "step": 123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.03306666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.001943190349265933, "learning_rate": 9.9836e-06, "loss": 0.0002, "num_tokens": 2151912.0, "reward": 6.494083881378174, "reward_std": 0.41056764125823975, "rewards/CLIPTextReward/mean": 1.1126810908317566, "rewards/CLIPTextReward/std": 0.22240617871284485, "rewards/HPSV21TextReward/mean": 0.8803911209106445, "rewards/HPSV21TextReward/std": 0.1202019639313221, "rewards/MANIQAReward/mean": 0.9934860467910767, "rewards/MANIQAReward/std": 0.1543145328760147, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, "step": 124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.03333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.001764823216944933, "learning_rate": 9.983466666666669e-06, "loss": 0.0002, "num_tokens": 2169232.0, "reward": 6.223010063171387, "reward_std": 0.4870622158050537, "rewards/CLIPTextReward/mean": 1.0433948636054993, "rewards/CLIPTextReward/std": 0.19244755059480667, "rewards/HPSV21TextReward/mean": 0.8162832260131836, "rewards/HPSV21TextReward/std": 0.13273374736309052, "rewards/MANIQAReward/mean": 0.9817787408828735, "rewards/MANIQAReward/std": 0.19965632259845734, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.1351561546325684, "rewards/QwenTextReward/std": 0.08714142441749573, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0336, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "kl": 0.0018342820694670081, "learning_rate": 9.983333333333333e-06, "loss": 0.0002, "num_tokens": 2186584.0, "reward": 6.176717281341553, "reward_std": 0.29012149572372437, "rewards/CLIPTextReward/mean": 1.020053744316101, "rewards/CLIPTextReward/std": 0.17557596415281296, "rewards/HPSV21TextReward/mean": 0.8094825744628906, "rewards/HPSV21TextReward/std": 0.05560995638370514, "rewards/MANIQAReward/mean": 1.0028008222579956, "rewards/MANIQAReward/std": 0.20480003952980042, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1671874523162842, "rewards/QwenTextReward/std": 0.03980984166264534, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.03386666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.002029046881943941, "learning_rate": 9.983200000000001e-06, "loss": 0.0002, "num_tokens": 2203984.0, "reward": 6.353525161743164, "reward_std": 0.39747315645217896, "rewards/CLIPTextReward/mean": 1.0958306789398193, "rewards/CLIPTextReward/std": 0.21575646102428436, "rewards/HPSV21TextReward/mean": 0.8271303176879883, "rewards/HPSV21TextReward/std": 0.07321495562791824, "rewards/MANIQAReward/mean": 0.9904155135154724, "rewards/MANIQAReward/std": 0.17507304251194, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.1617188453674316, "rewards/QwenTextReward/std": 0.042717430740594864, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.034133333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0018549091182649136, "learning_rate": 9.983066666666668e-06, "loss": 0.0002, "num_tokens": 2221312.0, "reward": 6.532737731933594, "reward_std": 0.3147696554660797, "rewards/CLIPTextReward/mean": 1.1676284074783325, "rewards/CLIPTextReward/std": 0.20117784291505814, "rewards/HPSV21TextReward/mean": 0.8389949798583984, "rewards/HPSV21TextReward/std": 0.10873442888259888, "rewards/MANIQAReward/mean": 0.9530848860740662, "rewards/MANIQAReward/std": 0.1483864039182663, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.06404346227645874, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0344, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.002199455164372921, "learning_rate": 9.982933333333334e-06, "loss": 0.0002, "num_tokens": 2238576.0, "reward": 6.328161239624023, "reward_std": 0.32629311084747314, "rewards/CLIPTextReward/mean": 1.0435682535171509, "rewards/CLIPTextReward/std": 0.20102593302726746, "rewards/HPSV21TextReward/mean": 0.8430194854736328, "rewards/HPSV21TextReward/std": 0.11192534118890762, "rewards/MANIQAReward/mean": 0.9631885290145874, "rewards/MANIQAReward/std": 0.18930527567863464, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.166015625, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, "step": 129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.034666666666666665, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0019883974455296993, "learning_rate": 9.9828e-06, "loss": 0.0002, "num_tokens": 2255872.0, "reward": 6.34254264831543, "reward_std": 0.4286326467990875, "rewards/CLIPTextReward/mean": 1.0906871557235718, "rewards/CLIPTextReward/std": 0.19571883976459503, "rewards/HPSV21TextReward/mean": 0.8333959579467773, "rewards/HPSV21TextReward/std": 0.058883652091026306, "rewards/MANIQAReward/mean": 0.9580481052398682, "rewards/MANIQAReward/std": 0.14943087100982666, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.1496093273162842, "rewards/QwenTextReward/std": 0.03792543709278107, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.03493333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0019190080929547548, "learning_rate": 9.982666666666667e-06, "loss": 0.0002, "num_tokens": 2273336.0, "reward": 6.276549339294434, "reward_std": 0.3504009246826172, "rewards/CLIPTextReward/mean": 1.0854588150978088, "rewards/CLIPTextReward/std": 0.20299603044986725, "rewards/HPSV21TextReward/mean": 0.8067693710327148, "rewards/HPSV21TextReward/std": 0.08084672689437866, "rewards/MANIQAReward/mean": 0.9799835085868835, "rewards/MANIQAReward/std": 0.20051877200603485, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1449217796325684, "rewards/QwenTextReward/std": 0.03836432099342346, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0352, "frac_reward_zero_std": 0.0, "grad_norm": 1.3984375, "kl": 0.0020587556064128876, "learning_rate": 9.982533333333335e-06, "loss": 0.0002, "num_tokens": 2290672.0, "reward": 6.436410903930664, "reward_std": 0.41164326667785645, "rewards/CLIPTextReward/mean": 1.102868378162384, "rewards/CLIPTextReward/std": 0.1942010223865509, "rewards/HPSV21TextReward/mean": 0.8459510803222656, "rewards/HPSV21TextReward/std": 0.0887582078576088, "rewards/MANIQAReward/mean": 1.031740665435791, "rewards/MANIQAReward/std": 0.14581337571144104, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.1554687023162842, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.03546666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.002030662726610899, "learning_rate": 9.982400000000001e-06, "loss": 0.0002, "num_tokens": 2307976.0, "reward": 6.364256858825684, "reward_std": 0.4065946042537689, "rewards/CLIPTextReward/mean": 1.0987563729286194, "rewards/CLIPTextReward/std": 0.18764332681894302, "rewards/HPSV21TextReward/mean": 0.8777074813842773, "rewards/HPSV21TextReward/std": 0.13593661412596703, "rewards/MANIQAReward/mean": 0.8976576328277588, "rewards/MANIQAReward/std": 0.15904058516025543, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.166015625, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.171875, "rewards/QwenWeirdDiscrimReward/std": 0.116794154047966, "step": 133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.03573333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.890625, "kl": 0.0023946776054799557, "learning_rate": 9.982266666666668e-06, "loss": 0.0002, "num_tokens": 2325456.0, "reward": 6.729565143585205, "reward_std": 0.3610880970954895, "rewards/CLIPTextReward/mean": 1.1710917353630066, "rewards/CLIPTextReward/std": 0.19065489619970322, "rewards/HPSV21TextReward/mean": 0.8526315689086914, "rewards/HPSV21TextReward/std": 0.0684349313378334, "rewards/MANIQAReward/mean": 1.1067278385162354, "rewards/MANIQAReward/std": 0.13912783563137054, "rewards/QwenFakeDiscrimReward/mean": 0.1875, "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, "rewards/QwenTextReward/mean": 1.1613280773162842, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.036, "frac_reward_zero_std": 0.0, "grad_norm": 1.71875, "kl": 0.0023967409506440163, "learning_rate": 9.982133333333334e-06, "loss": 0.0002, "num_tokens": 2342752.0, "reward": 6.26861572265625, "reward_std": 0.43260377645492554, "rewards/CLIPTextReward/mean": 1.0653359293937683, "rewards/CLIPTextReward/std": 0.2139836996793747, "rewards/HPSV21TextReward/mean": 0.8255853652954102, "rewards/HPSV21TextReward/std": 0.07783577591180801, "rewards/MANIQAReward/mean": 0.9984918236732483, "rewards/MANIQAReward/std": 0.14973679184913635, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.125, "rewards/QwenTextReward/std": 0.14295633137226105, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.03626666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3828125, "kl": 0.002043600659817457, "learning_rate": 9.982e-06, "loss": 0.0002, "num_tokens": 2360144.0, "reward": 6.444212436676025, "reward_std": 0.3863571882247925, "rewards/CLIPTextReward/mean": 1.0651435256004333, "rewards/CLIPTextReward/std": 0.1987275332212448, "rewards/HPSV21TextReward/mean": 0.8277597427368164, "rewards/HPSV21TextReward/std": 0.0786108672618866, "rewards/MANIQAReward/mean": 1.0498123168945312, "rewards/MANIQAReward/std": 0.1474582999944687, "rewards/QwenFakeDiscrimReward/mean": 0.20703125, "rewards/QwenFakeDiscrimReward/std": 0.09506355226039886, "rewards/QwenTextReward/mean": 1.1671874523162842, "rewards/QwenTextReward/std": 0.03750002756714821, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.036533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.004206032492220402, "learning_rate": 9.981866666666667e-06, "loss": 0.0004, "num_tokens": 2377384.0, "reward": 6.348177909851074, "reward_std": 0.47931671142578125, "rewards/CLIPTextReward/mean": 1.1119232177734375, "rewards/CLIPTextReward/std": 0.19578742235898972, "rewards/HPSV21TextReward/mean": 0.8302211761474609, "rewards/HPSV21TextReward/std": 0.11700878292322159, "rewards/MANIQAReward/mean": 0.9732642769813538, "rewards/MANIQAReward/std": 0.1702820360660553, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.1429686546325684, "rewards/QwenTextReward/std": 0.06643842905759811, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0368, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0020041209645569324, "learning_rate": 9.981733333333335e-06, "loss": 0.0002, "num_tokens": 2394704.0, "reward": 6.318357467651367, "reward_std": 0.4292410612106323, "rewards/CLIPTextReward/mean": 1.1253670454025269, "rewards/CLIPTextReward/std": 0.20304681360721588, "rewards/HPSV21TextReward/mean": 0.8212528228759766, "rewards/HPSV21TextReward/std": 0.14200227707624435, "rewards/MANIQAReward/mean": 0.9727740287780762, "rewards/MANIQAReward/std": 0.22543597221374512, "rewards/QwenFakeDiscrimReward/mean": 0.12109375, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.1281249523162842, "rewards/QwenTextReward/std": 0.08351172506809235, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.037066666666666664, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.002243955619633198, "learning_rate": 9.981600000000001e-06, "loss": 0.0002, "num_tokens": 2412072.0, "reward": 6.313478469848633, "reward_std": 0.4881005883216858, "rewards/CLIPTextReward/mean": 1.1092491745948792, "rewards/CLIPTextReward/std": 0.1954825296998024, "rewards/HPSV21TextReward/mean": 0.8195505142211914, "rewards/HPSV21TextReward/std": 0.13129733502864838, "rewards/MANIQAReward/mean": 1.0058796405792236, "rewards/MANIQAReward/std": 0.14454947412014008, "rewards/QwenFakeDiscrimReward/mean": 0.10546875, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.1453124284744263, "rewards/QwenTextReward/std": 0.07661943882703781, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 139 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.037333333333333336, "frac_reward_zero_std": 0.0, "grad_norm": 1.296875, "kl": 0.0019306773319840431, "learning_rate": 9.981466666666668e-06, "loss": 0.0002, "num_tokens": 2429280.0, "reward": 6.358804702758789, "reward_std": 0.42756497859954834, "rewards/CLIPTextReward/mean": 1.078059196472168, "rewards/CLIPTextReward/std": 0.1667657159268856, "rewards/HPSV21TextReward/mean": 0.8231105804443359, "rewards/HPSV21TextReward/std": 0.07944857329130173, "rewards/MANIQAReward/mean": 1.0154496431350708, "rewards/MANIQAReward/std": 0.167838916182518, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.166015625, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0376, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0023419142235070467, "learning_rate": 9.981333333333334e-06, "loss": 0.0002, "num_tokens": 2446632.0, "reward": 6.513288497924805, "reward_std": 0.3195926547050476, "rewards/CLIPTextReward/mean": 1.1330569982528687, "rewards/CLIPTextReward/std": 0.21059855818748474, "rewards/HPSV21TextReward/mean": 0.8392133712768555, "rewards/HPSV21TextReward/std": 0.07069305330514908, "rewards/MANIQAReward/mean": 1.0140602588653564, "rewards/MANIQAReward/std": 0.15958507359027863, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.03772257640957832, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 141 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.037866666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.002120975172147155, "learning_rate": 9.9812e-06, "loss": 0.0002, "num_tokens": 2463936.0, "reward": 6.509091377258301, "reward_std": 0.44019967317581177, "rewards/CLIPTextReward/mean": 1.1297532320022583, "rewards/CLIPTextReward/std": 0.22469547390937805, "rewards/HPSV21TextReward/mean": 0.8484363555908203, "rewards/HPSV21TextReward/std": 0.11415313929319382, "rewards/MANIQAReward/mean": 1.064821720123291, "rewards/MANIQAReward/std": 0.1917555332183838, "rewards/QwenFakeDiscrimReward/mean": 0.13671875, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.163671851158142, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.1875, "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, "step": 142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.03813333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0021487551275640726, "learning_rate": 9.981066666666668e-06, "loss": 0.0002, "num_tokens": 2481352.0, "reward": 6.365453243255615, "reward_std": 0.39431869983673096, "rewards/CLIPTextReward/mean": 1.108916997909546, "rewards/CLIPTextReward/std": 0.19837119430303574, "rewards/HPSV21TextReward/mean": 0.8179616928100586, "rewards/HPSV21TextReward/std": 0.14136216789484024, "rewards/MANIQAReward/mean": 1.0003676414489746, "rewards/MANIQAReward/std": 0.14379417896270752, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.151953101158142, "rewards/QwenTextReward/std": 0.0631992369890213, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0384, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.002176680602133274, "learning_rate": 9.980933333333335e-06, "loss": 0.0002, "num_tokens": 2498640.0, "reward": 6.363863945007324, "reward_std": 0.44721901416778564, "rewards/CLIPTextReward/mean": 1.0735910534858704, "rewards/CLIPTextReward/std": 0.18625196814537048, "rewards/HPSV21TextReward/mean": 0.8158817291259766, "rewards/HPSV21TextReward/std": 0.06047128513455391, "rewards/MANIQAReward/mean": 1.0064024925231934, "rewards/MANIQAReward/std": 0.16212058067321777, "rewards/QwenFakeDiscrimReward/mean": 0.203125, "rewards/QwenFakeDiscrimReward/std": 0.09834947437047958, "rewards/QwenTextReward/mean": 1.148828148841858, "rewards/QwenTextReward/std": 0.06790951639413834, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.03866666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0020736553706228733, "learning_rate": 9.980800000000001e-06, "loss": 0.0002, "num_tokens": 2516112.0, "reward": 6.5203447341918945, "reward_std": 0.4181431531906128, "rewards/CLIPTextReward/mean": 1.1117887496948242, "rewards/CLIPTextReward/std": 0.21418820321559906, "rewards/HPSV21TextReward/mean": 0.8538103103637695, "rewards/HPSV21TextReward/std": 0.10615929216146469, "rewards/MANIQAReward/mean": 1.0532093048095703, "rewards/MANIQAReward/std": 0.18675057590007782, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1453125476837158, "rewards/QwenTextReward/std": 0.0670931488275528, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.038933333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0022541475482285023, "learning_rate": 9.980666666666667e-06, "loss": 0.0002, "num_tokens": 2533504.0, "reward": 6.394295692443848, "reward_std": 0.5058107376098633, "rewards/CLIPTextReward/mean": 1.1201102137565613, "rewards/CLIPTextReward/std": 0.21684333682060242, "rewards/HPSV21TextReward/mean": 0.8399591445922852, "rewards/HPSV21TextReward/std": 0.13806646317243576, "rewards/MANIQAReward/mean": 0.988610565662384, "rewards/MANIQAReward/std": 0.1654188632965088, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1339844465255737, "rewards/QwenTextReward/std": 0.08572372794151306, "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, "step": 146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0392, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0021095983684062958, "learning_rate": 9.980533333333334e-06, "loss": 0.0002, "num_tokens": 2550824.0, "reward": 6.628732204437256, "reward_std": 0.4145030677318573, "rewards/CLIPTextReward/mean": 1.1308990716934204, "rewards/CLIPTextReward/std": 0.19320135563611984, "rewards/HPSV21TextReward/mean": 0.8865270614624023, "rewards/HPSV21TextReward/std": 0.1742558628320694, "rewards/MANIQAReward/mean": 1.0040363073349, "rewards/MANIQAReward/std": 0.1383267045021057, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.171875, "rewards/QwenTextReward/std": 0.03659627586603165, "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, "step": 147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.039466666666666664, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.0022249275352805853, "learning_rate": 9.980400000000002e-06, "loss": 0.0002, "num_tokens": 2568096.0, "reward": 6.243812561035156, "reward_std": 0.39003342390060425, "rewards/CLIPTextReward/mean": 1.0446674823760986, "rewards/CLIPTextReward/std": 0.22870275378227234, "rewards/HPSV21TextReward/mean": 0.7968297004699707, "rewards/HPSV21TextReward/std": 0.09016577154397964, "rewards/MANIQAReward/mean": 1.035818099975586, "rewards/MANIQAReward/std": 0.14643150568008423, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.1421875953674316, "rewards/QwenTextReward/std": 0.08635878562927246, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.039733333333333336, "frac_reward_zero_std": 0.0, "grad_norm": 1.25, "kl": 0.0019969972781836987, "learning_rate": 9.980266666666667e-06, "loss": 0.0002, "num_tokens": 2585368.0, "reward": 6.457371711730957, "reward_std": 0.38209855556488037, "rewards/CLIPTextReward/mean": 1.1367380023002625, "rewards/CLIPTextReward/std": 0.21426362544298172, "rewards/HPSV21TextReward/mean": 0.8346633911132812, "rewards/HPSV21TextReward/std": 0.1631467305123806, "rewards/MANIQAReward/mean": 0.980975329875946, "rewards/MANIQAReward/std": 0.22830736637115479, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.154687523841858, "rewards/QwenTextReward/std": 0.06884465366601944, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 149 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.0022310742642730474, "learning_rate": 9.980133333333335e-06, "loss": 0.0002, "num_tokens": 2603032.0, "reward": 6.2441020011901855, "reward_std": 0.4181395471096039, "rewards/CLIPTextReward/mean": 1.0882028341293335, "rewards/CLIPTextReward/std": 0.21721915900707245, "rewards/HPSV21TextReward/mean": 0.802515983581543, "rewards/HPSV21TextReward/std": 0.08247944712638855, "rewards/MANIQAReward/mean": 0.9611018896102905, "rewards/MANIQAReward/std": 0.15407003462314606, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.142187476158142, "rewards/QwenTextReward/std": 0.0691322535276413, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.040266666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.296875, "kl": 0.002213691361248493, "learning_rate": 9.980000000000001e-06, "loss": 0.0002, "num_tokens": 2620440.0, "reward": 6.366701602935791, "reward_std": 0.38833650946617126, "rewards/CLIPTextReward/mean": 1.1051047444343567, "rewards/CLIPTextReward/std": 0.20750798285007477, "rewards/HPSV21TextReward/mean": 0.8217039108276367, "rewards/HPSV21TextReward/std": 0.07610554248094559, "rewards/MANIQAReward/mean": 0.9685529470443726, "rewards/MANIQAReward/std": 0.18813130259513855, "rewards/QwenFakeDiscrimReward/mean": 0.1640625, "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, "rewards/QwenTextReward/mean": 1.1695313453674316, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04053333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0021160109899938107, "learning_rate": 9.979866666666667e-06, "loss": 0.0002, "num_tokens": 2637816.0, "reward": 6.335112571716309, "reward_std": 0.48140987753868103, "rewards/CLIPTextReward/mean": 1.090110182762146, "rewards/CLIPTextReward/std": 0.20128751546144485, "rewards/HPSV21TextReward/mean": 0.8006181716918945, "rewards/HPSV21TextReward/std": 0.07966043055057526, "rewards/MANIQAReward/mean": 0.9727966785430908, "rewards/MANIQAReward/std": 0.16122058033943176, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.1628906726837158, "rewards/QwenTextReward/std": 0.06929238885641098, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0408, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0022329348139464855, "learning_rate": 9.979733333333335e-06, "loss": 0.0002, "num_tokens": 2655216.0, "reward": 6.241914749145508, "reward_std": 0.4123866558074951, "rewards/CLIPTextReward/mean": 1.0632864832878113, "rewards/CLIPTextReward/std": 0.20508232712745667, "rewards/HPSV21TextReward/mean": 0.8128824234008789, "rewards/HPSV21TextReward/std": 0.07496695220470428, "rewards/MANIQAReward/mean": 0.9583266973495483, "rewards/MANIQAReward/std": 0.14616796374320984, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.12890625, "rewards/QwenTextReward/std": 0.10673908144235611, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04106666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.002218130510300398, "learning_rate": 9.9796e-06, "loss": 0.0002, "num_tokens": 2672568.0, "reward": 6.160798072814941, "reward_std": 0.36988532543182373, "rewards/CLIPTextReward/mean": 1.0214861631393433, "rewards/CLIPTextReward/std": 0.20107388496398926, "rewards/HPSV21TextReward/mean": 0.8351411819458008, "rewards/HPSV21TextReward/std": 0.07012557238340378, "rewards/MANIQAReward/mean": 0.9299652576446533, "rewards/MANIQAReward/std": 0.2092486470937729, "rewards/QwenFakeDiscrimReward/mean": 0.1875, "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, "rewards/QwenTextReward/mean": 1.123046875, "rewards/QwenTextReward/std": 0.08811347186565399, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04133333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.0023776735179126263, "learning_rate": 9.979466666666667e-06, "loss": 0.0002, "num_tokens": 2689960.0, "reward": 6.323511123657227, "reward_std": 0.4532202482223511, "rewards/CLIPTextReward/mean": 1.0690406560897827, "rewards/CLIPTextReward/std": 0.2158183827996254, "rewards/HPSV21TextReward/mean": 0.8493661880493164, "rewards/HPSV21TextReward/std": 0.18082763254642487, "rewards/MANIQAReward/mean": 1.0003697872161865, "rewards/MANIQAReward/std": 0.1615719050168991, "rewards/QwenFakeDiscrimReward/mean": 0.12890625, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.138671875, "rewards/QwenTextReward/std": 0.08716543763875961, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0416, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.002176026813685894, "learning_rate": 9.979333333333335e-06, "loss": 0.0002, "num_tokens": 2707200.0, "reward": 6.3571672439575195, "reward_std": 0.3851667046546936, "rewards/CLIPTextReward/mean": 1.1091200113296509, "rewards/CLIPTextReward/std": 0.1840614676475525, "rewards/HPSV21TextReward/mean": 0.8067169189453125, "rewards/HPSV21TextReward/std": 0.07062850892543793, "rewards/MANIQAReward/mean": 0.9915091395378113, "rewards/MANIQAReward/std": 0.17302347719669342, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04186666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.002383410930633545, "learning_rate": 9.979200000000001e-06, "loss": 0.0002, "num_tokens": 2724544.0, "reward": 6.514974594116211, "reward_std": 0.3913232684135437, "rewards/CLIPTextReward/mean": 1.1650667190551758, "rewards/CLIPTextReward/std": 0.2027941346168518, "rewards/HPSV21TextReward/mean": 0.8568296432495117, "rewards/HPSV21TextReward/std": 0.12092380970716476, "rewards/MANIQAReward/mean": 0.9797755479812622, "rewards/MANIQAReward/std": 0.20642335712909698, "rewards/QwenFakeDiscrimReward/mean": 0.11328125, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.1554687023162842, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.042133333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0022624253761023283, "learning_rate": 9.979066666666667e-06, "loss": 0.0002, "num_tokens": 2741944.0, "reward": 6.376334190368652, "reward_std": 0.4033576250076294, "rewards/CLIPTextReward/mean": 1.0273826122283936, "rewards/CLIPTextReward/std": 0.17035045847296715, "rewards/HPSV21TextReward/mean": 0.8467292785644531, "rewards/HPSV21TextReward/std": 0.12334073334932327, "rewards/MANIQAReward/mean": 1.0710792541503906, "rewards/MANIQAReward/std": 0.1529785394668579, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.162500023841858, "rewards/QwenTextReward/std": 0.03779647499322891, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 158 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0424, "frac_reward_zero_std": 0.0, "grad_norm": 1.296875, "kl": 0.0022776061668992043, "learning_rate": 9.978933333333334e-06, "loss": 0.0002, "num_tokens": 2759200.0, "reward": 6.441936016082764, "reward_std": 0.413552463054657, "rewards/CLIPTextReward/mean": 1.0375493168830872, "rewards/CLIPTextReward/std": 0.18821316957473755, "rewards/HPSV21TextReward/mean": 0.8919734954833984, "rewards/HPSV21TextReward/std": 0.18911729753017426, "rewards/MANIQAReward/mean": 1.0360153913497925, "rewards/MANIQAReward/std": 0.18743227422237396, "rewards/QwenFakeDiscrimReward/mean": 0.1953125, "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, "rewards/QwenTextReward/mean": 1.1484375, "rewards/QwenTextReward/std": 0.0625000149011612, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 159 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.042666666666666665, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "kl": 0.0021952856332063675, "learning_rate": 9.9788e-06, "loss": 0.0002, "num_tokens": 2776488.0, "reward": 6.365789413452148, "reward_std": 0.4353575110435486, "rewards/CLIPTextReward/mean": 1.08657968044281, "rewards/CLIPTextReward/std": 0.17875386402010918, "rewards/HPSV21TextReward/mean": 0.8376073837280273, "rewards/HPSV21TextReward/std": 0.11195782572031021, "rewards/MANIQAReward/mean": 0.9701499342918396, "rewards/MANIQAReward/std": 0.17033298313617706, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1566405296325684, "rewards/QwenTextReward/std": 0.03965182974934578, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04293333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0024832834023982286, "learning_rate": 9.978666666666668e-06, "loss": 0.0002, "num_tokens": 2793872.0, "reward": 6.553257942199707, "reward_std": 0.3911113739013672, "rewards/CLIPTextReward/mean": 1.1585108041763306, "rewards/CLIPTextReward/std": 0.19251200556755066, "rewards/HPSV21TextReward/mean": 0.8372735977172852, "rewards/HPSV21TextReward/std": 0.11238152533769608, "rewards/MANIQAReward/mean": 1.0202827453613281, "rewards/MANIQAReward/std": 0.13903705775737762, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1625001430511475, "rewards/QwenTextReward/std": 0.037796471267938614, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 161 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0432, "frac_reward_zero_std": 0.0, "grad_norm": 1.3828125, "kl": 0.0024976269342005253, "learning_rate": 9.978533333333335e-06, "loss": 0.0002, "num_tokens": 2811160.0, "reward": 6.327694892883301, "reward_std": 0.3723014295101166, "rewards/CLIPTextReward/mean": 1.0655624270439148, "rewards/CLIPTextReward/std": 0.20780667662620544, "rewards/HPSV21TextReward/mean": 0.8458852767944336, "rewards/HPSV21TextReward/std": 0.11304989457130432, "rewards/MANIQAReward/mean": 0.9915183782577515, "rewards/MANIQAReward/std": 0.21026821434497833, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03750002756714821, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04346666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0023513694759458303, "learning_rate": 9.978400000000001e-06, "loss": 0.0002, "num_tokens": 2828504.0, "reward": 6.148275375366211, "reward_std": 0.3822818398475647, "rewards/CLIPTextReward/mean": 1.0270581245422363, "rewards/CLIPTextReward/std": 0.1667494922876358, "rewards/HPSV21TextReward/mean": 0.8215246200561523, "rewards/HPSV21TextReward/std": 0.11641214042901993, "rewards/MANIQAReward/mean": 0.898766279220581, "rewards/MANIQAReward/std": 0.1385030746459961, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03750002756714821, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04373333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.002564219059422612, "learning_rate": 9.978266666666667e-06, "loss": 0.0003, "num_tokens": 2845856.0, "reward": 6.511420726776123, "reward_std": 0.4238984286785126, "rewards/CLIPTextReward/mean": 1.1618956923484802, "rewards/CLIPTextReward/std": 0.19716718792915344, "rewards/HPSV21TextReward/mean": 0.8364629745483398, "rewards/HPSV21TextReward/std": 0.10770930349826813, "rewards/MANIQAReward/mean": 1.012359380722046, "rewards/MANIQAReward/std": 0.15895412862300873, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.142968773841858, "rewards/QwenTextReward/std": 0.06643842905759811, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.044, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0024520137812942266, "learning_rate": 9.978133333333334e-06, "loss": 0.0002, "num_tokens": 2863304.0, "reward": 6.206347465515137, "reward_std": 0.40142396092414856, "rewards/CLIPTextReward/mean": 1.0258528590202332, "rewards/CLIPTextReward/std": 0.21651653200387955, "rewards/HPSV21TextReward/mean": 0.8280124664306641, "rewards/HPSV21TextReward/std": 0.09464716911315918, "rewards/MANIQAReward/mean": 0.970491886138916, "rewards/MANIQAReward/std": 0.13681063055992126, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1648437976837158, "rewards/QwenTextReward/std": 0.040019553154706955, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04426666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.002554597333073616, "learning_rate": 9.978000000000002e-06, "loss": 0.0003, "num_tokens": 2880624.0, "reward": 6.23911190032959, "reward_std": 0.40902307629585266, "rewards/CLIPTextReward/mean": 1.0364893674850464, "rewards/CLIPTextReward/std": 0.1683896705508232, "rewards/HPSV21TextReward/mean": 0.8219265937805176, "rewards/HPSV21TextReward/std": 0.11840753257274628, "rewards/MANIQAReward/mean": 0.9871234893798828, "rewards/MANIQAReward/std": 0.19988511502742767, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1484375, "rewards/QwenTextReward/std": 0.03503827005624771, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.044533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.0023744776844978333, "learning_rate": 9.977866666666668e-06, "loss": 0.0002, "num_tokens": 2898024.0, "reward": 6.0969343185424805, "reward_std": 0.32953351736068726, "rewards/CLIPTextReward/mean": 1.0005998611450195, "rewards/CLIPTextReward/std": 0.17174231261014938, "rewards/HPSV21TextReward/mean": 0.7955303192138672, "rewards/HPSV21TextReward/std": 0.06337456405162811, "rewards/MANIQAReward/mean": 0.971861720085144, "rewards/MANIQAReward/std": 0.14192479848861694, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1460936069488525, "rewards/QwenTextReward/std": 0.03398728370666504, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0448, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.002273611957207322, "learning_rate": 9.977733333333334e-06, "loss": 0.0002, "num_tokens": 2915264.0, "reward": 6.160523414611816, "reward_std": 0.3825192451477051, "rewards/CLIPTextReward/mean": 1.0522464513778687, "rewards/CLIPTextReward/std": 0.20829912275075912, "rewards/HPSV21TextReward/mean": 0.7879104614257812, "rewards/HPSV21TextReward/std": 0.07972346991300583, "rewards/MANIQAReward/mean": 0.9302089810371399, "rewards/MANIQAReward/std": 0.16054274141788483, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1476563215255737, "rewards/QwenTextReward/std": 0.08518407493829727, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.045066666666666665, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.002443203004077077, "learning_rate": 9.9776e-06, "loss": 0.0002, "num_tokens": 2932608.0, "reward": 6.5020294189453125, "reward_std": 0.484386682510376, "rewards/CLIPTextReward/mean": 1.1416503190994263, "rewards/CLIPTextReward/std": 0.1884015128016472, "rewards/HPSV21TextReward/mean": 0.8431720733642578, "rewards/HPSV21TextReward/std": 0.11021529138088226, "rewards/MANIQAReward/mean": 1.0234001874923706, "rewards/MANIQAReward/std": 0.18853236734867096, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1535155773162842, "rewards/QwenTextReward/std": 0.06869927048683167, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 169 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04533333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.296875, "kl": 0.0025329249911010265, "learning_rate": 9.977466666666667e-06, "loss": 0.0003, "num_tokens": 2949912.0, "reward": 6.431511878967285, "reward_std": 0.407115638256073, "rewards/CLIPTextReward/mean": 1.1464118957519531, "rewards/CLIPTextReward/std": 0.20208461582660675, "rewards/HPSV21TextReward/mean": 0.859166145324707, "rewards/HPSV21TextReward/std": 0.12445147335529327, "rewards/MANIQAReward/mean": 0.9699652791023254, "rewards/MANIQAReward/std": 0.1636308878660202, "rewards/QwenFakeDiscrimReward/mean": 0.125, "rewards/QwenFakeDiscrimReward/std": 0.1259881556034088, "rewards/QwenTextReward/mean": 1.1496093273162842, "rewards/QwenTextReward/std": 0.03792543709278107, "rewards/QwenWeirdDiscrimReward/mean": 0.17578125, "rewards/QwenWeirdDiscrimReward/std": 0.1151231899857521, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0456, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.0024203923530876637, "learning_rate": 9.977333333333335e-06, "loss": 0.0002, "num_tokens": 2967104.0, "reward": 6.227117538452148, "reward_std": 0.3750118017196655, "rewards/CLIPTextReward/mean": 1.0488858819007874, "rewards/CLIPTextReward/std": 0.20298413932323456, "rewards/HPSV21TextReward/mean": 0.8152523040771484, "rewards/HPSV21TextReward/std": 0.06391119211912155, "rewards/MANIQAReward/mean": 0.9804813265800476, "rewards/MANIQAReward/std": 0.13435937464237213, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1472656726837158, "rewards/QwenTextReward/std": 0.03453698009252548, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04586666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.002392132766544819, "learning_rate": 9.977200000000002e-06, "loss": 0.0002, "num_tokens": 2984488.0, "reward": 6.098992824554443, "reward_std": 0.41760361194610596, "rewards/CLIPTextReward/mean": 0.9913454055786133, "rewards/CLIPTextReward/std": 0.17980967462062836, "rewards/HPSV21TextReward/mean": 0.8134450912475586, "rewards/HPSV21TextReward/std": 0.08377031236886978, "rewards/MANIQAReward/mean": 0.9780833125114441, "rewards/MANIQAReward/std": 0.20588435232639313, "rewards/QwenFakeDiscrimReward/mean": 0.1640625, "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, "rewards/QwenTextReward/mean": 1.144140601158142, "rewards/QwenTextReward/std": 0.06677703559398651, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04613333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0026148189790546894, "learning_rate": 9.977066666666666e-06, "loss": 0.0003, "num_tokens": 3001728.0, "reward": 6.125906467437744, "reward_std": 0.3209711015224457, "rewards/CLIPTextReward/mean": 0.9812254905700684, "rewards/CLIPTextReward/std": 0.17670557647943497, "rewards/HPSV21TextReward/mean": 0.7952060699462891, "rewards/HPSV21TextReward/std": 0.06432045996189117, "rewards/MANIQAReward/mean": 1.0378872156143188, "rewards/MANIQAReward/std": 0.1683635413646698, "rewards/QwenFakeDiscrimReward/mean": 0.1875, "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, "rewards/QwenTextReward/mean": 1.12890625, "rewards/QwenTextReward/std": 0.09493301808834076, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0464, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.0021870234049856663, "learning_rate": 9.976933333333334e-06, "loss": 0.0002, "num_tokens": 3019096.0, "reward": 6.402133941650391, "reward_std": 0.28885915875434875, "rewards/CLIPTextReward/mean": 1.1439542174339294, "rewards/CLIPTextReward/std": 0.20595373958349228, "rewards/HPSV21TextReward/mean": 0.806889533996582, "rewards/HPSV21TextReward/std": 0.1222410574555397, "rewards/MANIQAReward/mean": 0.9363840818405151, "rewards/MANIQAReward/std": 0.19096361100673676, "rewards/QwenFakeDiscrimReward/mean": 0.1953125, "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03750002756714821, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.00249657966196537, "learning_rate": 9.9768e-06, "loss": 0.0002, "num_tokens": 3036536.0, "reward": 6.335534572601318, "reward_std": 0.4115269184112549, "rewards/CLIPTextReward/mean": 1.0820544362068176, "rewards/CLIPTextReward/std": 0.1995326355099678, "rewards/HPSV21TextReward/mean": 0.8357610702514648, "rewards/HPSV21TextReward/std": 0.07408785820007324, "rewards/MANIQAReward/mean": 0.9698255062103271, "rewards/MANIQAReward/std": 0.20197954773902893, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1472656726837158, "rewards/QwenTextReward/std": 0.06222037598490715, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.046933333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.00256872596219182, "learning_rate": 9.976666666666667e-06, "loss": 0.0003, "num_tokens": 3053840.0, "reward": 6.486705780029297, "reward_std": 0.4940839409828186, "rewards/CLIPTextReward/mean": 1.12992525100708, "rewards/CLIPTextReward/std": 0.19487006962299347, "rewards/HPSV21TextReward/mean": 0.8539524078369141, "rewards/HPSV21TextReward/std": 0.08571457117795944, "rewards/MANIQAReward/mean": 0.9849660396575928, "rewards/MANIQAReward/std": 0.19841398298740387, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1394531726837158, "rewards/QwenTextReward/std": 0.0833100974559784, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0472, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.002719606040045619, "learning_rate": 9.976533333333334e-06, "loss": 0.0003, "num_tokens": 3071208.0, "reward": 6.385272979736328, "reward_std": 0.38871097564697266, "rewards/CLIPTextReward/mean": 1.066335141658783, "rewards/CLIPTextReward/std": 0.20620963722467422, "rewards/HPSV21TextReward/mean": 0.8252353668212891, "rewards/HPSV21TextReward/std": 0.1381026692688465, "rewards/MANIQAReward/mean": 1.0204912424087524, "rewards/MANIQAReward/std": 0.13698863983154297, "rewards/QwenFakeDiscrimReward/mean": 0.19140625, "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, "rewards/QwenTextReward/mean": 1.151953101158142, "rewards/QwenTextReward/std": 0.03627080097794533, "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, "step": 177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.047466666666666664, "frac_reward_zero_std": 0.0, "grad_norm": 1.3984375, "kl": 0.002723751123994589, "learning_rate": 9.9764e-06, "loss": 0.0003, "num_tokens": 3088552.0, "reward": 6.708035469055176, "reward_std": 0.46653276681900024, "rewards/CLIPTextReward/mean": 1.1932567358016968, "rewards/CLIPTextReward/std": 0.2232767716050148, "rewards/HPSV21TextReward/mean": 0.9151687622070312, "rewards/HPSV21TextReward/std": 0.1896015703678131, "rewards/MANIQAReward/mean": 0.9661847949028015, "rewards/MANIQAReward/std": 0.19881469011306763, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.149999976158142, "rewards/QwenTextReward/std": 0.06813853234052658, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 178 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.047733333333333336, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0027056399267166853, "learning_rate": 9.976266666666668e-06, "loss": 0.0003, "num_tokens": 3105920.0, "reward": 6.258008003234863, "reward_std": 0.4084116816520691, "rewards/CLIPTextReward/mean": 1.0594270825386047, "rewards/CLIPTextReward/std": 0.19307219237089157, "rewards/HPSV21TextReward/mean": 0.8365631103515625, "rewards/HPSV21TextReward/std": 0.1051463857293129, "rewards/MANIQAReward/mean": 0.9742305278778076, "rewards/MANIQAReward/std": 0.17492277920246124, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1402344703674316, "rewards/QwenTextReward/std": 0.06002930551767349, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.048, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.002679926110431552, "learning_rate": 9.976133333333334e-06, "loss": 0.0003, "num_tokens": 3123304.0, "reward": 6.29672384262085, "reward_std": 0.35875028371810913, "rewards/CLIPTextReward/mean": 1.0460641980171204, "rewards/CLIPTextReward/std": 0.19920749962329865, "rewards/HPSV21TextReward/mean": 0.8230581283569336, "rewards/HPSV21TextReward/std": 0.07715026289224625, "rewards/MANIQAReward/mean": 1.0112135410308838, "rewards/MANIQAReward/std": 0.1757921427488327, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.1566405296325684, "rewards/QwenTextReward/std": 0.037332236766815186, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.048266666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.0026333339046686888, "learning_rate": 9.976e-06, "loss": 0.0003, "num_tokens": 3140632.0, "reward": 6.529520034790039, "reward_std": 0.37621504068374634, "rewards/CLIPTextReward/mean": 1.1153369545936584, "rewards/CLIPTextReward/std": 0.19007403403520584, "rewards/HPSV21TextReward/mean": 0.8622684478759766, "rewards/HPSV21TextReward/std": 0.056736063212156296, "rewards/MANIQAReward/mean": 1.0161064863204956, "rewards/MANIQAReward/std": 0.17400076985359192, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1519532203674316, "rewards/QwenTextReward/std": 0.03627080097794533, "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, "step": 181 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04853333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.002458418719470501, "learning_rate": 9.975866666666667e-06, "loss": 0.0002, "num_tokens": 3158040.0, "reward": 6.337980270385742, "reward_std": 0.43443620204925537, "rewards/CLIPTextReward/mean": 1.1128054857254028, "rewards/CLIPTextReward/std": 0.2091159000992775, "rewards/HPSV21TextReward/mean": 0.8239784240722656, "rewards/HPSV21TextReward/std": 0.07653089612722397, "rewards/MANIQAReward/mean": 0.9612877368927002, "rewards/MANIQAReward/std": 0.1491847187280655, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.1320312023162842, "rewards/QwenTextReward/std": 0.056250013411045074, "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, "step": 182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0488, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.002701894612982869, "learning_rate": 9.975733333333333e-06, "loss": 0.0003, "num_tokens": 3175392.0, "reward": 6.474644660949707, "reward_std": 0.4059009850025177, "rewards/CLIPTextReward/mean": 1.1510314345359802, "rewards/CLIPTextReward/std": 0.20656609535217285, "rewards/HPSV21TextReward/mean": 0.8551139831542969, "rewards/HPSV21TextReward/std": 0.1395208202302456, "rewards/MANIQAReward/mean": 0.9021977186203003, "rewards/MANIQAReward/std": 0.2259361743927002, "rewards/QwenFakeDiscrimReward/mean": 0.19921875, "rewards/QwenFakeDiscrimReward/std": 0.10137632489204407, "rewards/QwenTextReward/mean": 1.1343750953674316, "rewards/QwenTextReward/std": 0.08813546597957611, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 183 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04906666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.002543346956372261, "learning_rate": 9.975600000000002e-06, "loss": 0.0003, "num_tokens": 3192720.0, "reward": 6.173674583435059, "reward_std": 0.3256201446056366, "rewards/CLIPTextReward/mean": 1.0212828516960144, "rewards/CLIPTextReward/std": 0.15655865147709846, "rewards/HPSV21TextReward/mean": 0.8296718597412109, "rewards/HPSV21TextReward/std": 0.05253772437572479, "rewards/MANIQAReward/mean": 0.9475463032722473, "rewards/MANIQAReward/std": 0.1381036341190338, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1648437976837158, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, "step": 184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.421875, "kl": 0.00288868835195899, "learning_rate": 9.975466666666668e-06, "loss": 0.0003, "num_tokens": 3210160.0, "reward": 6.346151828765869, "reward_std": 0.27956622838974, "rewards/CLIPTextReward/mean": 1.0762921571731567, "rewards/CLIPTextReward/std": 0.2376992329955101, "rewards/HPSV21TextReward/mean": 0.7927217483520508, "rewards/HPSV21TextReward/std": 0.06304909288883209, "rewards/MANIQAReward/mean": 1.0503114461898804, "rewards/MANIQAReward/std": 0.1577564924955368, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1593749523162842, "rewards/QwenTextReward/std": 0.06922188401222229, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0496, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.0028415676206350327, "learning_rate": 9.975333333333334e-06, "loss": 0.0003, "num_tokens": 3227456.0, "reward": 6.274043083190918, "reward_std": 0.3639392852783203, "rewards/CLIPTextReward/mean": 1.0660734176635742, "rewards/CLIPTextReward/std": 0.1816587634384632, "rewards/HPSV21TextReward/mean": 0.8182039260864258, "rewards/HPSV21TextReward/std": 0.07657317072153091, "rewards/MANIQAReward/mean": 0.9797074198722839, "rewards/MANIQAReward/std": 0.1761608123779297, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1625001430511475, "rewards/QwenTextReward/std": 0.037796471267938614, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.04986666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0028599947690963745, "learning_rate": 9.9752e-06, "loss": 0.0003, "num_tokens": 3244976.0, "reward": 6.46965217590332, "reward_std": 0.37160903215408325, "rewards/CLIPTextReward/mean": 1.1449741125106812, "rewards/CLIPTextReward/std": 0.1829175353050232, "rewards/HPSV21TextReward/mean": 0.8212175369262695, "rewards/HPSV21TextReward/std": 0.05136280506849289, "rewards/MANIQAReward/mean": 1.021644115447998, "rewards/MANIQAReward/std": 0.1607440710067749, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.140625, "rewards/QwenTextReward/std": 0.07420212030410767, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.050133333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.002837129170075059, "learning_rate": 9.975066666666667e-06, "loss": 0.0003, "num_tokens": 3262384.0, "reward": 6.557543754577637, "reward_std": 0.4035520851612091, "rewards/CLIPTextReward/mean": 1.1948737502098083, "rewards/CLIPTextReward/std": 0.16932519897818565, "rewards/HPSV21TextReward/mean": 0.8404254913330078, "rewards/HPSV21TextReward/std": 0.12645912915468216, "rewards/MANIQAReward/mean": 1.0502266883850098, "rewards/MANIQAReward/std": 0.14965708553791046, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1359374523162842, "rewards/QwenTextReward/std": 0.09808434545993805, "rewards/QwenWeirdDiscrimReward/mean": 0.15234375, "rewards/QwenWeirdDiscrimReward/std": 0.12293684482574463, "step": 188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0504, "frac_reward_zero_std": 0.0, "grad_norm": 1.3828125, "kl": 0.002935183933004737, "learning_rate": 9.974933333333335e-06, "loss": 0.0003, "num_tokens": 3279672.0, "reward": 6.320755958557129, "reward_std": 0.327908992767334, "rewards/CLIPTextReward/mean": 1.0910344123840332, "rewards/CLIPTextReward/std": 0.19866855442523956, "rewards/HPSV21TextReward/mean": 0.8061733245849609, "rewards/HPSV21TextReward/std": 0.07433304190635681, "rewards/MANIQAReward/mean": 0.9818093180656433, "rewards/MANIQAReward/std": 0.1730826199054718, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03750002384185791, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.050666666666666665, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.002721617929637432, "learning_rate": 9.974800000000001e-06, "loss": 0.0003, "num_tokens": 3296968.0, "reward": 6.403863906860352, "reward_std": 0.4016572833061218, "rewards/CLIPTextReward/mean": 1.0907533168792725, "rewards/CLIPTextReward/std": 0.2178114652633667, "rewards/HPSV21TextReward/mean": 0.8505439758300781, "rewards/HPSV21TextReward/std": 0.09670589119195938, "rewards/MANIQAReward/mean": 1.00994074344635, "rewards/MANIQAReward/std": 0.15439607203006744, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.1675782203674316, "rewards/QwenTextReward/std": 0.0424170047044754, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05093333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0026723656337708235, "learning_rate": 9.974666666666666e-06, "loss": 0.0003, "num_tokens": 3314224.0, "reward": 6.2754364013671875, "reward_std": 0.3998555541038513, "rewards/CLIPTextReward/mean": 1.1050440073013306, "rewards/CLIPTextReward/std": 0.19693446159362793, "rewards/HPSV21TextReward/mean": 0.8342494964599609, "rewards/HPSV21TextReward/std": 0.07819408923387527, "rewards/MANIQAReward/mean": 0.923412024974823, "rewards/MANIQAReward/std": 0.17528018355369568, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.1414062976837158, "rewards/QwenTextReward/std": 0.06045772135257721, "rewards/QwenWeirdDiscrimReward/mean": 0.1875, "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, "step": 191 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0512, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.002696257084608078, "learning_rate": 9.974533333333334e-06, "loss": 0.0003, "num_tokens": 3331568.0, "reward": 6.29323673248291, "reward_std": 0.2823619246482849, "rewards/CLIPTextReward/mean": 1.008976697921753, "rewards/CLIPTextReward/std": 0.16197307035326958, "rewards/HPSV21TextReward/mean": 0.8356037139892578, "rewards/HPSV21TextReward/std": 0.06890447437763214, "rewards/MANIQAReward/mean": 1.03376305103302, "rewards/MANIQAReward/std": 0.15510410070419312, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, "step": 192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05146666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.296875, "kl": 0.002745483536273241, "learning_rate": 9.9744e-06, "loss": 0.0003, "num_tokens": 3349128.0, "reward": 6.267972946166992, "reward_std": 0.3895288109779358, "rewards/CLIPTextReward/mean": 1.0269521474838257, "rewards/CLIPTextReward/std": 0.1757335141301155, "rewards/HPSV21TextReward/mean": 0.823420524597168, "rewards/HPSV21TextReward/std": 0.07999104261398315, "rewards/MANIQAReward/mean": 1.050821304321289, "rewards/MANIQAReward/std": 0.18627648055553436, "rewards/QwenFakeDiscrimReward/mean": 0.19140625, "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, "rewards/QwenTextReward/mean": 1.1257812976837158, "rewards/QwenTextReward/std": 0.10331498086452484, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05173333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.002754206769168377, "learning_rate": 9.974266666666667e-06, "loss": 0.0003, "num_tokens": 3366480.0, "reward": 6.110285758972168, "reward_std": 0.38404354453086853, "rewards/CLIPTextReward/mean": 0.9982666969299316, "rewards/CLIPTextReward/std": 0.1378244236111641, "rewards/HPSV21TextReward/mean": 0.8200778961181641, "rewards/HPSV21TextReward/std": 0.07046207785606384, "rewards/MANIQAReward/mean": 0.9497687816619873, "rewards/MANIQAReward/std": 0.14510276913642883, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.1292967796325684, "rewards/QwenTextReward/std": 0.08220121264457703, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.052, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0027757398784160614, "learning_rate": 9.974133333333335e-06, "loss": 0.0003, "num_tokens": 3383832.0, "reward": 6.2371015548706055, "reward_std": 0.40047281980514526, "rewards/CLIPTextReward/mean": 1.050218403339386, "rewards/CLIPTextReward/std": 0.2267727330327034, "rewards/HPSV21TextReward/mean": 0.8354701995849609, "rewards/HPSV21TextReward/std": 0.07300396263599396, "rewards/MANIQAReward/mean": 0.9864278435707092, "rewards/MANIQAReward/std": 0.16592912375926971, "rewards/QwenFakeDiscrimReward/mean": 0.1328125, "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, "rewards/QwenTextReward/mean": 1.1511719226837158, "rewards/QwenTextReward/std": 0.06834637373685837, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05226666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.002869168994948268, "learning_rate": 9.974e-06, "loss": 0.0003, "num_tokens": 3401176.0, "reward": 6.375538349151611, "reward_std": 0.3194682002067566, "rewards/CLIPTextReward/mean": 1.1040527820587158, "rewards/CLIPTextReward/std": 0.21156348288059235, "rewards/HPSV21TextReward/mean": 0.8256998062133789, "rewards/HPSV21TextReward/std": 0.06539477407932281, "rewards/MANIQAReward/mean": 0.9972831010818481, "rewards/MANIQAReward/std": 0.18009570240974426, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1554687023162842, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.052533333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.002735574496909976, "learning_rate": 9.973866666666668e-06, "loss": 0.0003, "num_tokens": 3418504.0, "reward": 6.325824737548828, "reward_std": 0.38276347517967224, "rewards/CLIPTextReward/mean": 1.0841413140296936, "rewards/CLIPTextReward/std": 0.19735275954008102, "rewards/HPSV21TextReward/mean": 0.8420505523681641, "rewards/HPSV21TextReward/std": 0.08244600147008896, "rewards/MANIQAReward/mean": 0.967581570148468, "rewards/MANIQAReward/std": 0.18143029510974884, "rewards/QwenFakeDiscrimReward/mean": 0.1328125, "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, "rewards/QwenTextReward/mean": 1.138671875, "rewards/QwenTextReward/std": 0.14412638545036316, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0528, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.002685937564820051, "learning_rate": 9.973733333333334e-06, "loss": 0.0003, "num_tokens": 3435840.0, "reward": 6.567035675048828, "reward_std": 0.469005823135376, "rewards/CLIPTextReward/mean": 1.124721884727478, "rewards/CLIPTextReward/std": 0.2038327008485794, "rewards/HPSV21TextReward/mean": 0.8921480178833008, "rewards/HPSV21TextReward/std": 0.1626519188284874, "rewards/MANIQAReward/mean": 0.9938427209854126, "rewards/MANIQAReward/std": 0.15762895345687866, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1566405296325684, "rewards/QwenTextReward/std": 0.037332236766815186, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.053066666666666665, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0027355619240552187, "learning_rate": 9.9736e-06, "loss": 0.0003, "num_tokens": 3453160.0, "reward": 6.437889099121094, "reward_std": 0.38492536544799805, "rewards/CLIPTextReward/mean": 1.1193333864212036, "rewards/CLIPTextReward/std": 0.1936298832297325, "rewards/HPSV21TextReward/mean": 0.8576078414916992, "rewards/HPSV21TextReward/std": 0.13272829726338387, "rewards/MANIQAReward/mean": 0.9707251787185669, "rewards/MANIQAReward/std": 0.16395625472068787, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1460936069488525, "rewards/QwenTextReward/std": 0.03398728370666504, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.002614394761621952, "learning_rate": 9.973466666666669e-06, "loss": 0.0003, "num_tokens": 3470480.0, "reward": 6.575470447540283, "reward_std": 0.3986605405807495, "rewards/CLIPTextReward/mean": 1.113551139831543, "rewards/CLIPTextReward/std": 0.20553741604089737, "rewards/HPSV21TextReward/mean": 0.8867435455322266, "rewards/HPSV21TextReward/std": 0.1364203877747059, "rewards/MANIQAReward/mean": 1.0623810291290283, "rewards/MANIQAReward/std": 0.1529991775751114, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1531250476837158, "rewards/QwenTextReward/std": 0.036596279591321945, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0536, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.002905803034082055, "learning_rate": 9.973333333333333e-06, "loss": 0.0003, "num_tokens": 3487880.0, "reward": 6.478170394897461, "reward_std": 0.49340736865997314, "rewards/CLIPTextReward/mean": 1.1195530891418457, "rewards/CLIPTextReward/std": 0.20537645369768143, "rewards/HPSV21TextReward/mean": 0.831049919128418, "rewards/HPSV21TextReward/std": 0.08279094845056534, "rewards/MANIQAReward/mean": 1.0441515445709229, "rewards/MANIQAReward/std": 0.15340273082256317, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.1460938453674316, "rewards/QwenTextReward/std": 0.061916936188936234, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 201 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05386666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.002745371311903, "learning_rate": 9.973200000000001e-06, "loss": 0.0003, "num_tokens": 3505312.0, "reward": 6.431427001953125, "reward_std": 0.33977192640304565, "rewards/CLIPTextReward/mean": 1.1148839592933655, "rewards/CLIPTextReward/std": 0.2095688208937645, "rewards/HPSV21TextReward/mean": 0.8343162536621094, "rewards/HPSV21TextReward/std": 0.08807380497455597, "rewards/MANIQAReward/mean": 0.9779485464096069, "rewards/MANIQAReward/std": 0.15203088521957397, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1449220180511475, "rewards/QwenTextReward/std": 0.06158936396241188, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05413333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.002616970334202051, "learning_rate": 9.973066666666668e-06, "loss": 0.0003, "num_tokens": 3522752.0, "reward": 6.270486831665039, "reward_std": 0.3961990475654602, "rewards/CLIPTextReward/mean": 1.089812994003296, "rewards/CLIPTextReward/std": 0.2142699658870697, "rewards/HPSV21TextReward/mean": 0.7937088012695312, "rewards/HPSV21TextReward/std": 0.0994756817817688, "rewards/MANIQAReward/mean": 0.9725837707519531, "rewards/MANIQAReward/std": 0.14475348591804504, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1519532203674316, "rewards/QwenTextReward/std": 0.0386541411280632, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0544, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.0029000602662563324, "learning_rate": 9.972933333333334e-06, "loss": 0.0003, "num_tokens": 3540184.0, "reward": 6.369531154632568, "reward_std": 0.3710113763809204, "rewards/CLIPTextReward/mean": 1.1070588827133179, "rewards/CLIPTextReward/std": 0.2002292200922966, "rewards/HPSV21TextReward/mean": 0.7954778671264648, "rewards/HPSV21TextReward/std": 0.06739770621061325, "rewards/MANIQAReward/mean": 1.0457077026367188, "rewards/MANIQAReward/std": 0.12836866080760956, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1554687023162842, "rewards/QwenTextReward/std": 0.03945784643292427, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05466666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.4375, "kl": 0.0038333996199071407, "learning_rate": 9.972800000000002e-06, "loss": 0.0004, "num_tokens": 3557584.0, "reward": 6.196996688842773, "reward_std": 0.3954958915710449, "rewards/CLIPTextReward/mean": 1.0345997214317322, "rewards/CLIPTextReward/std": 0.20499616861343384, "rewards/HPSV21TextReward/mean": 0.7918825149536133, "rewards/HPSV21TextReward/std": 0.0736343041062355, "rewards/MANIQAReward/mean": 1.0256726741790771, "rewards/MANIQAReward/std": 0.1692875772714615, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1394531726837158, "rewards/QwenTextReward/std": 0.0833100974559784, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.054933333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.0026570966001600027, "learning_rate": 9.972666666666667e-06, "loss": 0.0003, "num_tokens": 3574896.0, "reward": 6.4823150634765625, "reward_std": 0.40404587984085083, "rewards/CLIPTextReward/mean": 1.1078510284423828, "rewards/CLIPTextReward/std": 0.20066715776920319, "rewards/HPSV21TextReward/mean": 0.846653938293457, "rewards/HPSV21TextReward/std": 0.12554003298282623, "rewards/MANIQAReward/mean": 1.0229145288467407, "rewards/MANIQAReward/std": 0.17728054523468018, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1636719703674316, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0552, "frac_reward_zero_std": 0.0, "grad_norm": 1.8046875, "kl": 0.002860800828784704, "learning_rate": 9.972533333333335e-06, "loss": 0.0003, "num_tokens": 3592240.0, "reward": 6.497309684753418, "reward_std": 0.4115002751350403, "rewards/CLIPTextReward/mean": 1.1606194376945496, "rewards/CLIPTextReward/std": 0.195783831179142, "rewards/HPSV21TextReward/mean": 0.84130859375, "rewards/HPSV21TextReward/std": 0.11015837639570236, "rewards/MANIQAReward/mean": 0.9379844665527344, "rewards/MANIQAReward/std": 0.17234882712364197, "rewards/QwenFakeDiscrimReward/mean": 0.1953125, "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, "rewards/QwenTextReward/mean": 1.1531250476837158, "rewards/QwenTextReward/std": 0.03659627586603165, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.055466666666666664, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.003817642340436578, "learning_rate": 9.972400000000001e-06, "loss": 0.0004, "num_tokens": 3609680.0, "reward": 6.229999542236328, "reward_std": 0.3975546360015869, "rewards/CLIPTextReward/mean": 1.076364815235138, "rewards/CLIPTextReward/std": 0.21021181344985962, "rewards/HPSV21TextReward/mean": 0.8293190002441406, "rewards/HPSV21TextReward/std": 0.08222578465938568, "rewards/MANIQAReward/mean": 0.9467570781707764, "rewards/MANIQAReward/std": 0.14959219098091125, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.1085937023162842, "rewards/QwenTextReward/std": 0.1296529918909073, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.055733333333333336, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.002774016000330448, "learning_rate": 9.972266666666668e-06, "loss": 0.0003, "num_tokens": 3626928.0, "reward": 6.374789237976074, "reward_std": 0.36420223116874695, "rewards/CLIPTextReward/mean": 1.0703842639923096, "rewards/CLIPTextReward/std": 0.21086003631353378, "rewards/HPSV21TextReward/mean": 0.8615245819091797, "rewards/HPSV21TextReward/std": 0.05977732688188553, "rewards/MANIQAReward/mean": 0.9527685642242432, "rewards/MANIQAReward/std": 0.15766838192939758, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.163671851158142, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 209 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.056, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.002732650376856327, "learning_rate": 9.972133333333334e-06, "loss": 0.0003, "num_tokens": 3644248.0, "reward": 6.305462837219238, "reward_std": 0.456421822309494, "rewards/CLIPTextReward/mean": 1.0700019001960754, "rewards/CLIPTextReward/std": 0.1975710168480873, "rewards/HPSV21TextReward/mean": 0.8302736282348633, "rewards/HPSV21TextReward/std": 0.0959872305393219, "rewards/MANIQAReward/mean": 0.992802619934082, "rewards/MANIQAReward/std": 0.15043874084949493, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.144921898841858, "rewards/QwenTextReward/std": 0.06158936396241188, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.056266666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.002957995980978012, "learning_rate": 9.972e-06, "loss": 0.0003, "num_tokens": 3661552.0, "reward": 6.458515167236328, "reward_std": 0.352350115776062, "rewards/CLIPTextReward/mean": 1.1411879062652588, "rewards/CLIPTextReward/std": 0.20142141729593277, "rewards/HPSV21TextReward/mean": 0.8212127685546875, "rewards/HPSV21TextReward/std": 0.0886462852358818, "rewards/MANIQAReward/mean": 1.009104609489441, "rewards/MANIQAReward/std": 0.15248696506023407, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1417968273162842, "rewards/QwenTextReward/std": 0.06607697904109955, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05653333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0038779843598604202, "learning_rate": 9.971866666666667e-06, "loss": 0.0004, "num_tokens": 3678864.0, "reward": 6.417540550231934, "reward_std": 0.43140143156051636, "rewards/CLIPTextReward/mean": 1.106486737728119, "rewards/CLIPTextReward/std": 0.20730093866586685, "rewards/HPSV21TextReward/mean": 0.863398551940918, "rewards/HPSV21TextReward/std": 0.06245674192905426, "rewards/MANIQAReward/mean": 1.007067322731018, "rewards/MANIQAReward/std": 0.2022874653339386, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.130859375, "rewards/QwenTextReward/std": 0.07595193386077881, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 212 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0568, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.003234908450394869, "learning_rate": 9.971733333333335e-06, "loss": 0.0003, "num_tokens": 3696200.0, "reward": 6.502376079559326, "reward_std": 0.4503940939903259, "rewards/CLIPTextReward/mean": 1.117896556854248, "rewards/CLIPTextReward/std": 0.2203962802886963, "rewards/HPSV21TextReward/mean": 0.8834381103515625, "rewards/HPSV21TextReward/std": 0.1342969425022602, "rewards/MANIQAReward/mean": 1.007910132408142, "rewards/MANIQAReward/std": 0.16741357743740082, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1636719703674316, "rewards/QwenTextReward/std": 0.04007181152701378, "rewards/QwenWeirdDiscrimReward/mean": 0.1796875, "rewards/QwenWeirdDiscrimReward/std": 0.11329087615013123, "step": 213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05706666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0028942429926246405, "learning_rate": 9.971600000000001e-06, "loss": 0.0003, "num_tokens": 3713480.0, "reward": 6.227326393127441, "reward_std": 0.3156473636627197, "rewards/CLIPTextReward/mean": 1.0346298217773438, "rewards/CLIPTextReward/std": 0.18641937524080276, "rewards/HPSV21TextReward/mean": 0.836186408996582, "rewards/HPSV21TextReward/std": 0.10609928518533707, "rewards/MANIQAReward/mean": 0.9368653297424316, "rewards/MANIQAReward/std": 0.18543748557567596, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.154296875, "rewards/QwenTextReward/std": 0.039227355271577835, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05733333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0032178538385778666, "learning_rate": 9.971466666666668e-06, "loss": 0.0003, "num_tokens": 3730968.0, "reward": 6.278270721435547, "reward_std": 0.5345197916030884, "rewards/CLIPTextReward/mean": 1.1357209086418152, "rewards/CLIPTextReward/std": 0.19665353745222092, "rewards/HPSV21TextReward/mean": 0.7758393287658691, "rewards/HPSV21TextReward/std": 0.07874829322099686, "rewards/MANIQAReward/mean": 0.9817126393318176, "rewards/MANIQAReward/std": 0.1834738850593567, "rewards/QwenFakeDiscrimReward/mean": 0.10546875, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.1570312976837158, "rewards/QwenTextReward/std": 0.042367640882730484, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0576, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0027981791645288467, "learning_rate": 9.971333333333334e-06, "loss": 0.0003, "num_tokens": 3748240.0, "reward": 6.433413028717041, "reward_std": 0.35207629203796387, "rewards/CLIPTextReward/mean": 1.0457903146743774, "rewards/CLIPTextReward/std": 0.1923365518450737, "rewards/HPSV21TextReward/mean": 0.8758325576782227, "rewards/HPSV21TextReward/std": 0.1333918683230877, "rewards/MANIQAReward/mean": 1.0339173078536987, "rewards/MANIQAReward/std": 0.15527579188346863, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03750002384185791, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.057866666666666663, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0031056725420057774, "learning_rate": 9.9712e-06, "loss": 0.0003, "num_tokens": 3765528.0, "reward": 6.364522933959961, "reward_std": 0.3756159543991089, "rewards/CLIPTextReward/mean": 1.0495604276657104, "rewards/CLIPTextReward/std": 0.17222807928919792, "rewards/HPSV21TextReward/mean": 0.8294382095336914, "rewards/HPSV21TextReward/std": 0.07039421051740646, "rewards/MANIQAReward/mean": 1.0276196002960205, "rewards/MANIQAReward/std": 0.21525166928768158, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1648437976837158, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, "step": 217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.058133333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0027923269663006067, "learning_rate": 9.971066666666668e-06, "loss": 0.0003, "num_tokens": 3783032.0, "reward": 6.405855178833008, "reward_std": 0.3659675717353821, "rewards/CLIPTextReward/mean": 1.1177189350128174, "rewards/CLIPTextReward/std": 0.198662169277668, "rewards/HPSV21TextReward/mean": 0.8444490432739258, "rewards/HPSV21TextReward/std": 0.07149509340524673, "rewards/MANIQAReward/mean": 0.9584724307060242, "rewards/MANIQAReward/std": 0.1412878930568695, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.151953101158142, "rewards/QwenTextReward/std": 0.03627080097794533, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0584, "frac_reward_zero_std": 0.0, "grad_norm": 1.4296875, "kl": 0.003411591053009033, "learning_rate": 9.970933333333335e-06, "loss": 0.0003, "num_tokens": 3800272.0, "reward": 6.541419506072998, "reward_std": 0.35087212920188904, "rewards/CLIPTextReward/mean": 1.1012699604034424, "rewards/CLIPTextReward/std": 0.1799757182598114, "rewards/HPSV21TextReward/mean": 0.8397064208984375, "rewards/HPSV21TextReward/std": 0.11745796352624893, "rewards/MANIQAReward/mean": 1.0981385707855225, "rewards/MANIQAReward/std": 0.14114679396152496, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.058666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0031459140591323376, "learning_rate": 9.970800000000001e-06, "loss": 0.0003, "num_tokens": 3817560.0, "reward": 6.071300029754639, "reward_std": 0.4156203866004944, "rewards/CLIPTextReward/mean": 1.0444402694702148, "rewards/CLIPTextReward/std": 0.20825284719467163, "rewards/HPSV21TextReward/mean": 0.8236780166625977, "rewards/HPSV21TextReward/std": 0.06731405109167099, "rewards/MANIQAReward/mean": 0.9178757667541504, "rewards/MANIQAReward/std": 0.142966166138649, "rewards/QwenFakeDiscrimReward/mean": 0.125, "rewards/QwenFakeDiscrimReward/std": 0.1259881556034088, "rewards/QwenTextReward/mean": 1.1203124523162842, "rewards/QwenTextReward/std": 0.12286874651908875, "rewards/QwenWeirdDiscrimReward/mean": 0.171875, "rewards/QwenWeirdDiscrimReward/std": 0.116794154047966, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05893333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.003471190808340907, "learning_rate": 9.970666666666668e-06, "loss": 0.0003, "num_tokens": 3834912.0, "reward": 6.415256500244141, "reward_std": 0.3701218366622925, "rewards/CLIPTextReward/mean": 1.1057547330856323, "rewards/CLIPTextReward/std": 0.20841475576162338, "rewards/HPSV21TextReward/mean": 0.8342828750610352, "rewards/HPSV21TextReward/std": 0.08610900491476059, "rewards/MANIQAReward/mean": 1.0148690938949585, "rewards/MANIQAReward/std": 0.15851165354251862, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1375000476837158, "rewards/QwenTextReward/std": 0.10681167244911194, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 221 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0592, "frac_reward_zero_std": 0.0, "grad_norm": 1.4453125, "kl": 0.0032579144462943077, "learning_rate": 9.970533333333334e-06, "loss": 0.0003, "num_tokens": 3852312.0, "reward": 6.209277153015137, "reward_std": 0.425199419260025, "rewards/CLIPTextReward/mean": 1.028854489326477, "rewards/CLIPTextReward/std": 0.2329263687133789, "rewards/HPSV21TextReward/mean": 0.8044242858886719, "rewards/HPSV21TextReward/std": 0.1179187074303627, "rewards/MANIQAReward/mean": 1.0532667636871338, "rewards/MANIQAReward/std": 0.1624266505241394, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1535155773162842, "rewards/QwenTextReward/std": 0.06869927048683167, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05946666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.003359306836500764, "learning_rate": 9.970400000000002e-06, "loss": 0.0003, "num_tokens": 3869640.0, "reward": 6.436337947845459, "reward_std": 0.4217972159385681, "rewards/CLIPTextReward/mean": 1.0638182163238525, "rewards/CLIPTextReward/std": 0.18361417949199677, "rewards/HPSV21TextReward/mean": 0.8837671279907227, "rewards/HPSV21TextReward/std": 0.13798930868506432, "rewards/MANIQAReward/mean": 1.0372610092163086, "rewards/MANIQAReward/std": 0.1778913289308548, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.15234375, "rewards/QwenTextReward/std": 0.06853322684764862, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.05973333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.003051916602998972, "learning_rate": 9.970266666666667e-06, "loss": 0.0003, "num_tokens": 3887000.0, "reward": 6.635891437530518, "reward_std": 0.45852988958358765, "rewards/CLIPTextReward/mean": 1.1718319654464722, "rewards/CLIPTextReward/std": 0.1853807494044304, "rewards/HPSV21TextReward/mean": 0.8488368988037109, "rewards/HPSV21TextReward/std": 0.11583320796489716, "rewards/MANIQAReward/mean": 1.0285377502441406, "rewards/MANIQAReward/std": 0.1676163673400879, "rewards/QwenFakeDiscrimReward/mean": 0.19140625, "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, "rewards/QwenTextReward/mean": 1.1480469703674316, "rewards/QwenTextReward/std": 0.0706130787730217, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06, "frac_reward_zero_std": 0.0, "grad_norm": 1.4609375, "kl": 0.003515611169859767, "learning_rate": 9.970133333333335e-06, "loss": 0.0004, "num_tokens": 3904352.0, "reward": 6.577434539794922, "reward_std": 0.38679757714271545, "rewards/CLIPTextReward/mean": 1.110971450805664, "rewards/CLIPTextReward/std": 0.2008369341492653, "rewards/HPSV21TextReward/mean": 0.8672590255737305, "rewards/HPSV21TextReward/std": 0.12437398359179497, "rewards/MANIQAReward/mean": 1.0385515689849854, "rewards/MANIQAReward/std": 0.1615229696035385, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.168359398841858, "rewards/QwenTextReward/std": 0.037332240492105484, "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, "step": 225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06026666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.0031529960688203573, "learning_rate": 9.970000000000001e-06, "loss": 0.0003, "num_tokens": 3921704.0, "reward": 6.4590959548950195, "reward_std": 0.35303205251693726, "rewards/CLIPTextReward/mean": 1.111771583557129, "rewards/CLIPTextReward/std": 0.20289433002471924, "rewards/HPSV21TextReward/mean": 0.8199443817138672, "rewards/HPSV21TextReward/std": 0.07744725793600082, "rewards/MANIQAReward/mean": 1.0612890720367432, "rewards/MANIQAReward/std": 0.16553428769111633, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1476563215255737, "rewards/QwenTextReward/std": 0.06765910983085632, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.060533333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.003157949075102806, "learning_rate": 9.969866666666667e-06, "loss": 0.0003, "num_tokens": 3938992.0, "reward": 6.377033233642578, "reward_std": 0.3737572729587555, "rewards/CLIPTextReward/mean": 1.0765042304992676, "rewards/CLIPTextReward/std": 0.21037136763334274, "rewards/HPSV21TextReward/mean": 0.8247890472412109, "rewards/HPSV21TextReward/std": 0.07355935126543045, "rewards/MANIQAReward/mean": 1.0334311723709106, "rewards/MANIQAReward/std": 0.17133867740631104, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.154296875, "rewards/QwenTextReward/std": 0.03688107430934906, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0608, "frac_reward_zero_std": 0.0, "grad_norm": 1.390625, "kl": 0.003242003731429577, "learning_rate": 9.969733333333334e-06, "loss": 0.0003, "num_tokens": 3956368.0, "reward": 6.312893867492676, "reward_std": 0.35467755794525146, "rewards/CLIPTextReward/mean": 1.090986728668213, "rewards/CLIPTextReward/std": 0.2179025486111641, "rewards/HPSV21TextReward/mean": 0.8224906921386719, "rewards/HPSV21TextReward/std": 0.07208240032196045, "rewards/MANIQAReward/mean": 0.9781261086463928, "rewards/MANIQAReward/std": 0.13308070600032806, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.140625, "rewards/QwenTextReward/std": 0.08363044261932373, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.061066666666666665, "frac_reward_zero_std": 0.0, "grad_norm": 1.390625, "kl": 0.0035423971712589264, "learning_rate": 9.9696e-06, "loss": 0.0004, "num_tokens": 3973664.0, "reward": 6.631295204162598, "reward_std": 0.31362563371658325, "rewards/CLIPTextReward/mean": 1.1526328921318054, "rewards/CLIPTextReward/std": 0.18516195937991142, "rewards/HPSV21TextReward/mean": 0.8869228363037109, "rewards/HPSV21TextReward/std": 0.12250558286905289, "rewards/MANIQAReward/mean": 0.9685895442962646, "rewards/MANIQAReward/std": 0.1559518277645111, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.169531226158142, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, "step": 229 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06133333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.40625, "kl": 0.003313015680760145, "learning_rate": 9.969466666666667e-06, "loss": 0.0003, "num_tokens": 3990984.0, "reward": 6.636979103088379, "reward_std": 0.31966114044189453, "rewards/CLIPTextReward/mean": 1.1439168453216553, "rewards/CLIPTextReward/std": 0.18489812687039375, "rewards/HPSV21TextReward/mean": 0.8825368881225586, "rewards/HPSV21TextReward/std": 0.12275548651814461, "rewards/MANIQAReward/mean": 1.0493061542510986, "rewards/MANIQAReward/std": 0.18121196329593658, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1519532203674316, "rewards/QwenTextReward/std": 0.0631992369890213, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0616, "frac_reward_zero_std": 0.0, "grad_norm": 1.3828125, "kl": 0.0030702250078320503, "learning_rate": 9.969333333333335e-06, "loss": 0.0003, "num_tokens": 4008440.0, "reward": 6.474734306335449, "reward_std": 0.43573421239852905, "rewards/CLIPTextReward/mean": 1.0921568274497986, "rewards/CLIPTextReward/std": 0.17702462896704674, "rewards/HPSV21TextReward/mean": 0.8393945693969727, "rewards/HPSV21TextReward/std": 0.0692838579416275, "rewards/MANIQAReward/mean": 1.0081157684326172, "rewards/MANIQAReward/std": 0.15015669167041779, "rewards/QwenFakeDiscrimReward/mean": 0.19140625, "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, "rewards/QwenTextReward/mean": 1.166015625, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.24609375, "rewards/QwenWeirdDiscrimReward/std": 0.03125, "step": 231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06186666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.40625, "kl": 0.0032026702538132668, "learning_rate": 9.969200000000001e-06, "loss": 0.0003, "num_tokens": 4025784.0, "reward": 6.492210388183594, "reward_std": 0.3398313820362091, "rewards/CLIPTextReward/mean": 1.1206194162368774, "rewards/CLIPTextReward/std": 0.2059924080967903, "rewards/HPSV21TextReward/mean": 0.8321371078491211, "rewards/HPSV21TextReward/std": 0.0851968377828598, "rewards/MANIQAReward/mean": 1.0527129173278809, "rewards/MANIQAReward/std": 0.12888813018798828, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.054621223360300064, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06213333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.625, "kl": 0.00353748956695199, "learning_rate": 9.969066666666667e-06, "loss": 0.0004, "num_tokens": 4043136.0, "reward": 6.196873664855957, "reward_std": 0.36793988943099976, "rewards/CLIPTextReward/mean": 1.0475822687149048, "rewards/CLIPTextReward/std": 0.19828400015830994, "rewards/HPSV21TextReward/mean": 0.8117570877075195, "rewards/HPSV21TextReward/std": 0.07968274503946304, "rewards/MANIQAReward/mean": 0.9211634397506714, "rewards/MANIQAReward/std": 0.15552812814712524, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.174218773841858, "rewards/QwenTextReward/std": 0.03590353950858116, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0624, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.003463034052401781, "learning_rate": 9.968933333333334e-06, "loss": 0.0003, "num_tokens": 4060512.0, "reward": 6.254439353942871, "reward_std": 0.41635259985923767, "rewards/CLIPTextReward/mean": 1.0879863500595093, "rewards/CLIPTextReward/std": 0.19300304353237152, "rewards/HPSV21TextReward/mean": 0.8314504623413086, "rewards/HPSV21TextReward/std": 0.0751388818025589, "rewards/MANIQAReward/mean": 0.9788466095924377, "rewards/MANIQAReward/std": 0.16533632576465607, "rewards/QwenFakeDiscrimReward/mean": 0.125, "rewards/QwenFakeDiscrimReward/std": 0.1259881556034088, "rewards/QwenTextReward/mean": 1.1359374523162842, "rewards/QwenTextReward/std": 0.06391262263059616, "rewards/QwenWeirdDiscrimReward/mean": 0.17578125, "rewards/QwenWeirdDiscrimReward/std": 0.1151231899857521, "step": 234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06266666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.0030904915183782578, "learning_rate": 9.9688e-06, "loss": 0.0003, "num_tokens": 4077760.0, "reward": 6.185770034790039, "reward_std": 0.4532530605792999, "rewards/CLIPTextReward/mean": 1.028242588043213, "rewards/CLIPTextReward/std": 0.18711334466934204, "rewards/HPSV21TextReward/mean": 0.8127584457397461, "rewards/HPSV21TextReward/std": 0.0767570361495018, "rewards/MANIQAReward/mean": 1.0061118602752686, "rewards/MANIQAReward/std": 0.14821553230285645, "rewards/QwenFakeDiscrimReward/mean": 0.1328125, "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03750002384185791, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06293333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0033108042553067207, "learning_rate": 9.968666666666668e-06, "loss": 0.0003, "num_tokens": 4095136.0, "reward": 6.450545310974121, "reward_std": 0.34508979320526123, "rewards/CLIPTextReward/mean": 1.1125239133834839, "rewards/CLIPTextReward/std": 0.18488702550530434, "rewards/HPSV21TextReward/mean": 0.83349609375, "rewards/HPSV21TextReward/std": 0.057444117963314056, "rewards/MANIQAReward/mean": 1.0319424867630005, "rewards/MANIQAReward/std": 0.15041349828243256, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.1554687023162842, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0632, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.0028371165972203016, "learning_rate": 9.968533333333335e-06, "loss": 0.0003, "num_tokens": 4112496.0, "reward": 6.167689323425293, "reward_std": 0.32739606499671936, "rewards/CLIPTextReward/mean": 1.048931360244751, "rewards/CLIPTextReward/std": 0.19749532639980316, "rewards/HPSV21TextReward/mean": 0.7899513244628906, "rewards/HPSV21TextReward/std": 0.07274974882602692, "rewards/MANIQAReward/mean": 0.9731267690658569, "rewards/MANIQAReward/std": 0.15662546455860138, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1378905773162842, "rewards/QwenTextReward/std": 0.104558065533638, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06346666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.003085110802203417, "learning_rate": 9.968400000000001e-06, "loss": 0.0003, "num_tokens": 4129840.0, "reward": 6.481772422790527, "reward_std": 0.35772961378097534, "rewards/CLIPTextReward/mean": 1.0813804864883423, "rewards/CLIPTextReward/std": 0.20720766484737396, "rewards/HPSV21TextReward/mean": 0.8641490936279297, "rewards/HPSV21TextReward/std": 0.17907562106847763, "rewards/MANIQAReward/mean": 1.0344631671905518, "rewards/MANIQAReward/std": 0.14367032051086426, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1500000953674316, "rewards/QwenTextReward/std": 0.07637627422809601, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06373333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.002941595856100321, "learning_rate": 9.968266666666667e-06, "loss": 0.0003, "num_tokens": 4147096.0, "reward": 6.353859901428223, "reward_std": 0.3981371819972992, "rewards/CLIPTextReward/mean": 1.063906729221344, "rewards/CLIPTextReward/std": 0.18121198564767838, "rewards/HPSV21TextReward/mean": 0.81640625, "rewards/HPSV21TextReward/std": 0.07464800775051117, "rewards/MANIQAReward/mean": 0.9889370203018188, "rewards/MANIQAReward/std": 0.15643689036369324, "rewards/QwenFakeDiscrimReward/mean": 0.19921875, "rewards/QwenFakeDiscrimReward/std": 0.10137632489204407, "rewards/QwenTextReward/mean": 1.1628906726837158, "rewards/QwenTextReward/std": 0.06929238885641098, "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, "step": 239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.064, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.003230172209441662, "learning_rate": 9.968133333333334e-06, "loss": 0.0003, "num_tokens": 4164432.0, "reward": 6.608020782470703, "reward_std": 0.3431330919265747, "rewards/CLIPTextReward/mean": 1.215427041053772, "rewards/CLIPTextReward/std": 0.19819049537181854, "rewards/HPSV21TextReward/mean": 0.8417644500732422, "rewards/HPSV21TextReward/std": 0.05505716800689697, "rewards/MANIQAReward/mean": 0.9971534013748169, "rewards/MANIQAReward/std": 0.14450205862522125, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1449217796325684, "rewards/QwenTextReward/std": 0.10598927736282349, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06426666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.003244362771511078, "learning_rate": 9.968000000000002e-06, "loss": 0.0003, "num_tokens": 4181760.0, "reward": 6.384057521820068, "reward_std": 0.4403102993965149, "rewards/CLIPTextReward/mean": 1.1254714727401733, "rewards/CLIPTextReward/std": 0.1991724595427513, "rewards/HPSV21TextReward/mean": 0.8270206451416016, "rewards/HPSV21TextReward/std": 0.07428202033042908, "rewards/MANIQAReward/mean": 1.000557541847229, "rewards/MANIQAReward/std": 0.16089297831058502, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.142578125, "rewards/QwenTextReward/std": 0.034698180854320526, "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, "step": 241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06453333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.003257544245570898, "learning_rate": 9.967866666666668e-06, "loss": 0.0003, "num_tokens": 4199208.0, "reward": 6.362607479095459, "reward_std": 0.3074047863483429, "rewards/CLIPTextReward/mean": 1.0759841203689575, "rewards/CLIPTextReward/std": 0.19806966930627823, "rewards/HPSV21TextReward/mean": 0.8266582489013672, "rewards/HPSV21TextReward/std": 0.09093856066465378, "rewards/MANIQAReward/mean": 1.0159164667129517, "rewards/MANIQAReward/std": 0.14935745298862457, "rewards/QwenFakeDiscrimReward/mean": 0.1875, "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, "rewards/QwenTextReward/mean": 1.123437523841858, "rewards/QwenTextReward/std": 0.13347278535366058, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0648, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.003465704619884491, "learning_rate": 9.967733333333335e-06, "loss": 0.0003, "num_tokens": 4216568.0, "reward": 6.463565826416016, "reward_std": 0.3935210704803467, "rewards/CLIPTextReward/mean": 1.057560682296753, "rewards/CLIPTextReward/std": 0.17162110283970833, "rewards/HPSV21TextReward/mean": 0.8747730255126953, "rewards/HPSV21TextReward/std": 0.11632852256298065, "rewards/MANIQAReward/mean": 1.0449917316436768, "rewards/MANIQAReward/std": 0.18094049394130707, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.1554687023162842, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06506666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.265625, "kl": 0.0031383931636810303, "learning_rate": 9.967600000000001e-06, "loss": 0.0003, "num_tokens": 4233936.0, "reward": 6.2581281661987305, "reward_std": 0.4083012044429779, "rewards/CLIPTextReward/mean": 1.1018357276916504, "rewards/CLIPTextReward/std": 0.20741251856088638, "rewards/HPSV21TextReward/mean": 0.811309814453125, "rewards/HPSV21TextReward/std": 0.10790691524744034, "rewards/MANIQAReward/mean": 0.953321099281311, "rewards/MANIQAReward/std": 0.18221963942050934, "rewards/QwenFakeDiscrimReward/mean": 0.12109375, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.146484375, "rewards/QwenTextReward/std": 0.039557892829179764, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06533333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0034496267326176167, "learning_rate": 9.967466666666667e-06, "loss": 0.0003, "num_tokens": 4251464.0, "reward": 6.258942127227783, "reward_std": 0.467055082321167, "rewards/CLIPTextReward/mean": 1.0869176387786865, "rewards/CLIPTextReward/std": 0.22874008864164352, "rewards/HPSV21TextReward/mean": 0.8402419090270996, "rewards/HPSV21TextReward/std": 0.11818192154169083, "rewards/MANIQAReward/mean": 0.9710292816162109, "rewards/MANIQAReward/std": 0.16109530627727509, "rewards/QwenFakeDiscrimReward/mean": 0.1171875, "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, "rewards/QwenTextReward/mean": 1.140625, "rewards/QwenTextReward/std": 0.03659627214074135, "rewards/QwenWeirdDiscrimReward/mean": 0.17578125, "rewards/QwenWeirdDiscrimReward/std": 0.1151231899857521, "step": 245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0656, "frac_reward_zero_std": 0.0, "grad_norm": 1.3984375, "kl": 0.004035755526274443, "learning_rate": 9.967333333333334e-06, "loss": 0.0004, "num_tokens": 4268856.0, "reward": 6.789213180541992, "reward_std": 0.35714399814605713, "rewards/CLIPTextReward/mean": 1.1455610394477844, "rewards/CLIPTextReward/std": 0.20604808628559113, "rewards/HPSV21TextReward/mean": 0.9044771194458008, "rewards/HPSV21TextReward/std": 0.13277480378746986, "rewards/MANIQAReward/mean": 1.1317150592803955, "rewards/MANIQAReward/std": 0.192893385887146, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06586666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.4375, "kl": 0.003506520064547658, "learning_rate": 9.967200000000002e-06, "loss": 0.0004, "num_tokens": 4286104.0, "reward": 6.166922569274902, "reward_std": 0.3984561562538147, "rewards/CLIPTextReward/mean": 1.0409660339355469, "rewards/CLIPTextReward/std": 0.20897932350635529, "rewards/HPSV21TextReward/mean": 0.8085098266601562, "rewards/HPSV21TextReward/std": 0.07335306704044342, "rewards/MANIQAReward/mean": 0.988283097743988, "rewards/MANIQAReward/std": 0.1673584133386612, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.1281249523162842, "rewards/QwenTextReward/std": 0.11104661971330643, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06613333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0034983076620846987, "learning_rate": 9.967066666666666e-06, "loss": 0.0004, "num_tokens": 4303496.0, "reward": 6.271515846252441, "reward_std": 0.32968008518218994, "rewards/CLIPTextReward/mean": 1.08308744430542, "rewards/CLIPTextReward/std": 0.18308281898498535, "rewards/HPSV21TextReward/mean": 0.827845573425293, "rewards/HPSV21TextReward/std": 0.05251236632466316, "rewards/MANIQAReward/mean": 0.8984781503677368, "rewards/MANIQAReward/std": 0.16231420636177063, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.168359398841858, "rewards/QwenTextReward/std": 0.03965182974934578, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0664, "frac_reward_zero_std": 0.0, "grad_norm": 1.296875, "kl": 0.0032820648048073053, "learning_rate": 9.966933333333334e-06, "loss": 0.0003, "num_tokens": 4320896.0, "reward": 6.3601202964782715, "reward_std": 0.3747427463531494, "rewards/CLIPTextReward/mean": 1.0786243677139282, "rewards/CLIPTextReward/std": 0.18130076304078102, "rewards/HPSV21TextReward/mean": 0.8438816070556641, "rewards/HPSV21TextReward/std": 0.07765045017004013, "rewards/MANIQAReward/mean": 1.0408895015716553, "rewards/MANIQAReward/std": 0.12134683132171631, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1343750953674316, "rewards/QwenTextReward/std": 0.0574767179787159, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 249 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.0035632310900837183, "learning_rate": 9.966800000000001e-06, "loss": 0.0004, "num_tokens": 4338272.0, "reward": 6.261956214904785, "reward_std": 0.3757457435131073, "rewards/CLIPTextReward/mean": 1.097538948059082, "rewards/CLIPTextReward/std": 0.2125762403011322, "rewards/HPSV21TextReward/mean": 0.8204212188720703, "rewards/HPSV21TextReward/std": 0.06764323264360428, "rewards/MANIQAReward/mean": 0.9693953990936279, "rewards/MANIQAReward/std": 0.13938410580158234, "rewards/QwenFakeDiscrimReward/mean": 0.12890625, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.140234351158142, "rewards/QwenTextReward/std": 0.030412914231419563, "rewards/QwenWeirdDiscrimReward/mean": 0.1875, "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06693333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3828125, "kl": 0.003846655134111643, "learning_rate": 9.966666666666667e-06, "loss": 0.0004, "num_tokens": 4355688.0, "reward": 6.575259208679199, "reward_std": 0.43733108043670654, "rewards/CLIPTextReward/mean": 1.1088823080062866, "rewards/CLIPTextReward/std": 0.1903865486383438, "rewards/HPSV21TextReward/mean": 0.9043560028076172, "rewards/HPSV21TextReward/std": 0.15610383450984955, "rewards/MANIQAReward/mean": 1.0280795097351074, "rewards/MANIQAReward/std": 0.153267964720726, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1496093273162842, "rewards/QwenTextReward/std": 0.06275618821382523, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0672, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.003265757579356432, "learning_rate": 9.966533333333334e-06, "loss": 0.0003, "num_tokens": 4372928.0, "reward": 6.416788101196289, "reward_std": 0.3261052966117859, "rewards/CLIPTextReward/mean": 1.0913305878639221, "rewards/CLIPTextReward/std": 0.18725522607564926, "rewards/HPSV21TextReward/mean": 0.8135738372802734, "rewards/HPSV21TextReward/std": 0.08006454259157181, "rewards/MANIQAReward/mean": 1.035104513168335, "rewards/MANIQAReward/std": 0.1181311383843422, "rewards/QwenFakeDiscrimReward/mean": 0.20703125, "rewards/QwenFakeDiscrimReward/std": 0.09506355226039886, "rewards/QwenTextReward/mean": 1.1304688453674316, "rewards/QwenTextReward/std": 0.10488740354776382, "rewards/QwenWeirdDiscrimReward/mean": 0.234375, "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, "step": 252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06746666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.003604835830628872, "learning_rate": 9.9664e-06, "loss": 0.0004, "num_tokens": 4390336.0, "reward": 6.431319236755371, "reward_std": 0.39881986379623413, "rewards/CLIPTextReward/mean": 1.1042168140411377, "rewards/CLIPTextReward/std": 0.1991683542728424, "rewards/HPSV21TextReward/mean": 0.8092203140258789, "rewards/HPSV21TextReward/std": 0.07349780946969986, "rewards/MANIQAReward/mean": 1.038038969039917, "rewards/MANIQAReward/std": 0.163633793592453, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06773333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.2734375, "kl": 0.0028017221484333277, "learning_rate": 9.966266666666668e-06, "loss": 0.0003, "num_tokens": 4407592.0, "reward": 6.429580211639404, "reward_std": 0.43548667430877686, "rewards/CLIPTextReward/mean": 1.0889440774917603, "rewards/CLIPTextReward/std": 0.19870372861623764, "rewards/HPSV21TextReward/mean": 0.8702507019042969, "rewards/HPSV21TextReward/std": 0.1553412601351738, "rewards/MANIQAReward/mean": 0.9592373967170715, "rewards/MANIQAReward/std": 0.19040289521217346, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1613280773162842, "rewards/QwenTextReward/std": 0.06407613307237625, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.068, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0030375674832612276, "learning_rate": 9.966133333333334e-06, "loss": 0.0003, "num_tokens": 4424856.0, "reward": 6.504112243652344, "reward_std": 0.44497406482696533, "rewards/CLIPTextReward/mean": 1.1087359189987183, "rewards/CLIPTextReward/std": 0.19022604078054428, "rewards/HPSV21TextReward/mean": 0.8200454711914062, "rewards/HPSV21TextReward/std": 0.13096706569194794, "rewards/MANIQAReward/mean": 1.0363925695419312, "rewards/MANIQAReward/std": 0.16490438580513, "rewards/QwenFakeDiscrimReward/mean": 0.19140625, "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, "rewards/QwenTextReward/mean": 1.1765625476837158, "rewards/QwenTextReward/std": 0.03503827005624771, "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, "step": 255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06826666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0036664570216089487, "learning_rate": 9.966e-06, "loss": 0.0004, "num_tokens": 4442272.0, "reward": 6.318208694458008, "reward_std": 0.37837815284729004, "rewards/CLIPTextReward/mean": 1.0883460640907288, "rewards/CLIPTextReward/std": 0.2118919938802719, "rewards/HPSV21TextReward/mean": 0.8206787109375, "rewards/HPSV21TextReward/std": 0.05644918233156204, "rewards/MANIQAReward/mean": 0.971252977848053, "rewards/MANIQAReward/std": 0.1337847262620926, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.146093726158142, "rewards/QwenTextReward/std": 0.06191693991422653, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06853333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.6171875, "kl": 0.0037016423884779215, "learning_rate": 9.965866666666667e-06, "loss": 0.0004, "num_tokens": 4459688.0, "reward": 6.354341506958008, "reward_std": 0.42910268902778625, "rewards/CLIPTextReward/mean": 1.0966795682907104, "rewards/CLIPTextReward/std": 0.18399309739470482, "rewards/HPSV21TextReward/mean": 0.8211269378662109, "rewards/HPSV21TextReward/std": 0.07162735611200333, "rewards/MANIQAReward/mean": 0.958962619304657, "rewards/MANIQAReward/std": 0.1704968512058258, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1613280773162842, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0688, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.003262406447902322, "learning_rate": 9.965733333333334e-06, "loss": 0.0003, "num_tokens": 4476952.0, "reward": 6.296139717102051, "reward_std": 0.3946528434753418, "rewards/CLIPTextReward/mean": 1.0265021324157715, "rewards/CLIPTextReward/std": 0.1562262624502182, "rewards/HPSV21TextReward/mean": 0.8317289352416992, "rewards/HPSV21TextReward/std": 0.13052061945199966, "rewards/MANIQAReward/mean": 1.031239628791809, "rewards/MANIQAReward/std": 0.15305881202220917, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.149999976158142, "rewards/QwenTextReward/std": 0.06813853234052658, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06906666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.2890625, "kl": 0.002926542889326811, "learning_rate": 9.965600000000002e-06, "loss": 0.0003, "num_tokens": 4494312.0, "reward": 6.2506866455078125, "reward_std": 0.42295345664024353, "rewards/CLIPTextReward/mean": 1.095668911933899, "rewards/CLIPTextReward/std": 0.19134847819805145, "rewards/HPSV21TextReward/mean": 0.7793178558349609, "rewards/HPSV21TextReward/std": 0.06817104667425156, "rewards/MANIQAReward/mean": 0.9956351518630981, "rewards/MANIQAReward/std": 0.19141975045204163, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1222655773162842, "rewards/QwenTextReward/std": 0.09439840167760849, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 259 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06933333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0038751026149839163, "learning_rate": 9.965466666666668e-06, "loss": 0.0004, "num_tokens": 4511528.0, "reward": 6.591069221496582, "reward_std": 0.4039503335952759, "rewards/CLIPTextReward/mean": 1.0908901691436768, "rewards/CLIPTextReward/std": 0.19581331312656403, "rewards/HPSV21TextReward/mean": 0.9031896591186523, "rewards/HPSV21TextReward/std": 0.19001787155866623, "rewards/MANIQAReward/mean": 1.0818156003952026, "rewards/MANIQAReward/std": 0.19983670115470886, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.138281226158142, "rewards/QwenTextReward/std": 0.08297169953584671, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0696, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0037587848491966724, "learning_rate": 9.965333333333334e-06, "loss": 0.0004, "num_tokens": 4528864.0, "reward": 6.491024017333984, "reward_std": 0.3804985284805298, "rewards/CLIPTextReward/mean": 1.0769048929214478, "rewards/CLIPTextReward/std": 0.1824543960392475, "rewards/HPSV21TextReward/mean": 0.8643198013305664, "rewards/HPSV21TextReward/std": 0.11065323650836945, "rewards/MANIQAReward/mean": 1.0597467422485352, "rewards/MANIQAReward/std": 0.14535771310329437, "rewards/QwenFakeDiscrimReward/mean": 0.1640625, "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, "rewards/QwenTextReward/mean": 1.154296875, "rewards/QwenTextReward/std": 0.06355144083499908, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 261 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.06986666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.40625, "kl": 0.0034617320634424686, "learning_rate": 9.9652e-06, "loss": 0.0003, "num_tokens": 4546272.0, "reward": 6.14242696762085, "reward_std": 0.396884024143219, "rewards/CLIPTextReward/mean": 1.0349934697151184, "rewards/CLIPTextReward/std": 0.20142310112714767, "rewards/HPSV21TextReward/mean": 0.7808294296264648, "rewards/HPSV21TextReward/std": 0.08118784427642822, "rewards/MANIQAReward/mean": 1.0135157108306885, "rewards/MANIQAReward/std": 0.14665435254573822, "rewards/QwenFakeDiscrimReward/mean": 0.125, "rewards/QwenFakeDiscrimReward/std": 0.1259881556034088, "rewards/QwenTextReward/mean": 1.1496093273162842, "rewards/QwenTextReward/std": 0.03549320250749588, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07013333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3828125, "kl": 0.003490220755338669, "learning_rate": 9.965066666666667e-06, "loss": 0.0003, "num_tokens": 4563752.0, "reward": 6.163957595825195, "reward_std": 0.385267436504364, "rewards/CLIPTextReward/mean": 1.0602093935012817, "rewards/CLIPTextReward/std": 0.2071426585316658, "rewards/HPSV21TextReward/mean": 0.7925214767456055, "rewards/HPSV21TextReward/std": 0.08600976318120956, "rewards/MANIQAReward/mean": 1.0194330215454102, "rewards/MANIQAReward/std": 0.1785665899515152, "rewards/QwenFakeDiscrimReward/mean": 0.109375, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.146093726158142, "rewards/QwenTextReward/std": 0.03398728370666504, "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, "step": 263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0704, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.004037609323859215, "learning_rate": 9.964933333333333e-06, "loss": 0.0004, "num_tokens": 4581080.0, "reward": 6.484213352203369, "reward_std": 0.3694634437561035, "rewards/CLIPTextReward/mean": 1.1062145233154297, "rewards/CLIPTextReward/std": 0.1875825934112072, "rewards/HPSV21TextReward/mean": 0.8689737319946289, "rewards/HPSV21TextReward/std": 0.10702138394117355, "rewards/MANIQAReward/mean": 0.9932119846343994, "rewards/MANIQAReward/std": 0.14993032813072205, "rewards/QwenFakeDiscrimReward/mean": 0.1640625, "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.06391262263059616, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07066666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.00397456856444478, "learning_rate": 9.964800000000002e-06, "loss": 0.0004, "num_tokens": 4598448.0, "reward": 6.335943222045898, "reward_std": 0.3582858443260193, "rewards/CLIPTextReward/mean": 1.0521622896194458, "rewards/CLIPTextReward/std": 0.21286197006702423, "rewards/HPSV21TextReward/mean": 0.8410053253173828, "rewards/HPSV21TextReward/std": 0.13608239963650703, "rewards/MANIQAReward/mean": 1.0343737602233887, "rewards/MANIQAReward/std": 0.15465207397937775, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.144140601158142, "rewards/QwenTextReward/std": 0.08448517322540283, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07093333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.375, "kl": 0.0033863503485918045, "learning_rate": 9.964666666666666e-06, "loss": 0.0003, "num_tokens": 4615800.0, "reward": 6.447694778442383, "reward_std": 0.3566947877407074, "rewards/CLIPTextReward/mean": 1.1556851863861084, "rewards/CLIPTextReward/std": 0.21023477613925934, "rewards/HPSV21TextReward/mean": 0.8281173706054688, "rewards/HPSV21TextReward/std": 0.08502472192049026, "rewards/MANIQAReward/mean": 0.9918079972267151, "rewards/MANIQAReward/std": 0.17592301964759827, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1484375, "rewards/QwenTextReward/std": 0.0625000149011612, "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, "step": 266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0712, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0034869378432631493, "learning_rate": 9.964533333333334e-06, "loss": 0.0003, "num_tokens": 4633048.0, "reward": 6.298396110534668, "reward_std": 0.4051980674266815, "rewards/CLIPTextReward/mean": 1.0385345220565796, "rewards/CLIPTextReward/std": 0.20875421911478043, "rewards/HPSV21TextReward/mean": 0.834111213684082, "rewards/HPSV21TextReward/std": 0.07624765485525131, "rewards/MANIQAReward/mean": 1.0234172344207764, "rewards/MANIQAReward/std": 0.18777620792388916, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.166406273841858, "rewards/QwenTextReward/std": 0.07046034932136536, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07146666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.0043066563084721565, "learning_rate": 9.9644e-06, "loss": 0.0004, "num_tokens": 4650384.0, "reward": 6.33132266998291, "reward_std": 0.3792267441749573, "rewards/CLIPTextReward/mean": 1.0630069971084595, "rewards/CLIPTextReward/std": 0.208414725959301, "rewards/HPSV21TextReward/mean": 0.8625335693359375, "rewards/HPSV21TextReward/std": 0.1617307998239994, "rewards/MANIQAReward/mean": 0.9564133882522583, "rewards/MANIQAReward/std": 0.1984596848487854, "rewards/QwenFakeDiscrimReward/mean": 0.1640625, "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, "rewards/QwenTextReward/mean": 1.156640648841858, "rewards/QwenTextReward/std": 0.06381431967020035, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07173333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.421875, "kl": 0.004046045243740082, "learning_rate": 9.964266666666667e-06, "loss": 0.0004, "num_tokens": 4667816.0, "reward": 6.5365142822265625, "reward_std": 0.3556900918483734, "rewards/CLIPTextReward/mean": 1.0756992101669312, "rewards/CLIPTextReward/std": 0.19139912724494934, "rewards/HPSV21TextReward/mean": 0.8689680099487305, "rewards/HPSV21TextReward/std": 0.05170682817697525, "rewards/MANIQAReward/mean": 1.0948357582092285, "rewards/MANIQAReward/std": 0.14325103163719177, "rewards/QwenFakeDiscrimReward/mean": 0.16796875, "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, "rewards/QwenTextReward/mean": 1.1695313453674316, "rewards/QwenTextReward/std": 0.03945784643292427, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 269 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.072, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "kl": 0.003918897360563278, "learning_rate": 9.964133333333335e-06, "loss": 0.0004, "num_tokens": 4685160.0, "reward": 6.477409839630127, "reward_std": 0.41067662835121155, "rewards/CLIPTextReward/mean": 1.1299338340759277, "rewards/CLIPTextReward/std": 0.20776088535785675, "rewards/HPSV21TextReward/mean": 0.8103847503662109, "rewards/HPSV21TextReward/std": 0.10611110925674438, "rewards/MANIQAReward/mean": 1.0385692119598389, "rewards/MANIQAReward/std": 0.18805089592933655, "rewards/QwenFakeDiscrimReward/mean": 0.1875, "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, "rewards/QwenTextReward/mean": 1.1636719703674316, "rewards/QwenTextReward/std": 0.037778012454509735, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07226666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.0036119483411312103, "learning_rate": 9.964e-06, "loss": 0.0004, "num_tokens": 4702432.0, "reward": 6.398820877075195, "reward_std": 0.37300944328308105, "rewards/CLIPTextReward/mean": 1.0580294132232666, "rewards/CLIPTextReward/std": 0.18786108493804932, "rewards/HPSV21TextReward/mean": 0.8465709686279297, "rewards/HPSV21TextReward/std": 0.0728863999247551, "rewards/MANIQAReward/mean": 1.0263386964797974, "rewards/MANIQAReward/std": 0.15372315049171448, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1648437976837158, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 271 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07253333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3125, "kl": 0.0035398683976382017, "learning_rate": 9.963866666666668e-06, "loss": 0.0004, "num_tokens": 4719864.0, "reward": 6.652245044708252, "reward_std": 0.4208158552646637, "rewards/CLIPTextReward/mean": 1.1644315719604492, "rewards/CLIPTextReward/std": 0.1786278672516346, "rewards/HPSV21TextReward/mean": 0.8648881912231445, "rewards/HPSV21TextReward/std": 0.1337926834821701, "rewards/MANIQAReward/mean": 0.9908708930015564, "rewards/MANIQAReward/std": 0.14858315885066986, "rewards/QwenFakeDiscrimReward/mean": 0.19140625, "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, "rewards/QwenTextReward/mean": 1.1730469465255737, "rewards/QwenTextReward/std": 0.03627080097794533, "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, "step": 272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0728, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.004338162951171398, "learning_rate": 9.963733333333334e-06, "loss": 0.0004, "num_tokens": 4737176.0, "reward": 6.562404632568359, "reward_std": 0.3299024701118469, "rewards/CLIPTextReward/mean": 1.0884344577789307, "rewards/CLIPTextReward/std": 0.18314744904637337, "rewards/HPSV21TextReward/mean": 0.9037189483642578, "rewards/HPSV21TextReward/std": 0.13727488741278648, "rewards/MANIQAReward/mean": 1.048019528388977, "rewards/MANIQAReward/std": 0.18316024541854858, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1589844226837158, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07306666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0039003107231110334, "learning_rate": 9.9636e-06, "loss": 0.0004, "num_tokens": 4754480.0, "reward": 6.383108615875244, "reward_std": 0.3999929428100586, "rewards/CLIPTextReward/mean": 1.079944133758545, "rewards/CLIPTextReward/std": 0.1960008516907692, "rewards/HPSV21TextReward/mean": 0.8467903137207031, "rewards/HPSV21TextReward/std": 0.06765388697385788, "rewards/MANIQAReward/mean": 1.0222177505493164, "rewards/MANIQAReward/std": 0.19649730622768402, "rewards/QwenFakeDiscrimReward/mean": 0.1328125, "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, "rewards/QwenTextReward/mean": 1.1636719703674316, "rewards/QwenTextReward/std": 0.04007181152701378, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.28125, "kl": 0.0034153100568801165, "learning_rate": 9.963466666666669e-06, "loss": 0.0003, "num_tokens": 4771792.0, "reward": 6.4820380210876465, "reward_std": 0.4463214874267578, "rewards/CLIPTextReward/mean": 1.1188676357269287, "rewards/CLIPTextReward/std": 0.20363353937864304, "rewards/HPSV21TextReward/mean": 0.876103401184082, "rewards/HPSV21TextReward/std": 0.11718452721834183, "rewards/MANIQAReward/mean": 0.973345935344696, "rewards/MANIQAReward/std": 0.18426600098609924, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1398438215255737, "rewards/QwenTextReward/std": 0.08748229593038559, "rewards/QwenWeirdDiscrimReward/mean": 0.203125, "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, "step": 275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0736, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.003312772372737527, "learning_rate": 9.963333333333333e-06, "loss": 0.0003, "num_tokens": 4789144.0, "reward": 6.420487403869629, "reward_std": 0.4493219256401062, "rewards/CLIPTextReward/mean": 1.107895851135254, "rewards/CLIPTextReward/std": 0.2051476389169693, "rewards/HPSV21TextReward/mean": 0.8210859298706055, "rewards/HPSV21TextReward/std": 0.1308876797556877, "rewards/MANIQAReward/mean": 1.0277581214904785, "rewards/MANIQAReward/std": 0.16148918867111206, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.1324219703674316, "rewards/QwenTextReward/std": 0.0810006633400917, "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, "step": 276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07386666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.003616444068029523, "learning_rate": 9.963200000000001e-06, "loss": 0.0004, "num_tokens": 4806400.0, "reward": 6.341896057128906, "reward_std": 0.4027109146118164, "rewards/CLIPTextReward/mean": 1.1019320487976074, "rewards/CLIPTextReward/std": 0.25059743225574493, "rewards/HPSV21TextReward/mean": 0.8123483657836914, "rewards/HPSV21TextReward/std": 0.07598406821489334, "rewards/MANIQAReward/mean": 1.025053858757019, "rewards/MANIQAReward/std": 0.16003963351249695, "rewards/QwenFakeDiscrimReward/mean": 0.14453125, "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, "rewards/QwenTextReward/mean": 1.13671875, "rewards/QwenTextReward/std": 0.07867565751075745, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07413333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.0036018122918903828, "learning_rate": 9.963066666666668e-06, "loss": 0.0004, "num_tokens": 4823672.0, "reward": 6.3944621086120605, "reward_std": 0.43935614824295044, "rewards/CLIPTextReward/mean": 1.075329065322876, "rewards/CLIPTextReward/std": 0.19888444989919662, "rewards/HPSV21TextReward/mean": 0.833500862121582, "rewards/HPSV21TextReward/std": 0.06530586630105972, "rewards/MANIQAReward/mean": 1.0189895629882812, "rewards/MANIQAReward/std": 0.14748038351535797, "rewards/QwenFakeDiscrimReward/mean": 0.1875, "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, "rewards/QwenTextReward/mean": 1.1554688215255737, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0744, "frac_reward_zero_std": 0.0, "grad_norm": 1.3984375, "kl": 0.0039861504919826984, "learning_rate": 9.962933333333334e-06, "loss": 0.0004, "num_tokens": 4840992.0, "reward": 6.240785598754883, "reward_std": 0.3987360894680023, "rewards/CLIPTextReward/mean": 1.0433543920516968, "rewards/CLIPTextReward/std": 0.1878664493560791, "rewards/HPSV21TextReward/mean": 0.8436336517333984, "rewards/HPSV21TextReward/std": 0.06514830887317657, "rewards/MANIQAReward/mean": 0.9824346303939819, "rewards/MANIQAReward/std": 0.16924676299095154, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.1484375, "rewards/QwenTextReward/std": 0.0625000149011612, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 279 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07466666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.3359375, "kl": 0.0036991594824939966, "learning_rate": 9.9628e-06, "loss": 0.0004, "num_tokens": 4858328.0, "reward": 6.502524375915527, "reward_std": 0.3771998882293701, "rewards/CLIPTextReward/mean": 1.1242562532424927, "rewards/CLIPTextReward/std": 0.1967948004603386, "rewards/HPSV21TextReward/mean": 0.8170080184936523, "rewards/HPSV21TextReward/std": 0.1085859090089798, "rewards/MANIQAReward/mean": 1.062964677810669, "rewards/MANIQAReward/std": 0.16206982731819153, "rewards/QwenFakeDiscrimReward/mean": 0.18359375, "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, "rewards/QwenTextReward/mean": 1.142968773841858, "rewards/QwenTextReward/std": 0.06643842905759811, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07493333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.003956177271902561, "learning_rate": 9.962666666666667e-06, "loss": 0.0004, "num_tokens": 4875776.0, "reward": 6.581701278686523, "reward_std": 0.43749040365219116, "rewards/CLIPTextReward/mean": 1.104413628578186, "rewards/CLIPTextReward/std": 0.19979221373796463, "rewards/HPSV21TextReward/mean": 0.9073686599731445, "rewards/HPSV21TextReward/std": 0.15387574583292007, "rewards/MANIQAReward/mean": 0.9991519451141357, "rewards/MANIQAReward/std": 0.18088753521442413, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.168359398841858, "rewards/QwenTextReward/std": 0.03965182974934578, "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, "step": 281 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0752, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.00417393259704113, "learning_rate": 9.962533333333333e-06, "loss": 0.0004, "num_tokens": 4893080.0, "reward": 6.471848487854004, "reward_std": 0.3436872959136963, "rewards/CLIPTextReward/mean": 1.1173217296600342, "rewards/CLIPTextReward/std": 0.21878229081630707, "rewards/HPSV21TextReward/mean": 0.8560466766357422, "rewards/HPSV21TextReward/std": 0.0951707735657692, "rewards/MANIQAReward/mean": 1.0106582641601562, "rewards/MANIQAReward/std": 0.163259819149971, "rewards/QwenFakeDiscrimReward/mean": 0.1796875, "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, "rewards/QwenTextReward/mean": 1.1433594226837158, "rewards/QwenTextReward/std": 0.08833134919404984, "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, "step": 282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07546666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.003545068670064211, "learning_rate": 9.962400000000001e-06, "loss": 0.0004, "num_tokens": 4910616.0, "reward": 6.3966569900512695, "reward_std": 0.3278886675834656, "rewards/CLIPTextReward/mean": 1.133664846420288, "rewards/CLIPTextReward/std": 0.20752384513616562, "rewards/HPSV21TextReward/mean": 0.7828845977783203, "rewards/HPSV21TextReward/std": 0.07142157107591629, "rewards/MANIQAReward/mean": 1.0354331731796265, "rewards/MANIQAReward/std": 0.13326719403266907, "rewards/QwenFakeDiscrimReward/mean": 0.17578125, "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, "rewards/QwenTextReward/mean": 1.1414062976837158, "rewards/QwenTextReward/std": 0.08787830919027328, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07573333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.40625, "kl": 0.003691106103360653, "learning_rate": 9.962266666666668e-06, "loss": 0.0004, "num_tokens": 4927920.0, "reward": 6.352651596069336, "reward_std": 0.37639120221138, "rewards/CLIPTextReward/mean": 1.0819745659828186, "rewards/CLIPTextReward/std": 0.21546390652656555, "rewards/HPSV21TextReward/mean": 0.8350515365600586, "rewards/HPSV21TextReward/std": 0.12028282135725021, "rewards/MANIQAReward/mean": 1.0287556648254395, "rewards/MANIQAReward/std": 0.14155684411525726, "rewards/QwenFakeDiscrimReward/mean": 0.13671875, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.126562476158142, "rewards/QwenTextReward/std": 0.13663147389888763, "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, "step": 284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.076, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0040000625886023045, "learning_rate": 9.962133333333334e-06, "loss": 0.0004, "num_tokens": 4945336.0, "reward": 6.344567775726318, "reward_std": 0.42905884981155396, "rewards/CLIPTextReward/mean": 1.1392810344696045, "rewards/CLIPTextReward/std": 0.1911049485206604, "rewards/HPSV21TextReward/mean": 0.8198728561401367, "rewards/HPSV21TextReward/std": 0.054194964468479156, "rewards/MANIQAReward/mean": 0.9813380241394043, "rewards/MANIQAReward/std": 0.1418820321559906, "rewards/QwenFakeDiscrimReward/mean": 0.12890625, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.144140601158142, "rewards/QwenTextReward/std": 0.04974982514977455, "rewards/QwenWeirdDiscrimReward/mean": 0.171875, "rewards/QwenWeirdDiscrimReward/std": 0.116794154047966, "step": 285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07626666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.0036982204765081406, "learning_rate": 9.962e-06, "loss": 0.0004, "num_tokens": 4962568.0, "reward": 6.429768085479736, "reward_std": 0.4796310067176819, "rewards/CLIPTextReward/mean": 1.109709620475769, "rewards/CLIPTextReward/std": 0.22573811560869217, "rewards/HPSV21TextReward/mean": 0.8387088775634766, "rewards/HPSV21TextReward/std": 0.10024221241474152, "rewards/MANIQAReward/mean": 1.0161340236663818, "rewards/MANIQAReward/std": 0.17700877785682678, "rewards/QwenFakeDiscrimReward/mean": 0.12890625, "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, "rewards/QwenTextReward/mean": 1.1730469465255737, "rewards/QwenTextReward/std": 0.03627080097794533, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07653333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.40625, "kl": 0.004039062652736902, "learning_rate": 9.961866666666667e-06, "loss": 0.0004, "num_tokens": 4979920.0, "reward": 6.669560432434082, "reward_std": 0.3349897563457489, "rewards/CLIPTextReward/mean": 1.1544265747070312, "rewards/CLIPTextReward/std": 0.20579788833856583, "rewards/HPSV21TextReward/mean": 0.8574304580688477, "rewards/HPSV21TextReward/std": 0.1492508500814438, "rewards/MANIQAReward/mean": 1.031002163887024, "rewards/MANIQAReward/std": 0.1384875327348709, "rewards/QwenFakeDiscrimReward/mean": 0.21484375, "rewards/QwenFakeDiscrimReward/std": 0.0875956118106842, "rewards/QwenTextReward/mean": 1.157812476158142, "rewards/QwenTextReward/std": 0.03750002384185791, "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, "step": 287 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0768, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.004077456891536713, "learning_rate": 9.961733333333335e-06, "loss": 0.0004, "num_tokens": 4997352.0, "reward": 6.327436923980713, "reward_std": 0.4377618432044983, "rewards/CLIPTextReward/mean": 1.0494632124900818, "rewards/CLIPTextReward/std": 0.17974548041820526, "rewards/HPSV21TextReward/mean": 0.8615455627441406, "rewards/HPSV21TextReward/std": 0.12188783288002014, "rewards/MANIQAReward/mean": 1.0417473316192627, "rewards/MANIQAReward/std": 0.17893892526626587, "rewards/QwenFakeDiscrimReward/mean": 0.1328125, "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, "rewards/QwenTextReward/mean": 1.1472656726837158, "rewards/QwenTextReward/std": 0.06222037598490715, "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, "step": 288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07706666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.34375, "kl": 0.00398043030872941, "learning_rate": 9.961600000000001e-06, "loss": 0.0004, "num_tokens": 5014744.0, "reward": 6.2923078536987305, "reward_std": 0.41273707151412964, "rewards/CLIPTextReward/mean": 1.0740737915039062, "rewards/CLIPTextReward/std": 0.20466717332601547, "rewards/HPSV21TextReward/mean": 0.8345117568969727, "rewards/HPSV21TextReward/std": 0.06559011340141296, "rewards/MANIQAReward/mean": 0.963417649269104, "rewards/MANIQAReward/std": 0.16190515458583832, "rewards/QwenFakeDiscrimReward/mean": 0.1328125, "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, "rewards/QwenTextReward/mean": 1.16015625, "rewards/QwenTextReward/std": 0.03772258013486862, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 289 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07733333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "kl": 0.004222570452839136, "learning_rate": 9.961466666666668e-06, "loss": 0.0004, "num_tokens": 5032048.0, "reward": 6.57753849029541, "reward_std": 0.37709754705429077, "rewards/CLIPTextReward/mean": 1.1433212757110596, "rewards/CLIPTextReward/std": 0.18561596423387527, "rewards/HPSV21TextReward/mean": 0.8779010772705078, "rewards/HPSV21TextReward/std": 0.13092831522226334, "rewards/MANIQAReward/mean": 1.0280625820159912, "rewards/MANIQAReward/std": 0.1869654357433319, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.1554687023162842, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0776, "frac_reward_zero_std": 0.0, "grad_norm": 1.40625, "kl": 0.004480007104575634, "learning_rate": 9.961333333333334e-06, "loss": 0.0004, "num_tokens": 5049368.0, "reward": 6.583841800689697, "reward_std": 0.3834884762763977, "rewards/CLIPTextReward/mean": 1.1373045444488525, "rewards/CLIPTextReward/std": 0.20375385880470276, "rewards/HPSV21TextReward/mean": 0.8537569046020508, "rewards/HPSV21TextReward/std": 0.08211877197027206, "rewards/MANIQAReward/mean": 1.082578420639038, "rewards/MANIQAReward/std": 0.13105171918869019, "rewards/QwenFakeDiscrimReward/mean": 0.1484375, "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, "rewards/QwenTextReward/mean": 1.1519532203674316, "rewards/QwenTextReward/std": 0.03627080097794533, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 291 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07786666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.390625, "kl": 0.005785368382930756, "learning_rate": 9.9612e-06, "loss": 0.0006, "num_tokens": 5066744.0, "reward": 6.347508430480957, "reward_std": 0.36312463879585266, "rewards/CLIPTextReward/mean": 1.019821047782898, "rewards/CLIPTextReward/std": 0.1682324931025505, "rewards/HPSV21TextReward/mean": 0.8669137954711914, "rewards/HPSV21TextReward/std": 0.1736130267381668, "rewards/MANIQAReward/mean": 1.0318512916564941, "rewards/MANIQAReward/std": 0.15442673861980438, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.1593749523162842, "rewards/QwenTextReward/std": 0.06922188401222229, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07813333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.328125, "kl": 0.004109468311071396, "learning_rate": 9.961066666666669e-06, "loss": 0.0004, "num_tokens": 5084104.0, "reward": 6.442000389099121, "reward_std": 0.48246192932128906, "rewards/CLIPTextReward/mean": 1.1207281947135925, "rewards/CLIPTextReward/std": 0.19325513392686844, "rewards/HPSV21TextReward/mean": 0.8586606979370117, "rewards/HPSV21TextReward/std": 0.1198316216468811, "rewards/MANIQAReward/mean": 0.9582223296165466, "rewards/MANIQAReward/std": 0.16580310463905334, "rewards/QwenFakeDiscrimReward/mean": 0.16015625, "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, "rewards/QwenTextReward/mean": 1.1578125953674316, "rewards/QwenTextReward/std": 0.03750002756714821, "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, "step": 293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0784, "frac_reward_zero_std": 0.0, "grad_norm": 1.3671875, "kl": 0.004067155998200178, "learning_rate": 9.960933333333335e-06, "loss": 0.0004, "num_tokens": 5101344.0, "reward": 6.1931233406066895, "reward_std": 0.31273579597473145, "rewards/CLIPTextReward/mean": 0.9793084859848022, "rewards/CLIPTextReward/std": 0.17051473259925842, "rewards/HPSV21TextReward/mean": 0.8084526062011719, "rewards/HPSV21TextReward/std": 0.06458032876253128, "rewards/MANIQAReward/mean": 1.121117115020752, "rewards/MANIQAReward/std": 0.16443513333797455, "rewards/QwenFakeDiscrimReward/mean": 0.15234375, "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, "rewards/QwenTextReward/mean": 1.1449217796325684, "rewards/QwenTextReward/std": 0.03338678926229477, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07866666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 1.359375, "kl": 0.003640000708401203, "learning_rate": 9.960800000000001e-06, "loss": 0.0004, "num_tokens": 5118600.0, "reward": 6.4132184982299805, "reward_std": 0.4460734724998474, "rewards/CLIPTextReward/mean": 1.1124546527862549, "rewards/CLIPTextReward/std": 0.21535417437553406, "rewards/HPSV21TextReward/mean": 0.8255329132080078, "rewards/HPSV21TextReward/std": 0.06593691557645798, "rewards/MANIQAReward/mean": 1.0157592296600342, "rewards/MANIQAReward/std": 0.1600320190191269, "rewards/QwenFakeDiscrimReward/mean": 0.171875, "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, "rewards/QwenTextReward/mean": 1.134765625, "rewards/QwenTextReward/std": 0.08000710606575012, "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, "step": 295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07893333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 1.3828125, "kl": 0.003906239289790392, "learning_rate": 9.960666666666668e-06, "loss": 0.0004, "num_tokens": 5135992.0, "reward": 6.093643665313721, "reward_std": 0.3523331880569458, "rewards/CLIPTextReward/mean": 1.0198302865028381, "rewards/CLIPTextReward/std": 0.18580127507448196, "rewards/HPSV21TextReward/mean": 0.7943859100341797, "rewards/HPSV21TextReward/std": 0.061968978494405746, "rewards/MANIQAReward/mean": 0.9851330518722534, "rewards/MANIQAReward/std": 0.17740575969219208, "rewards/QwenFakeDiscrimReward/mean": 0.13671875, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.1441404819488525, "rewards/QwenTextReward/std": 0.06677703559398651, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.0792, "frac_reward_zero_std": 0.0, "grad_norm": 1.4140625, "kl": 0.003932563588023186, "learning_rate": 9.960533333333334e-06, "loss": 0.0004, "num_tokens": 5153304.0, "reward": 6.546805381774902, "reward_std": 0.3591303825378418, "rewards/CLIPTextReward/mean": 1.0995697379112244, "rewards/CLIPTextReward/std": 0.20298991352319717, "rewards/HPSV21TextReward/mean": 0.8044948577880859, "rewards/HPSV21TextReward/std": 0.08719000220298767, "rewards/MANIQAReward/mean": 1.1082077026367188, "rewards/MANIQAReward/std": 0.14398537576198578, "rewards/QwenFakeDiscrimReward/mean": 0.2109375, "rewards/QwenFakeDiscrimReward/std": 0.09149063378572464, "rewards/QwenTextReward/mean": 1.1695313453674316, "rewards/QwenTextReward/std": 0.03712613880634308, "rewards/QwenWeirdDiscrimReward/mean": 0.25, "rewards/QwenWeirdDiscrimReward/std": 0.0, "step": 297 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07946666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 1.390625, "kl": 0.004132324364036322, "learning_rate": 9.9604e-06, "loss": 0.0004, "num_tokens": 5170608.0, "reward": 6.462998390197754, "reward_std": 0.4576346278190613, "rewards/CLIPTextReward/mean": 1.1270618438720703, "rewards/CLIPTextReward/std": 0.19543292373418808, "rewards/HPSV21TextReward/mean": 0.8457469940185547, "rewards/HPSV21TextReward/std": 0.12634971737861633, "rewards/MANIQAReward/mean": 1.071286916732788, "rewards/MANIQAReward/std": 0.14734399318695068, "rewards/QwenFakeDiscrimReward/mean": 0.11328125, "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, "rewards/QwenTextReward/mean": 1.1335937976837158, "rewards/QwenTextReward/std": 0.06287093460559845, "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, "step": 298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.07973333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "kl": 0.003896570298820734, "learning_rate": 9.960266666666667e-06, "loss": 0.0004, "num_tokens": 5187856.0, "reward": 6.554800033569336, "reward_std": 0.4156927168369293, "rewards/CLIPTextReward/mean": 1.114656925201416, "rewards/CLIPTextReward/std": 0.19729189574718475, "rewards/HPSV21TextReward/mean": 0.860011100769043, "rewards/HPSV21TextReward/std": 0.13110067322850227, "rewards/MANIQAReward/mean": 1.0789012908935547, "rewards/MANIQAReward/std": 0.16471092402935028, "rewards/QwenFakeDiscrimReward/mean": 0.15625, "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, "rewards/QwenTextReward/mean": 1.1515625715255737, "rewards/QwenTextReward/std": 0.04473801702260971, "rewards/QwenWeirdDiscrimReward/mean": 0.21875, "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, "step": 299 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 256.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 256.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 256.0, "completions/min_terminated_length": 0.0, "epoch": 0.08, "frac_reward_zero_std": 0.0, "grad_norm": 1.3515625, "kl": 0.003982767462730408, "learning_rate": 9.960133333333333e-06, "loss": 0.0004, "num_tokens": 5205168.0, "reward": 6.407749176025391, "reward_std": 0.45647531747817993, "rewards/CLIPTextReward/mean": 1.0897849798202515, "rewards/CLIPTextReward/std": 0.21174846589565277, "rewards/HPSV21TextReward/mean": 0.8501195907592773, "rewards/HPSV21TextReward/std": 0.11032982170581818, "rewards/MANIQAReward/mean": 1.010362148284912, "rewards/MANIQAReward/std": 0.18477247655391693, "rewards/QwenFakeDiscrimReward/mean": 0.140625, "rewards/QwenFakeDiscrimReward/std": 0.125, "rewards/QwenTextReward/mean": 1.166015625, "rewards/QwenTextReward/std": 0.03763000667095184, "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, "step": 300 } ], "logging_steps": 1.0, "max_steps": 75000, "num_input_tokens_seen": 5205168, "num_train_epochs": 20, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 16, "trial_name": null, "trial_params": null }