diff --git "a/checkpoint-300/trainer_state.json" "b/checkpoint-300/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-300/trainer_state.json" @@ -0,0 +1,10834 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.08, + "eval_steps": 500, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0002666666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.00017316703451797366, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 17440.0, + "reward": 6.223865509033203, + "reward_std": 0.3913865387439728, + "rewards/CLIPTextReward/mean": 1.062528371810913, + "rewards/CLIPTextReward/std": 0.18956568092107773, + "rewards/HPSV21TextReward/mean": 0.8446283340454102, + "rewards/HPSV21TextReward/std": 0.13295899704098701, + "rewards/MANIQAReward/mean": 0.9095516800880432, + "rewards/MANIQAReward/std": 0.18011662364006042, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1484375, + "rewards/QwenTextReward/std": 0.03503827005624771, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 1 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0005333333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.00028873729752376676, + "learning_rate": 9.999866666666667e-06, + "loss": 0.0, + "num_tokens": 34768.0, + "reward": 6.316446304321289, + "reward_std": 0.43581461906433105, + "rewards/CLIPTextReward/mean": 1.1428767442703247, + "rewards/CLIPTextReward/std": 0.2098335102200508, + "rewards/HPSV21TextReward/mean": 0.8283510208129883, + "rewards/HPSV21TextReward/std": 0.0815248042345047, + "rewards/MANIQAReward/mean": 0.9150067567825317, + "rewards/MANIQAReward/std": 0.1554507613182068, + "rewards/QwenFakeDiscrimReward/mean": 0.1171875, + "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, + "rewards/QwenTextReward/mean": 1.154296875, + "rewards/QwenTextReward/std": 0.03688107803463936, + "rewards/QwenWeirdDiscrimReward/mean": 0.1875, + "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, + "step": 2 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0008, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.00028006985667161644, + "learning_rate": 9.999733333333335e-06, + "loss": 0.0, + "num_tokens": 52128.0, + "reward": 6.023024559020996, + "reward_std": 0.35070011019706726, + "rewards/CLIPTextReward/mean": 0.9945527911186218, + "rewards/CLIPTextReward/std": 0.16822098195552826, + "rewards/HPSV21TextReward/mean": 0.7806148529052734, + "rewards/HPSV21TextReward/std": 0.08261141180992126, + "rewards/MANIQAReward/mean": 0.9121421575546265, + "rewards/MANIQAReward/std": 0.21801920235157013, + "rewards/QwenFakeDiscrimReward/mean": 0.1875, + "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, + "rewards/QwenTextReward/mean": 1.146484375, + "rewards/QwenTextReward/std": 0.06738707423210144, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 3 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0010666666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.00030690443236380816, + "learning_rate": 9.9996e-06, + "loss": 0.0, + "num_tokens": 69472.0, + "reward": 6.15365743637085, + "reward_std": 0.38576382398605347, + "rewards/CLIPTextReward/mean": 1.0728851556777954, + "rewards/CLIPTextReward/std": 0.21109368652105331, + "rewards/HPSV21TextReward/mean": 0.8231821060180664, + "rewards/HPSV21TextReward/std": 0.07272331416606903, + "rewards/MANIQAReward/mean": 0.8876948952674866, + "rewards/MANIQAReward/std": 0.17049363255500793, + "rewards/QwenFakeDiscrimReward/mean": 0.12890625, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.1613280773162842, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, + "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, + "step": 4 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0013333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0003715873172041029, + "learning_rate": 9.999466666666668e-06, + "loss": 0.0, + "num_tokens": 86816.0, + "reward": 6.4580397605896, + "reward_std": 0.512778103351593, + "rewards/CLIPTextReward/mean": 1.1607547998428345, + "rewards/CLIPTextReward/std": 0.1884431168437004, + "rewards/HPSV21TextReward/mean": 0.844273567199707, + "rewards/HPSV21TextReward/std": 0.09659288078546524, + "rewards/MANIQAReward/mean": 0.9534515142440796, + "rewards/MANIQAReward/std": 0.1852920651435852, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1507811546325684, + "rewards/QwenTextReward/std": 0.03590354323387146, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 5 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0016, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.0003576505114324391, + "learning_rate": 9.999333333333334e-06, + "loss": 0.0, + "num_tokens": 104080.0, + "reward": 6.372293472290039, + "reward_std": 0.38334107398986816, + "rewards/CLIPTextReward/mean": 1.0605176091194153, + "rewards/CLIPTextReward/std": 0.19939535111188889, + "rewards/HPSV21TextReward/mean": 0.8251571655273438, + "rewards/HPSV21TextReward/std": 0.11583757400512695, + "rewards/MANIQAReward/mean": 1.0298501253128052, + "rewards/MANIQAReward/std": 0.20007139444351196, + "rewards/QwenFakeDiscrimReward/mean": 0.1875, + "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, + "rewards/QwenTextReward/mean": 1.1453125476837158, + "rewards/QwenTextReward/std": 0.08473525196313858, + "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, + "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, + "step": 6 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0018666666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0004226515593472868, + "learning_rate": 9.9992e-06, + "loss": 0.0, + "num_tokens": 121456.0, + "reward": 6.498950481414795, + "reward_std": 0.4029247760772705, + "rewards/CLIPTextReward/mean": 1.1567931771278381, + "rewards/CLIPTextReward/std": 0.18052731454372406, + "rewards/HPSV21TextReward/mean": 0.8444890975952148, + "rewards/HPSV21TextReward/std": 0.1347436159849167, + "rewards/MANIQAReward/mean": 0.9831047058105469, + "rewards/MANIQAReward/std": 0.15593110024929047, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.146093726158142, + "rewards/QwenTextReward/std": 0.03398728370666504, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 7 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0021333333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.296875, + "kl": 0.00035783814382739365, + "learning_rate": 9.999066666666669e-06, + "loss": 0.0, + "num_tokens": 138880.0, + "reward": 6.2124223709106445, + "reward_std": 0.4332669973373413, + "rewards/CLIPTextReward/mean": 1.1610087156295776, + "rewards/CLIPTextReward/std": 0.21405618637800217, + "rewards/HPSV21TextReward/mean": 0.8069028854370117, + "rewards/HPSV21TextReward/std": 0.08297182619571686, + "rewards/MANIQAReward/mean": 0.8281617164611816, + "rewards/MANIQAReward/std": 0.15055784583091736, + "rewards/QwenFakeDiscrimReward/mean": 0.12109375, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.1437500715255737, + "rewards/QwenTextReward/std": 0.03779647499322891, + "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, + "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, + "step": 8 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0024, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.00041289368527941406, + "learning_rate": 9.998933333333334e-06, + "loss": 0.0, + "num_tokens": 156200.0, + "reward": 6.300518989562988, + "reward_std": 0.4562382400035858, + "rewards/CLIPTextReward/mean": 1.0610955953598022, + "rewards/CLIPTextReward/std": 0.20977036654949188, + "rewards/HPSV21TextReward/mean": 0.8444728851318359, + "rewards/HPSV21TextReward/std": 0.08883989602327347, + "rewards/MANIQAReward/mean": 0.9604760408401489, + "rewards/MANIQAReward/std": 0.20047466456890106, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1304688453674316, + "rewards/QwenTextReward/std": 0.0844530537724495, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 9 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0026666666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0004245891759637743, + "learning_rate": 9.998800000000002e-06, + "loss": 0.0, + "num_tokens": 173536.0, + "reward": 6.316299915313721, + "reward_std": 0.441649466753006, + "rewards/CLIPTextReward/mean": 1.1014129519462585, + "rewards/CLIPTextReward/std": 0.20749090611934662, + "rewards/HPSV21TextReward/mean": 0.823883056640625, + "rewards/HPSV21TextReward/std": 0.07232779264450073, + "rewards/MANIQAReward/mean": 0.9332858324050903, + "rewards/MANIQAReward/std": 0.19034986197948456, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1417968273162842, + "rewards/QwenTextReward/std": 0.08393294364213943, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 10 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0029333333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0004424244980327785, + "learning_rate": 9.998666666666668e-06, + "loss": 0.0, + "num_tokens": 190824.0, + "reward": 6.211538314819336, + "reward_std": 0.2898920178413391, + "rewards/CLIPTextReward/mean": 1.0182899236679077, + "rewards/CLIPTextReward/std": 0.20580250024795532, + "rewards/HPSV21TextReward/mean": 0.7981195449829102, + "rewards/HPSV21TextReward/std": 0.08515609055757523, + "rewards/MANIQAReward/mean": 1.0154380798339844, + "rewards/MANIQAReward/std": 0.1735469251871109, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.1531250476837158, + "rewards/QwenTextReward/std": 0.03659627586603165, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 11 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0032, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.0004862526257056743, + "learning_rate": 9.998533333333334e-06, + "loss": 0.0, + "num_tokens": 208072.0, + "reward": 6.214302062988281, + "reward_std": 0.36605674028396606, + "rewards/CLIPTextReward/mean": 1.0899392366409302, + "rewards/CLIPTextReward/std": 0.18800289183855057, + "rewards/HPSV21TextReward/mean": 0.8150472640991211, + "rewards/HPSV21TextReward/std": 0.0666610449552536, + "rewards/MANIQAReward/mean": 0.9449541568756104, + "rewards/MANIQAReward/std": 0.21699047088623047, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1429686546325684, + "rewards/QwenTextReward/std": 0.06776899099349976, + "rewards/QwenWeirdDiscrimReward/mean": 0.16796875, + "rewards/QwenWeirdDiscrimReward/std": 0.1183105930685997, + "step": 12 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0034666666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.00047596223885193467, + "learning_rate": 9.9984e-06, + "loss": 0.0, + "num_tokens": 225400.0, + "reward": 6.290804862976074, + "reward_std": 0.370810329914093, + "rewards/CLIPTextReward/mean": 1.0436859726905823, + "rewards/CLIPTextReward/std": 0.18754911422729492, + "rewards/HPSV21TextReward/mean": 0.8343992233276367, + "rewards/HPSV21TextReward/std": 0.13207274675369263, + "rewards/MANIQAReward/mean": 1.0338529348373413, + "rewards/MANIQAReward/std": 0.1687883734703064, + "rewards/QwenFakeDiscrimReward/mean": 0.1640625, + "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, + "rewards/QwenTextReward/mean": 1.1257812976837158, + "rewards/QwenTextReward/std": 0.12182222306728363, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 13 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0037333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.0004692480724770576, + "learning_rate": 9.998266666666667e-06, + "loss": 0.0, + "num_tokens": 242768.0, + "reward": 6.429315567016602, + "reward_std": 0.3880622386932373, + "rewards/CLIPTextReward/mean": 1.1203209161758423, + "rewards/CLIPTextReward/std": 0.19741670042276382, + "rewards/HPSV21TextReward/mean": 0.8249845504760742, + "rewards/HPSV21TextReward/std": 0.05856947973370552, + "rewards/MANIQAReward/mean": 0.9805015921592712, + "rewards/MANIQAReward/std": 0.16263115406036377, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.148046851158142, + "rewards/QwenTextReward/std": 0.07859187573194504, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 14 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.004, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.265625, + "kl": 0.00043514688150025904, + "learning_rate": 9.998133333333333e-06, + "loss": 0.0, + "num_tokens": 260160.0, + "reward": 6.392142295837402, + "reward_std": 0.3614084720611572, + "rewards/CLIPTextReward/mean": 1.1251448392868042, + "rewards/CLIPTextReward/std": 0.18251177296042442, + "rewards/HPSV21TextReward/mean": 0.8510913848876953, + "rewards/HPSV21TextReward/std": 0.07994198799133301, + "rewards/MANIQAReward/mean": 0.9330289363861084, + "rewards/MANIQAReward/std": 0.15623073279857635, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1511719226837158, + "rewards/QwenTextReward/std": 0.06834637373685837, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 15 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.004266666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3203125, + "kl": 0.0004994600312784314, + "learning_rate": 9.998000000000002e-06, + "loss": 0.0, + "num_tokens": 277400.0, + "reward": 6.754908561706543, + "reward_std": 0.37522512674331665, + "rewards/CLIPTextReward/mean": 1.207650899887085, + "rewards/CLIPTextReward/std": 0.16807805746793747, + "rewards/HPSV21TextReward/mean": 0.8673477172851562, + "rewards/HPSV21TextReward/std": 0.0916677750647068, + "rewards/MANIQAReward/mean": 1.0424113273620605, + "rewards/MANIQAReward/std": 0.22193863987922668, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.171875, + "rewards/QwenTextReward/std": 0.03659627586603165, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 16 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.004533333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.53125, + "kl": 0.0008303101640194654, + "learning_rate": 9.997866666666668e-06, + "loss": 0.0001, + "num_tokens": 294712.0, + "reward": 6.324787139892578, + "reward_std": 0.4522237181663513, + "rewards/CLIPTextReward/mean": 1.104324460029602, + "rewards/CLIPTextReward/std": 0.19701702892780304, + "rewards/HPSV21TextReward/mean": 0.8078851699829102, + "rewards/HPSV21TextReward/std": 0.07371688634157181, + "rewards/MANIQAReward/mean": 0.9988051652908325, + "rewards/MANIQAReward/std": 0.1549089401960373, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03750002756714821, + "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, + "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, + "step": 17 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0048, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0005571045912802219, + "learning_rate": 9.997733333333334e-06, + "loss": 0.0001, + "num_tokens": 312136.0, + "reward": 6.244279384613037, + "reward_std": 0.4300357699394226, + "rewards/CLIPTextReward/mean": 1.0606692433357239, + "rewards/CLIPTextReward/std": 0.19169732928276062, + "rewards/HPSV21TextReward/mean": 0.8216180801391602, + "rewards/HPSV21TextReward/std": 0.09397384524345398, + "rewards/MANIQAReward/mean": 0.9781421422958374, + "rewards/MANIQAReward/std": 0.19963569939136505, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1500000953674316, + "rewards/QwenTextReward/std": 0.06813853234052658, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 18 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.005066666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0005525641608983278, + "learning_rate": 9.9976e-06, + "loss": 0.0001, + "num_tokens": 329464.0, + "reward": 6.356129169464111, + "reward_std": 0.3178771734237671, + "rewards/CLIPTextReward/mean": 1.0747361183166504, + "rewards/CLIPTextReward/std": 0.2152412012219429, + "rewards/HPSV21TextReward/mean": 0.8245840072631836, + "rewards/HPSV21TextReward/std": 0.07334348559379578, + "rewards/MANIQAReward/mean": 1.0098326206207275, + "rewards/MANIQAReward/std": 0.17456312477588654, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.1453125476837158, + "rewards/QwenTextReward/std": 0.0670931488275528, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 19 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.005333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3203125, + "kl": 0.000582196400500834, + "learning_rate": 9.997466666666667e-06, + "loss": 0.0001, + "num_tokens": 346704.0, + "reward": 6.182271957397461, + "reward_std": 0.347652792930603, + "rewards/CLIPTextReward/mean": 1.0524665117263794, + "rewards/CLIPTextReward/std": 0.19090761989355087, + "rewards/HPSV21TextReward/mean": 0.8114709854125977, + "rewards/HPSV21TextReward/std": 0.06487897783517838, + "rewards/MANIQAReward/mean": 0.91963130235672, + "rewards/MANIQAReward/std": 0.1376141756772995, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1519532203674316, + "rewards/QwenTextReward/std": 0.03627080097794533, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 20 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0056, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0006274660117924213, + "learning_rate": 9.997333333333335e-06, + "loss": 0.0001, + "num_tokens": 364112.0, + "reward": 6.596420764923096, + "reward_std": 0.455806165933609, + "rewards/CLIPTextReward/mean": 1.1512434482574463, + "rewards/CLIPTextReward/std": 0.20681702345609665, + "rewards/HPSV21TextReward/mean": 0.8719406127929688, + "rewards/HPSV21TextReward/std": 0.11565934866666794, + "rewards/MANIQAReward/mean": 1.0477088689804077, + "rewards/MANIQAReward/std": 0.13334345817565918, + "rewards/QwenFakeDiscrimReward/mean": 0.125, + "rewards/QwenFakeDiscrimReward/std": 0.1259881556034088, + "rewards/QwenTextReward/mean": 1.150781273841858, + "rewards/QwenTextReward/std": 0.03590353950858116, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 21 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.005866666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3203125, + "kl": 0.0005800112849101424, + "learning_rate": 9.997200000000001e-06, + "loss": 0.0001, + "num_tokens": 381584.0, + "reward": 6.364524841308594, + "reward_std": 0.4369554817676544, + "rewards/CLIPTextReward/mean": 1.150652289390564, + "rewards/CLIPTextReward/std": 0.2127503752708435, + "rewards/HPSV21TextReward/mean": 0.8205938339233398, + "rewards/HPSV21TextReward/std": 0.10877228528261185, + "rewards/MANIQAReward/mean": 0.9673448801040649, + "rewards/MANIQAReward/std": 0.144505575299263, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.146093726158142, + "rewards/QwenTextReward/std": 0.03398728370666504, + "rewards/QwenWeirdDiscrimReward/mean": 0.1640625, + "rewards/QwenWeirdDiscrimReward/std": 0.11967839300632477, + "step": 22 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0061333333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.000612370204180479, + "learning_rate": 9.997066666666668e-06, + "loss": 0.0001, + "num_tokens": 398896.0, + "reward": 6.500385761260986, + "reward_std": 0.37200304865837097, + "rewards/CLIPTextReward/mean": 1.1138979196548462, + "rewards/CLIPTextReward/std": 0.18904101103544235, + "rewards/HPSV21TextReward/mean": 0.8669729232788086, + "rewards/HPSV21TextReward/std": 0.1344263069331646, + "rewards/MANIQAReward/mean": 0.9835654497146606, + "rewards/MANIQAReward/std": 0.1934371143579483, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.156640648841858, + "rewards/QwenTextReward/std": 0.037332236766815186, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 23 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0064, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0006386711029335856, + "learning_rate": 9.996933333333334e-06, + "loss": 0.0001, + "num_tokens": 416392.0, + "reward": 6.323739051818848, + "reward_std": 0.4089418649673462, + "rewards/CLIPTextReward/mean": 1.0975658893585205, + "rewards/CLIPTextReward/std": 0.1965111717581749, + "rewards/HPSV21TextReward/mean": 0.8068838119506836, + "rewards/HPSV21TextReward/std": 0.08229941129684448, + "rewards/MANIQAReward/mean": 0.9601527452468872, + "rewards/MANIQAReward/std": 0.1424037218093872, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.040019553154706955, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 24 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.006666666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.265625, + "kl": 0.0006111857946962118, + "learning_rate": 9.9968e-06, + "loss": 0.0001, + "num_tokens": 433824.0, + "reward": 6.292370796203613, + "reward_std": 0.3929201662540436, + "rewards/CLIPTextReward/mean": 1.1571096181869507, + "rewards/CLIPTextReward/std": 0.2031644508242607, + "rewards/HPSV21TextReward/mean": 0.7992210388183594, + "rewards/HPSV21TextReward/std": 0.06519654393196106, + "rewards/MANIQAReward/mean": 0.9660371541976929, + "rewards/MANIQAReward/std": 0.16348601877689362, + "rewards/QwenFakeDiscrimReward/mean": 0.11328125, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.140234351158142, + "rewards/QwenTextReward/std": 0.08037824183702469, + "rewards/QwenWeirdDiscrimReward/mean": 0.16015625, + "rewards/QwenWeirdDiscrimReward/std": 0.12090259045362473, + "step": 25 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.006933333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.000643837614916265, + "learning_rate": 9.996666666666669e-06, + "loss": 0.0001, + "num_tokens": 451288.0, + "reward": 6.366451740264893, + "reward_std": 0.44354701042175293, + "rewards/CLIPTextReward/mean": 1.1308492422103882, + "rewards/CLIPTextReward/std": 0.19625339657068253, + "rewards/HPSV21TextReward/mean": 0.8046188354492188, + "rewards/HPSV21TextReward/std": 0.06655003875494003, + "rewards/MANIQAReward/mean": 0.9287184476852417, + "rewards/MANIQAReward/std": 0.16072098910808563, + "rewards/QwenFakeDiscrimReward/mean": 0.19140625, + "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, + "rewards/QwenTextReward/mean": 1.168359398841858, + "rewards/QwenTextReward/std": 0.03965182974934578, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 26 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0072, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.0006543556228280067, + "learning_rate": 9.996533333333333e-06, + "loss": 0.0001, + "num_tokens": 468712.0, + "reward": 6.34525728225708, + "reward_std": 0.44636788964271545, + "rewards/CLIPTextReward/mean": 1.1022841930389404, + "rewards/CLIPTextReward/std": 0.20151617377996445, + "rewards/HPSV21TextReward/mean": 0.8236732482910156, + "rewards/HPSV21TextReward/std": 0.0780109316110611, + "rewards/MANIQAReward/mean": 0.9542800784111023, + "rewards/MANIQAReward/std": 0.15946204960346222, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 27 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.007466666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2734375, + "kl": 0.0005756819737143815, + "learning_rate": 9.996400000000001e-06, + "loss": 0.0001, + "num_tokens": 485992.0, + "reward": 6.045117378234863, + "reward_std": 0.29585862159729004, + "rewards/CLIPTextReward/mean": 1.0063416957855225, + "rewards/CLIPTextReward/std": 0.1854785531759262, + "rewards/HPSV21TextReward/mean": 0.7990350723266602, + "rewards/HPSV21TextReward/std": 0.08045350760221481, + "rewards/MANIQAReward/mean": 0.9308480024337769, + "rewards/MANIQAReward/std": 0.13604064285755157, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1480469703674316, + "rewards/QwenTextReward/std": 0.0706130787730217, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 28 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.007733333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.000711756176315248, + "learning_rate": 9.996266666666668e-06, + "loss": 0.0001, + "num_tokens": 503344.0, + "reward": 6.625212669372559, + "reward_std": 0.37315335869789124, + "rewards/CLIPTextReward/mean": 1.1993611454963684, + "rewards/CLIPTextReward/std": 0.18592771887779236, + "rewards/HPSV21TextReward/mean": 0.8432188034057617, + "rewards/HPSV21TextReward/std": 0.0620620995759964, + "rewards/MANIQAReward/mean": 1.0455212593078613, + "rewards/MANIQAReward/std": 0.16021911799907684, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.150781273841858, + "rewards/QwenTextReward/std": 0.03590353950858116, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 29 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.008, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.000612220203038305, + "learning_rate": 9.996133333333334e-06, + "loss": 0.0001, + "num_tokens": 520648.0, + "reward": 6.518412113189697, + "reward_std": 0.40200844407081604, + "rewards/CLIPTextReward/mean": 1.1843986511230469, + "rewards/CLIPTextReward/std": 0.19694440811872482, + "rewards/HPSV21TextReward/mean": 0.8349838256835938, + "rewards/HPSV21TextReward/std": 0.0758972018957138, + "rewards/MANIQAReward/mean": 0.9241786003112793, + "rewards/MANIQAReward/std": 0.14468914270401, + "rewards/QwenFakeDiscrimReward/mean": 0.1875, + "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, + "rewards/QwenTextReward/mean": 1.1531250476837158, + "rewards/QwenTextReward/std": 0.08183173090219498, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 30 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.008266666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0005961424903944135, + "learning_rate": 9.996e-06, + "loss": 0.0001, + "num_tokens": 538040.0, + "reward": 6.067929267883301, + "reward_std": 0.3891369700431824, + "rewards/CLIPTextReward/mean": 1.0164735317230225, + "rewards/CLIPTextReward/std": 0.19765233993530273, + "rewards/HPSV21TextReward/mean": 0.7834997177124023, + "rewards/HPSV21TextReward/std": 0.07358982414007187, + "rewards/MANIQAReward/mean": 0.9195454716682434, + "rewards/MANIQAReward/std": 0.1801491528749466, + "rewards/QwenFakeDiscrimReward/mean": 0.19921875, + "rewards/QwenFakeDiscrimReward/std": 0.10137632489204407, + "rewards/QwenTextReward/mean": 1.1343750953674316, + "rewards/QwenTextReward/std": 0.11456441134214401, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 31 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.008533333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.265625, + "kl": 0.0006076592253521085, + "learning_rate": 9.995866666666667e-06, + "loss": 0.0001, + "num_tokens": 555432.0, + "reward": 6.158683776855469, + "reward_std": 0.37738487124443054, + "rewards/CLIPTextReward/mean": 1.07703697681427, + "rewards/CLIPTextReward/std": 0.18969328701496124, + "rewards/HPSV21TextReward/mean": 0.8205890655517578, + "rewards/HPSV21TextReward/std": 0.10732360929250717, + "rewards/MANIQAReward/mean": 0.8689001798629761, + "rewards/MANIQAReward/std": 0.15288065373897552, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1429686546325684, + "rewards/QwenTextReward/std": 0.06643842905759811, + "rewards/QwenWeirdDiscrimReward/mean": 0.1796875, + "rewards/QwenWeirdDiscrimReward/std": 0.11329087615013123, + "step": 32 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0088, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4453125, + "kl": 0.0007134638144634664, + "learning_rate": 9.995733333333333e-06, + "loss": 0.0001, + "num_tokens": 572752.0, + "reward": 6.217061996459961, + "reward_std": 0.3514450490474701, + "rewards/CLIPTextReward/mean": 1.031137228012085, + "rewards/CLIPTextReward/std": 0.20571595430374146, + "rewards/HPSV21TextReward/mean": 0.8295192718505859, + "rewards/HPSV21TextReward/std": 0.09005855023860931, + "rewards/MANIQAReward/mean": 0.9406708478927612, + "rewards/MANIQAReward/std": 0.15536098182201385, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.156640648841858, + "rewards/QwenTextReward/std": 0.037332236766815186, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 33 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.009066666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0006718998192809522, + "learning_rate": 9.995600000000001e-06, + "loss": 0.0001, + "num_tokens": 590176.0, + "reward": 6.396887302398682, + "reward_std": 0.35195744037628174, + "rewards/CLIPTextReward/mean": 1.087852418422699, + "rewards/CLIPTextReward/std": 0.2179098054766655, + "rewards/HPSV21TextReward/mean": 0.8623418807983398, + "rewards/HPSV21TextReward/std": 0.12201350927352905, + "rewards/MANIQAReward/mean": 0.9789204001426697, + "rewards/MANIQAReward/std": 0.1579286903142929, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.158203125, + "rewards/QwenTextReward/std": 0.04250461608171463, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 34 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.009333333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0006477828719653189, + "learning_rate": 9.995466666666668e-06, + "loss": 0.0001, + "num_tokens": 607512.0, + "reward": 6.4933390617370605, + "reward_std": 0.4455246329307556, + "rewards/CLIPTextReward/mean": 1.1464911699295044, + "rewards/CLIPTextReward/std": 0.20508115738630295, + "rewards/HPSV21TextReward/mean": 0.8466615676879883, + "rewards/HPSV21TextReward/std": 0.06445857137441635, + "rewards/MANIQAReward/mean": 0.9734399318695068, + "rewards/MANIQAReward/std": 0.18179957568645477, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.162500023841858, + "rewards/QwenTextReward/std": 0.040089212357997894, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 35 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0096, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.0006656671175733209, + "learning_rate": 9.995333333333334e-06, + "loss": 0.0001, + "num_tokens": 624928.0, + "reward": 6.222779750823975, + "reward_std": 0.513312578201294, + "rewards/CLIPTextReward/mean": 1.0915876626968384, + "rewards/CLIPTextReward/std": 0.21901841461658478, + "rewards/HPSV21TextReward/mean": 0.8346834182739258, + "rewards/HPSV21TextReward/std": 0.07725655287504196, + "rewards/MANIQAReward/mean": 0.8718000054359436, + "rewards/MANIQAReward/std": 0.19936029613018036, + "rewards/QwenFakeDiscrimReward/mean": 0.1328125, + "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, + "rewards/QwenTextReward/mean": 1.154687523841858, + "rewards/QwenTextReward/std": 0.06664808094501495, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 36 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.009866666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.515625, + "kl": 0.0007665607845410705, + "learning_rate": 9.9952e-06, + "loss": 0.0001, + "num_tokens": 642304.0, + "reward": 6.087282180786133, + "reward_std": 0.4493650197982788, + "rewards/CLIPTextReward/mean": 1.0269891023635864, + "rewards/CLIPTextReward/std": 0.19465582817792892, + "rewards/HPSV21TextReward/mean": 0.8145656585693359, + "rewards/HPSV21TextReward/std": 0.06945210695266724, + "rewards/MANIQAReward/mean": 0.908078670501709, + "rewards/MANIQAReward/std": 0.15028269588947296, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.15234375, + "rewards/QwenTextReward/std": 0.06853322684764862, + "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, + "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, + "step": 37 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.010133333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.0006748391315340996, + "learning_rate": 9.995066666666667e-06, + "loss": 0.0001, + "num_tokens": 659616.0, + "reward": 6.521594047546387, + "reward_std": 0.28268447518348694, + "rewards/CLIPTextReward/mean": 1.1239572763442993, + "rewards/CLIPTextReward/std": 0.19386181235313416, + "rewards/HPSV21TextReward/mean": 0.8642807006835938, + "rewards/HPSV21TextReward/std": 0.16283436119556427, + "rewards/MANIQAReward/mean": 0.9767587184906006, + "rewards/MANIQAReward/std": 0.13164694607257843, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.154296875, + "rewards/QwenTextReward/std": 0.03688107803463936, + "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, + "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, + "step": 38 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0104, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.0006359356339089572, + "learning_rate": 9.994933333333335e-06, + "loss": 0.0001, + "num_tokens": 677008.0, + "reward": 6.220743179321289, + "reward_std": 0.41407012939453125, + "rewards/CLIPTextReward/mean": 1.0410704612731934, + "rewards/CLIPTextReward/std": 0.16566266119480133, + "rewards/HPSV21TextReward/mean": 0.8238363265991211, + "rewards/HPSV21TextReward/std": 0.10602444410324097, + "rewards/MANIQAReward/mean": 0.9217893481254578, + "rewards/MANIQAReward/std": 0.1595626324415207, + "rewards/QwenFakeDiscrimReward/mean": 0.1953125, + "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, + "rewards/QwenTextReward/mean": 1.1472656726837158, + "rewards/QwenTextReward/std": 0.03453698009252548, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 39 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.010666666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.000720483367331326, + "learning_rate": 9.994800000000001e-06, + "loss": 0.0001, + "num_tokens": 694360.0, + "reward": 6.090251922607422, + "reward_std": 0.3836488425731659, + "rewards/CLIPTextReward/mean": 1.0377899408340454, + "rewards/CLIPTextReward/std": 0.20379655063152313, + "rewards/HPSV21TextReward/mean": 0.8056297302246094, + "rewards/HPSV21TextReward/std": 0.0683428943157196, + "rewards/MANIQAReward/mean": 0.9120062589645386, + "rewards/MANIQAReward/std": 0.1517135202884674, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1554687023162842, + "rewards/QwenTextReward/std": 0.03945784643292427, + "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, + "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, + "step": 40 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.010933333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.25, + "kl": 0.0007212897762656212, + "learning_rate": 9.994666666666668e-06, + "loss": 0.0001, + "num_tokens": 711672.0, + "reward": 6.369787216186523, + "reward_std": 0.3154414892196655, + "rewards/CLIPTextReward/mean": 1.116544485092163, + "rewards/CLIPTextReward/std": 0.2094419151544571, + "rewards/HPSV21TextReward/mean": 0.8303442001342773, + "rewards/HPSV21TextReward/std": 0.06440813094377518, + "rewards/MANIQAReward/mean": 0.969369113445282, + "rewards/MANIQAReward/std": 0.19423282146453857, + "rewards/QwenFakeDiscrimReward/mean": 0.13671875, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.1707031726837158, + "rewards/QwenTextReward/std": 0.03688107430934906, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 41 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0112, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4375, + "kl": 0.0007738057756796479, + "learning_rate": 9.994533333333334e-06, + "loss": 0.0001, + "num_tokens": 729000.0, + "reward": 6.430874347686768, + "reward_std": 0.43917739391326904, + "rewards/CLIPTextReward/mean": 1.0832951068878174, + "rewards/CLIPTextReward/std": 0.19060203433036804, + "rewards/HPSV21TextReward/mean": 0.8397102355957031, + "rewards/HPSV21TextReward/std": 0.11080950498580933, + "rewards/MANIQAReward/mean": 1.0235352516174316, + "rewards/MANIQAReward/std": 0.15490847826004028, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1511719226837158, + "rewards/QwenTextReward/std": 0.06834637373685837, + "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, + "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, + "step": 42 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.011466666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.000737136579118669, + "learning_rate": 9.9944e-06, + "loss": 0.0001, + "num_tokens": 746304.0, + "reward": 6.281833648681641, + "reward_std": 0.3190336525440216, + "rewards/CLIPTextReward/mean": 1.044514775276184, + "rewards/CLIPTextReward/std": 0.18819903582334518, + "rewards/HPSV21TextReward/mean": 0.8453836441040039, + "rewards/HPSV21TextReward/std": 0.05973060056567192, + "rewards/MANIQAReward/mean": 0.9825053215026855, + "rewards/MANIQAReward/std": 0.16364580392837524, + "rewards/QwenFakeDiscrimReward/mean": 0.1640625, + "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, + "rewards/QwenTextReward/mean": 1.1484375, + "rewards/QwenTextReward/std": 0.03503827005624771, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 43 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.011733333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0007885091472417116, + "learning_rate": 9.994266666666668e-06, + "loss": 0.0001, + "num_tokens": 763560.0, + "reward": 6.076322555541992, + "reward_std": 0.45350897312164307, + "rewards/CLIPTextReward/mean": 1.0211127996444702, + "rewards/CLIPTextReward/std": 0.19371271133422852, + "rewards/HPSV21TextReward/mean": 0.8039369583129883, + "rewards/HPSV21TextReward/std": 0.07984976470470428, + "rewards/MANIQAReward/mean": 0.9441919326782227, + "rewards/MANIQAReward/std": 0.15260986983776093, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1500000953674316, + "rewards/QwenTextReward/std": 0.040824852883815765, + "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, + "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, + "step": 44 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.012, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0007469256524927914, + "learning_rate": 9.994133333333335e-06, + "loss": 0.0001, + "num_tokens": 780912.0, + "reward": 6.200069427490234, + "reward_std": 0.4303773045539856, + "rewards/CLIPTextReward/mean": 1.0523357391357422, + "rewards/CLIPTextReward/std": 0.20511925965547562, + "rewards/HPSV21TextReward/mean": 0.850214958190918, + "rewards/HPSV21TextReward/std": 0.10265272110700607, + "rewards/MANIQAReward/mean": 0.9160617589950562, + "rewards/MANIQAReward/std": 0.16560041904449463, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.150781273841858, + "rewards/QwenTextReward/std": 0.06298917531967163, + "rewards/QwenWeirdDiscrimReward/mean": 0.1875, + "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, + "step": 45 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.012266666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4296875, + "kl": 0.0008448648732155561, + "learning_rate": 9.994000000000001e-06, + "loss": 0.0001, + "num_tokens": 798256.0, + "reward": 6.494973182678223, + "reward_std": 0.36463749408721924, + "rewards/CLIPTextReward/mean": 1.0624587535858154, + "rewards/CLIPTextReward/std": 0.17267398536205292, + "rewards/HPSV21TextReward/mean": 0.8356428146362305, + "rewards/HPSV21TextReward/std": 0.13331294804811478, + "rewards/MANIQAReward/mean": 1.0733790397644043, + "rewards/MANIQAReward/std": 0.17399130761623383, + "rewards/QwenFakeDiscrimReward/mean": 0.22265625, + "rewards/QwenFakeDiscrimReward/std": 0.07864411175251007, + "rewards/QwenTextReward/mean": 1.156640648841858, + "rewards/QwenTextReward/std": 0.037332240492105484, + "rewards/QwenWeirdDiscrimReward/mean": 0.24609375, + "rewards/QwenWeirdDiscrimReward/std": 0.03125, + "step": 46 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.012533333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.0007328381179831922, + "learning_rate": 9.993866666666668e-06, + "loss": 0.0001, + "num_tokens": 815576.0, + "reward": 6.38761043548584, + "reward_std": 0.3840087056159973, + "rewards/CLIPTextReward/mean": 1.13616943359375, + "rewards/CLIPTextReward/std": 0.19060511887073517, + "rewards/HPSV21TextReward/mean": 0.8550443649291992, + "rewards/HPSV21TextReward/std": 0.07190331816673279, + "rewards/MANIQAReward/mean": 0.8887766599655151, + "rewards/MANIQAReward/std": 0.17912423610687256, + "rewards/QwenFakeDiscrimReward/mean": 0.1640625, + "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, + "rewards/QwenTextReward/mean": 1.1453125476837158, + "rewards/QwenTextReward/std": 0.0648372694849968, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 47 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0128, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0007853477727621794, + "learning_rate": 9.993733333333334e-06, + "loss": 0.0001, + "num_tokens": 832856.0, + "reward": 6.501211643218994, + "reward_std": 0.43350669741630554, + "rewards/CLIPTextReward/mean": 1.1371588110923767, + "rewards/CLIPTextReward/std": 0.20060991495847702, + "rewards/HPSV21TextReward/mean": 0.8673725128173828, + "rewards/HPSV21TextReward/std": 0.12160860747098923, + "rewards/MANIQAReward/mean": 0.9319926500320435, + "rewards/MANIQAReward/std": 0.18476037681102753, + "rewards/QwenFakeDiscrimReward/mean": 0.19140625, + "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, + "rewards/QwenTextReward/mean": 1.161718726158142, + "rewards/QwenTextReward/std": 0.04796842113137245, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 48 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.013066666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0008283444913104177, + "learning_rate": 9.9936e-06, + "loss": 0.0001, + "num_tokens": 850200.0, + "reward": 6.140354156494141, + "reward_std": 0.4246932864189148, + "rewards/CLIPTextReward/mean": 1.0640084743499756, + "rewards/CLIPTextReward/std": 0.19425738602876663, + "rewards/HPSV21TextReward/mean": 0.8056106567382812, + "rewards/HPSV21TextReward/std": 0.07409681379795074, + "rewards/MANIQAReward/mean": 0.9229907989501953, + "rewards/MANIQAReward/std": 0.19340187311172485, + "rewards/QwenFakeDiscrimReward/mean": 0.1328125, + "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, + "rewards/QwenTextReward/mean": 1.157812476158142, + "rewards/QwenTextReward/std": 0.03980984166264534, + "rewards/QwenWeirdDiscrimReward/mean": 0.1875, + "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, + "step": 49 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.013333333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0007959783542901278, + "learning_rate": 9.993466666666668e-06, + "loss": 0.0001, + "num_tokens": 867576.0, + "reward": 6.259000778198242, + "reward_std": 0.5205031037330627, + "rewards/CLIPTextReward/mean": 1.073885202407837, + "rewards/CLIPTextReward/std": 0.19922605156898499, + "rewards/HPSV21TextReward/mean": 0.8068361282348633, + "rewards/HPSV21TextReward/std": 0.07939974218606949, + "rewards/MANIQAReward/mean": 0.9553705453872681, + "rewards/MANIQAReward/std": 0.16773752868175507, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1476562023162842, + "rewards/QwenTextReward/std": 0.04425728693604469, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 50 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0136, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.0007928041159175336, + "learning_rate": 9.993333333333333e-06, + "loss": 0.0001, + "num_tokens": 885056.0, + "reward": 6.2416276931762695, + "reward_std": 0.33849644660949707, + "rewards/CLIPTextReward/mean": 1.0327467322349548, + "rewards/CLIPTextReward/std": 0.21467745304107666, + "rewards/HPSV21TextReward/mean": 0.8171548843383789, + "rewards/HPSV21TextReward/std": 0.07690542936325073, + "rewards/MANIQAReward/mean": 1.025027871131897, + "rewards/MANIQAReward/std": 0.1482948362827301, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1339843273162842, + "rewards/QwenTextReward/std": 0.10013555735349655, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 51 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.013866666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.0008756698807701468, + "learning_rate": 9.993200000000001e-06, + "loss": 0.0001, + "num_tokens": 902304.0, + "reward": 6.167211532592773, + "reward_std": 0.5379205346107483, + "rewards/CLIPTextReward/mean": 1.0588380098342896, + "rewards/CLIPTextReward/std": 0.19965583831071854, + "rewards/HPSV21TextReward/mean": 0.7852449417114258, + "rewards/HPSV21TextReward/std": 0.08360420167446136, + "rewards/MANIQAReward/mean": 0.9634205102920532, + "rewards/MANIQAReward/std": 0.16673992574214935, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 52 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.014133333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4296875, + "kl": 0.0009323436534032226, + "learning_rate": 9.993066666666668e-06, + "loss": 0.0001, + "num_tokens": 919728.0, + "reward": 6.469545364379883, + "reward_std": 0.3471949100494385, + "rewards/CLIPTextReward/mean": 1.1538089513778687, + "rewards/CLIPTextReward/std": 0.20266949385404587, + "rewards/HPSV21TextReward/mean": 0.7827653884887695, + "rewards/HPSV21TextReward/std": 0.08379530906677246, + "rewards/MANIQAReward/mean": 1.067490816116333, + "rewards/MANIQAReward/std": 0.17146986722946167, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.146093726158142, + "rewards/QwenTextReward/std": 0.03398728370666504, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 53 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0144, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0008969584014266729, + "learning_rate": 9.992933333333334e-06, + "loss": 0.0001, + "num_tokens": 937080.0, + "reward": 6.236813545227051, + "reward_std": 0.3845016658306122, + "rewards/CLIPTextReward/mean": 1.0523546934127808, + "rewards/CLIPTextReward/std": 0.19251258671283722, + "rewards/HPSV21TextReward/mean": 0.8053016662597656, + "rewards/HPSV21TextReward/std": 0.1164521649479866, + "rewards/MANIQAReward/mean": 0.9812659025192261, + "rewards/MANIQAReward/std": 0.1439070701599121, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1613280773162842, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 54 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.014666666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0008943352149799466, + "learning_rate": 9.9928e-06, + "loss": 0.0001, + "num_tokens": 954624.0, + "reward": 6.408685684204102, + "reward_std": 0.36431431770324707, + "rewards/CLIPTextReward/mean": 1.1066354513168335, + "rewards/CLIPTextReward/std": 0.1999390497803688, + "rewards/HPSV21TextReward/mean": 0.832585334777832, + "rewards/HPSV21TextReward/std": 0.08066749572753906, + "rewards/MANIQAReward/mean": 1.012665867805481, + "rewards/MANIQAReward/std": 0.17072820663452148, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.142578125, + "rewards/QwenTextReward/std": 0.032021742314100266, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 55 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.014933333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0008820436196401715, + "learning_rate": 9.992666666666667e-06, + "loss": 0.0001, + "num_tokens": 971936.0, + "reward": 6.167948246002197, + "reward_std": 0.2834055423736572, + "rewards/CLIPTextReward/mean": 0.9877440929412842, + "rewards/CLIPTextReward/std": 0.1475164331495762, + "rewards/HPSV21TextReward/mean": 0.8200302124023438, + "rewards/HPSV21TextReward/std": 0.08542144298553467, + "rewards/MANIQAReward/mean": 0.9781813621520996, + "rewards/MANIQAReward/std": 0.1652689427137375, + "rewards/QwenFakeDiscrimReward/mean": 0.1953125, + "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, + "rewards/QwenTextReward/mean": 1.15234375, + "rewards/QwenTextReward/std": 0.07672860473394394, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 56 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0152, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.0008737280732020736, + "learning_rate": 9.992533333333335e-06, + "loss": 0.0001, + "num_tokens": 989296.0, + "reward": 6.1255717277526855, + "reward_std": 0.47202640771865845, + "rewards/CLIPTextReward/mean": 1.0381349325180054, + "rewards/CLIPTextReward/std": 0.1850999891757965, + "rewards/HPSV21TextReward/mean": 0.810758113861084, + "rewards/HPSV21TextReward/std": 0.07670347392559052, + "rewards/MANIQAReward/mean": 0.9480981826782227, + "rewards/MANIQAReward/std": 0.17108558118343353, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1398437023162842, + "rewards/QwenTextReward/std": 0.08748229593038559, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 57 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.015466666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.0010484395315870643, + "learning_rate": 9.992400000000001e-06, + "loss": 0.0001, + "num_tokens": 1006584.0, + "reward": 6.327513217926025, + "reward_std": 0.4302184581756592, + "rewards/CLIPTextReward/mean": 1.0683895349502563, + "rewards/CLIPTextReward/std": 0.20896943658590317, + "rewards/HPSV21TextReward/mean": 0.8449268341064453, + "rewards/HPSV21TextReward/std": 0.18004687875509262, + "rewards/MANIQAReward/mean": 0.9700211882591248, + "rewards/MANIQAReward/std": 0.19353777170181274, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1636719703674316, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 58 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.015733333333333332, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.0008922148845158517, + "learning_rate": 9.992266666666667e-06, + "loss": 0.0001, + "num_tokens": 1024008.0, + "reward": 6.32124662399292, + "reward_std": 0.3739131689071655, + "rewards/CLIPTextReward/mean": 1.0668448209762573, + "rewards/CLIPTextReward/std": 0.19243191927671432, + "rewards/HPSV21TextReward/mean": 0.8356704711914062, + "rewards/HPSV21TextReward/std": 0.07387026399374008, + "rewards/MANIQAReward/mean": 0.9986376166343689, + "rewards/MANIQAReward/std": 0.1391114592552185, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.138671875, + "rewards/QwenTextReward/std": 0.09469351917505264, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 59 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.016, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.0008508785394951701, + "learning_rate": 9.992133333333334e-06, + "loss": 0.0001, + "num_tokens": 1041288.0, + "reward": 6.385502815246582, + "reward_std": 0.37456315755844116, + "rewards/CLIPTextReward/mean": 1.0719799399375916, + "rewards/CLIPTextReward/std": 0.19807685166597366, + "rewards/HPSV21TextReward/mean": 0.8345270156860352, + "rewards/HPSV21TextReward/std": 0.10260890424251556, + "rewards/MANIQAReward/mean": 1.0138952732086182, + "rewards/MANIQAReward/std": 0.16401827335357666, + "rewards/QwenFakeDiscrimReward/mean": 0.20703125, + "rewards/QwenFakeDiscrimReward/std": 0.09506355226039886, + "rewards/QwenTextReward/mean": 1.1484375, + "rewards/QwenTextReward/std": 0.03503827005624771, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 60 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.016266666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.000956534524448216, + "learning_rate": 9.992e-06, + "loss": 0.0001, + "num_tokens": 1058704.0, + "reward": 6.24791145324707, + "reward_std": 0.3534240126609802, + "rewards/CLIPTextReward/mean": 1.087317705154419, + "rewards/CLIPTextReward/std": 0.2023688331246376, + "rewards/HPSV21TextReward/mean": 0.7986822128295898, + "rewards/HPSV21TextReward/std": 0.06513593345880508, + "rewards/MANIQAReward/mean": 0.89153653383255, + "rewards/MANIQAReward/std": 0.1612655520439148, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.174218773841858, + "rewards/QwenTextReward/std": 0.03590353950858116, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 61 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.016533333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0009708399884402752, + "learning_rate": 9.991866666666668e-06, + "loss": 0.0001, + "num_tokens": 1076168.0, + "reward": 6.217422962188721, + "reward_std": 0.3819130063056946, + "rewards/CLIPTextReward/mean": 1.085562825202942, + "rewards/CLIPTextReward/std": 0.1930897980928421, + "rewards/HPSV21TextReward/mean": 0.7957496643066406, + "rewards/HPSV21TextReward/std": 0.08908992260694504, + "rewards/MANIQAReward/mean": 0.9255013465881348, + "rewards/MANIQAReward/std": 0.15410436689853668, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.166015625, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 62 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0168, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.296875, + "kl": 0.0009530046954751015, + "learning_rate": 9.991733333333335e-06, + "loss": 0.0001, + "num_tokens": 1093560.0, + "reward": 6.324612140655518, + "reward_std": 0.32763224840164185, + "rewards/CLIPTextReward/mean": 1.0497437715530396, + "rewards/CLIPTextReward/std": 0.19730062782764435, + "rewards/HPSV21TextReward/mean": 0.8403587341308594, + "rewards/HPSV21TextReward/std": 0.11551825702190399, + "rewards/MANIQAReward/mean": 0.9768288731575012, + "rewards/MANIQAReward/std": 0.18326574563980103, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1652344465255737, + "rewards/QwenTextReward/std": 0.06923867762088776, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 63 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.017066666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.390625, + "kl": 0.0010437159799039364, + "learning_rate": 9.991600000000001e-06, + "loss": 0.0001, + "num_tokens": 1110864.0, + "reward": 6.267642021179199, + "reward_std": 0.3985981345176697, + "rewards/CLIPTextReward/mean": 1.0354081988334656, + "rewards/CLIPTextReward/std": 0.196964293718338, + "rewards/HPSV21TextReward/mean": 0.816436767578125, + "rewards/HPSV21TextReward/std": 0.1313062310218811, + "rewards/MANIQAReward/mean": 1.0315299034118652, + "rewards/MANIQAReward/std": 0.18300341069698334, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1496093273162842, + "rewards/QwenTextReward/std": 0.06275618821382523, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 64 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.017333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.0010261985007673502, + "learning_rate": 9.991466666666667e-06, + "loss": 0.0001, + "num_tokens": 1128240.0, + "reward": 6.297903060913086, + "reward_std": 0.3682844340801239, + "rewards/CLIPTextReward/mean": 1.05167156457901, + "rewards/CLIPTextReward/std": 0.1987253725528717, + "rewards/HPSV21TextReward/mean": 0.8304023742675781, + "rewards/HPSV21TextReward/std": 0.11583104729652405, + "rewards/MANIQAReward/mean": 0.9892240762710571, + "rewards/MANIQAReward/std": 0.1848040372133255, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1500000953674316, + "rewards/QwenTextReward/std": 0.06813853234052658, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 65 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0176, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0009466060437262058, + "learning_rate": 9.991333333333334e-06, + "loss": 0.0001, + "num_tokens": 1145576.0, + "reward": 6.463624000549316, + "reward_std": 0.436023086309433, + "rewards/CLIPTextReward/mean": 1.1064223051071167, + "rewards/CLIPTextReward/std": 0.21644117683172226, + "rewards/HPSV21TextReward/mean": 0.8701066970825195, + "rewards/HPSV21TextReward/std": 0.1395508013665676, + "rewards/MANIQAReward/mean": 1.0152537822723389, + "rewards/MANIQAReward/std": 0.16907808184623718, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1554687023162842, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, + "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, + "step": 66 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.017866666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.0010095324832946062, + "learning_rate": 9.9912e-06, + "loss": 0.0001, + "num_tokens": 1163008.0, + "reward": 6.304980278015137, + "reward_std": 0.4456925094127655, + "rewards/CLIPTextReward/mean": 1.058637797832489, + "rewards/CLIPTextReward/std": 0.19116178900003433, + "rewards/HPSV21TextReward/mean": 0.8249893188476562, + "rewards/HPSV21TextReward/std": 0.07794276624917984, + "rewards/MANIQAReward/mean": 1.0181949138641357, + "rewards/MANIQAReward/std": 0.15045630931854248, + "rewards/QwenFakeDiscrimReward/mean": 0.1640625, + "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, + "rewards/QwenTextReward/mean": 1.140625, + "rewards/QwenTextReward/std": 0.08363043516874313, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 67 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.018133333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.001076919841580093, + "learning_rate": 9.991066666666668e-06, + "loss": 0.0001, + "num_tokens": 1180448.0, + "reward": 6.19140625, + "reward_std": 0.33228224515914917, + "rewards/CLIPTextReward/mean": 1.0292329788208008, + "rewards/CLIPTextReward/std": 0.18924038857221603, + "rewards/HPSV21TextReward/mean": 0.8369541168212891, + "rewards/HPSV21TextReward/std": 0.11581806093454361, + "rewards/MANIQAReward/mean": 0.9867665767669678, + "rewards/MANIQAReward/std": 0.13873277604579926, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.124609351158142, + "rewards/QwenTextReward/std": 0.09692619740962982, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 68 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0184, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0010598800145089626, + "learning_rate": 9.990933333333335e-06, + "loss": 0.0001, + "num_tokens": 1197824.0, + "reward": 6.298628807067871, + "reward_std": 0.4064304828643799, + "rewards/CLIPTextReward/mean": 1.0958648920059204, + "rewards/CLIPTextReward/std": 0.1893087550997734, + "rewards/HPSV21TextReward/mean": 0.846552848815918, + "rewards/HPSV21TextReward/std": 0.09922610223293304, + "rewards/MANIQAReward/mean": 0.9290274977684021, + "rewards/MANIQAReward/std": 0.15360622107982635, + "rewards/QwenFakeDiscrimReward/mean": 0.12890625, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.141015648841858, + "rewards/QwenTextReward/std": 0.07694144546985626, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 69 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.018666666666666668, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.001305924030020833, + "learning_rate": 9.990800000000001e-06, + "loss": 0.0001, + "num_tokens": 1215128.0, + "reward": 6.0216965675354, + "reward_std": 0.3705746829509735, + "rewards/CLIPTextReward/mean": 0.9692765474319458, + "rewards/CLIPTextReward/std": 0.17790019512176514, + "rewards/HPSV21TextReward/mean": 0.8086051940917969, + "rewards/HPSV21TextReward/std": 0.07346846163272858, + "rewards/MANIQAReward/mean": 0.9409325122833252, + "rewards/MANIQAReward/std": 0.156536266207695, + "rewards/QwenFakeDiscrimReward/mean": 0.13671875, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.1695313453674316, + "rewards/QwenTextReward/std": 0.03945784643292427, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 70 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.018933333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0010778771247714758, + "learning_rate": 9.990666666666667e-06, + "loss": 0.0001, + "num_tokens": 1232456.0, + "reward": 6.8039398193359375, + "reward_std": 0.30482178926467896, + "rewards/CLIPTextReward/mean": 1.1879304647445679, + "rewards/CLIPTextReward/std": 0.1860901564359665, + "rewards/HPSV21TextReward/mean": 0.8902063369750977, + "rewards/HPSV21TextReward/std": 0.06591881066560745, + "rewards/MANIQAReward/mean": 1.088681697845459, + "rewards/MANIQAReward/std": 0.15087370574474335, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.156640648841858, + "rewards/QwenTextReward/std": 0.037332240492105484, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 71 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0192, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0010855778818950057, + "learning_rate": 9.990533333333334e-06, + "loss": 0.0001, + "num_tokens": 1249840.0, + "reward": 6.250958442687988, + "reward_std": 0.3621181547641754, + "rewards/CLIPTextReward/mean": 1.0959141254425049, + "rewards/CLIPTextReward/std": 0.1913270279765129, + "rewards/HPSV21TextReward/mean": 0.7908287048339844, + "rewards/HPSV21TextReward/std": 0.0678899884223938, + "rewards/MANIQAReward/mean": 0.9958319664001465, + "rewards/MANIQAReward/std": 0.1409938782453537, + "rewards/QwenFakeDiscrimReward/mean": 0.13671875, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.1417968273162842, + "rewards/QwenTextReward/std": 0.06607697904109955, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 72 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.019466666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.001005525584332645, + "learning_rate": 9.990400000000002e-06, + "loss": 0.0001, + "num_tokens": 1267272.0, + "reward": 6.217660903930664, + "reward_std": 0.4128098487854004, + "rewards/CLIPTextReward/mean": 1.0573053359985352, + "rewards/CLIPTextReward/std": 0.20548022538423538, + "rewards/HPSV21TextReward/mean": 0.8063507080078125, + "rewards/HPSV21TextReward/std": 0.1057361587882042, + "rewards/MANIQAReward/mean": 0.9743335247039795, + "rewards/MANIQAReward/std": 0.15329712629318237, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1566407680511475, + "rewards/QwenTextReward/std": 0.037332236766815186, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 73 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.019733333333333332, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3203125, + "kl": 0.0011021631071344018, + "learning_rate": 9.990266666666666e-06, + "loss": 0.0001, + "num_tokens": 1284600.0, + "reward": 6.18872594833374, + "reward_std": 0.47515836358070374, + "rewards/CLIPTextReward/mean": 1.0537782311439514, + "rewards/CLIPTextReward/std": 0.19655564427375793, + "rewards/HPSV21TextReward/mean": 0.8184757232666016, + "rewards/HPSV21TextReward/std": 0.061357416212558746, + "rewards/MANIQAReward/mean": 0.9172650575637817, + "rewards/MANIQAReward/std": 0.14114581048488617, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1636719703674316, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 74 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.02, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0009673364111222327, + "learning_rate": 9.990133333333335e-06, + "loss": 0.0001, + "num_tokens": 1302080.0, + "reward": 6.431103706359863, + "reward_std": 0.4042501449584961, + "rewards/CLIPTextReward/mean": 1.1340290307998657, + "rewards/CLIPTextReward/std": 0.18132489174604416, + "rewards/HPSV21TextReward/mean": 0.812891960144043, + "rewards/HPSV21TextReward/std": 0.06444458663463593, + "rewards/MANIQAReward/mean": 0.9536682963371277, + "rewards/MANIQAReward/std": 0.18433940410614014, + "rewards/QwenFakeDiscrimReward/mean": 0.1875, + "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, + "rewards/QwenTextReward/mean": 1.157812476158142, + "rewards/QwenTextReward/std": 0.03750002384185791, + "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, + "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, + "step": 75 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.020266666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0010889782570302486, + "learning_rate": 9.990000000000001e-06, + "loss": 0.0001, + "num_tokens": 1319464.0, + "reward": 6.319072246551514, + "reward_std": 0.38241666555404663, + "rewards/CLIPTextReward/mean": 1.1177687048912048, + "rewards/CLIPTextReward/std": 0.20952477306127548, + "rewards/HPSV21TextReward/mean": 0.8423194885253906, + "rewards/HPSV21TextReward/std": 0.11822888255119324, + "rewards/MANIQAReward/mean": 0.8778019547462463, + "rewards/MANIQAReward/std": 0.1526302546262741, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03750002756714821, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 76 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.020533333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.0011135638924315572, + "learning_rate": 9.989866666666667e-06, + "loss": 0.0001, + "num_tokens": 1336768.0, + "reward": 6.1349029541015625, + "reward_std": 0.3507642149925232, + "rewards/CLIPTextReward/mean": 1.0508588552474976, + "rewards/CLIPTextReward/std": 0.19878961890935898, + "rewards/HPSV21TextReward/mean": 0.8079948425292969, + "rewards/HPSV21TextReward/std": 0.06525566428899765, + "rewards/MANIQAReward/mean": 0.985555112361908, + "rewards/MANIQAReward/std": 0.15456393361091614, + "rewards/QwenFakeDiscrimReward/mean": 0.10546875, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.142578125, + "rewards/QwenTextReward/std": 0.032021742314100266, + "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, + "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, + "step": 77 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0208, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0011158613488078117, + "learning_rate": 9.989733333333335e-06, + "loss": 0.0001, + "num_tokens": 1354136.0, + "reward": 6.195856094360352, + "reward_std": 0.4714621305465698, + "rewards/CLIPTextReward/mean": 1.021101474761963, + "rewards/CLIPTextReward/std": 0.18839417397975922, + "rewards/HPSV21TextReward/mean": 0.8526582717895508, + "rewards/HPSV21TextReward/std": 0.13694702461361885, + "rewards/MANIQAReward/mean": 0.9741174578666687, + "rewards/MANIQAReward/std": 0.16313214600086212, + "rewards/QwenFakeDiscrimReward/mean": 0.13671875, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.130468726158142, + "rewards/QwenTextReward/std": 0.08550374954938889, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 78 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.021066666666666668, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.0010789756197482347, + "learning_rate": 9.9896e-06, + "loss": 0.0001, + "num_tokens": 1371456.0, + "reward": 6.297922134399414, + "reward_std": 0.4766380190849304, + "rewards/CLIPTextReward/mean": 1.0652101039886475, + "rewards/CLIPTextReward/std": 0.1847854182124138, + "rewards/HPSV21TextReward/mean": 0.8232393264770508, + "rewards/HPSV21TextReward/std": 0.08456060290336609, + "rewards/MANIQAReward/mean": 0.9893825650215149, + "rewards/MANIQAReward/std": 0.17369645833969116, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.137109398841858, + "rewards/QwenTextReward/std": 0.08261503279209137, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 79 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.021333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.001203256892040372, + "learning_rate": 9.989466666666668e-06, + "loss": 0.0001, + "num_tokens": 1388880.0, + "reward": 6.433945655822754, + "reward_std": 0.40557098388671875, + "rewards/CLIPTextReward/mean": 1.121677041053772, + "rewards/CLIPTextReward/std": 0.21415946632623672, + "rewards/HPSV21TextReward/mean": 0.8016910552978516, + "rewards/HPSV21TextReward/std": 0.07152381539344788, + "rewards/MANIQAReward/mean": 1.029787302017212, + "rewards/MANIQAReward/std": 0.17881350219249725, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 80 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0216, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3203125, + "kl": 0.0010657173115760088, + "learning_rate": 9.989333333333334e-06, + "loss": 0.0001, + "num_tokens": 1406288.0, + "reward": 6.374731063842773, + "reward_std": 0.355613112449646, + "rewards/CLIPTextReward/mean": 1.1012027859687805, + "rewards/CLIPTextReward/std": 0.20229313522577286, + "rewards/HPSV21TextReward/mean": 0.8576145172119141, + "rewards/HPSV21TextReward/std": 0.07686951756477356, + "rewards/MANIQAReward/mean": 0.9442054033279419, + "rewards/MANIQAReward/std": 0.15360897779464722, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1496093273162842, + "rewards/QwenTextReward/std": 0.06275618821382523, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 81 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.021866666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0011162076843902469, + "learning_rate": 9.989200000000001e-06, + "loss": 0.0001, + "num_tokens": 1423504.0, + "reward": 6.265508651733398, + "reward_std": 0.3726772367954254, + "rewards/CLIPTextReward/mean": 1.0828431844711304, + "rewards/CLIPTextReward/std": 0.2197393849492073, + "rewards/HPSV21TextReward/mean": 0.7947602272033691, + "rewards/HPSV21TextReward/std": 0.096592977643013, + "rewards/MANIQAReward/mean": 1.0126456022262573, + "rewards/MANIQAReward/std": 0.17459847033023834, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.130468726158142, + "rewards/QwenTextReward/std": 0.09904994070529938, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 82 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.022133333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.0012128560338169336, + "learning_rate": 9.989066666666667e-06, + "loss": 0.0001, + "num_tokens": 1440856.0, + "reward": 6.381606578826904, + "reward_std": 0.46134722232818604, + "rewards/CLIPTextReward/mean": 1.09823340177536, + "rewards/CLIPTextReward/std": 0.21456897258758545, + "rewards/HPSV21TextReward/mean": 0.8542604446411133, + "rewards/HPSV21TextReward/std": 0.15725306794047356, + "rewards/MANIQAReward/mean": 0.9957593083381653, + "rewards/MANIQAReward/std": 0.210353285074234, + "rewards/QwenFakeDiscrimReward/mean": 0.13671875, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.144921898841858, + "rewards/QwenTextReward/std": 0.03338678926229477, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 83 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0224, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2734375, + "kl": 0.0010785171762108803, + "learning_rate": 9.988933333333334e-06, + "loss": 0.0001, + "num_tokens": 1458104.0, + "reward": 6.2626566886901855, + "reward_std": 0.36470773816108704, + "rewards/CLIPTextReward/mean": 1.0574107766151428, + "rewards/CLIPTextReward/std": 0.18804269284009933, + "rewards/HPSV21TextReward/mean": 0.8381462097167969, + "rewards/HPSV21TextReward/std": 0.10469352453947067, + "rewards/MANIQAReward/mean": 0.9285739064216614, + "rewards/MANIQAReward/std": 0.17206481099128723, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 84 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.02266666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0012498938012868166, + "learning_rate": 9.9888e-06, + "loss": 0.0001, + "num_tokens": 1475472.0, + "reward": 6.434211254119873, + "reward_std": 0.44619888067245483, + "rewards/CLIPTextReward/mean": 1.1103341579437256, + "rewards/CLIPTextReward/std": 0.20621351152658463, + "rewards/HPSV21TextReward/mean": 0.8277511596679688, + "rewards/HPSV21TextReward/std": 0.12262411415576935, + "rewards/MANIQAReward/mean": 1.024056315422058, + "rewards/MANIQAReward/std": 0.1818513125181198, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 85 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.022933333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0011960228439420462, + "learning_rate": 9.988666666666668e-06, + "loss": 0.0001, + "num_tokens": 1492792.0, + "reward": 6.389847755432129, + "reward_std": 0.384074866771698, + "rewards/CLIPTextReward/mean": 1.0775142908096313, + "rewards/CLIPTextReward/std": 0.1948176771402359, + "rewards/HPSV21TextReward/mean": 0.8495254516601562, + "rewards/HPSV21TextReward/std": 0.15474874898791313, + "rewards/MANIQAReward/mean": 0.9799091815948486, + "rewards/MANIQAReward/std": 0.15445655584335327, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1496094465255737, + "rewards/QwenTextReward/std": 0.03549320250749588, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 86 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0232, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0011886307038366795, + "learning_rate": 9.988533333333334e-06, + "loss": 0.0001, + "num_tokens": 1510184.0, + "reward": 6.298969268798828, + "reward_std": 0.4068778455257416, + "rewards/CLIPTextReward/mean": 1.0830622911453247, + "rewards/CLIPTextReward/std": 0.1865195631980896, + "rewards/HPSV21TextReward/mean": 0.8122434616088867, + "rewards/HPSV21TextReward/std": 0.07124301791191101, + "rewards/MANIQAReward/mean": 0.9450761675834656, + "rewards/MANIQAReward/std": 0.1854465901851654, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1687500476837158, + "rewards/QwenTextReward/std": 0.06900657713413239, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 87 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.023466666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.0013361312448978424, + "learning_rate": 9.9884e-06, + "loss": 0.0001, + "num_tokens": 1527536.0, + "reward": 6.399479866027832, + "reward_std": 0.4067446291446686, + "rewards/CLIPTextReward/mean": 1.0961484909057617, + "rewards/CLIPTextReward/std": 0.18726787716150284, + "rewards/HPSV21TextReward/mean": 0.8359432220458984, + "rewards/HPSV21TextReward/std": 0.09672457724809647, + "rewards/MANIQAReward/mean": 0.9806088209152222, + "rewards/MANIQAReward/std": 0.15164178609848022, + "rewards/QwenFakeDiscrimReward/mean": 0.19921875, + "rewards/QwenFakeDiscrimReward/std": 0.10137632489204407, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 88 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.023733333333333332, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.0012429766356945038, + "learning_rate": 9.988266666666667e-06, + "loss": 0.0001, + "num_tokens": 1544928.0, + "reward": 6.399818420410156, + "reward_std": 0.4257749319076538, + "rewards/CLIPTextReward/mean": 1.0604230761528015, + "rewards/CLIPTextReward/std": 0.18725504726171494, + "rewards/HPSV21TextReward/mean": 0.8571786880493164, + "rewards/HPSV21TextReward/std": 0.1451178975403309, + "rewards/MANIQAReward/mean": 1.0099272727966309, + "rewards/MANIQAReward/std": 0.17368283867835999, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.171875, + "rewards/QwenTextReward/std": 0.036596279591321945, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 89 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.024, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.0012510295491665602, + "learning_rate": 9.988133333333334e-06, + "loss": 0.0001, + "num_tokens": 1562248.0, + "reward": 6.323746681213379, + "reward_std": 0.4743574261665344, + "rewards/CLIPTextReward/mean": 1.1159039735794067, + "rewards/CLIPTextReward/std": 0.20765963941812515, + "rewards/HPSV21TextReward/mean": 0.8270845413208008, + "rewards/HPSV21TextReward/std": 0.13083595782518387, + "rewards/MANIQAReward/mean": 0.9799572229385376, + "rewards/MANIQAReward/std": 0.1780419647693634, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1335937976837158, + "rewards/QwenTextReward/std": 0.11736347526311874, + "rewards/QwenWeirdDiscrimReward/mean": 0.17578125, + "rewards/QwenWeirdDiscrimReward/std": 0.1151231899857521, + "step": 90 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.024266666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0012100131716579199, + "learning_rate": 9.988000000000002e-06, + "loss": 0.0001, + "num_tokens": 1579496.0, + "reward": 6.273241996765137, + "reward_std": 0.5074683427810669, + "rewards/CLIPTextReward/mean": 1.0791380405426025, + "rewards/CLIPTextReward/std": 0.21002036333084106, + "rewards/HPSV21TextReward/mean": 0.8155717849731445, + "rewards/HPSV21TextReward/std": 0.08209662139415741, + "rewards/MANIQAReward/mean": 0.9748379588127136, + "rewards/MANIQAReward/std": 0.16974200308322906, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.1457030773162842, + "rewards/QwenTextReward/std": 0.08881437033414841, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 91 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.024533333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.0011587683111429214, + "learning_rate": 9.987866666666668e-06, + "loss": 0.0001, + "num_tokens": 1596816.0, + "reward": 5.989029407501221, + "reward_std": 0.5233784914016724, + "rewards/CLIPTextReward/mean": 1.0351091623306274, + "rewards/CLIPTextReward/std": 0.1978161856532097, + "rewards/HPSV21TextReward/mean": 0.7633829116821289, + "rewards/HPSV21TextReward/std": 0.13681436330080032, + "rewards/MANIQAReward/mean": 0.9248576164245605, + "rewards/MANIQAReward/std": 0.23903001844882965, + "rewards/QwenFakeDiscrimReward/mean": 0.1640625, + "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, + "rewards/QwenTextReward/mean": 1.100000023841858, + "rewards/QwenTextReward/std": 0.15634721517562866, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 92 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0248, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0014097951352596283, + "learning_rate": 9.987733333333334e-06, + "loss": 0.0001, + "num_tokens": 1614064.0, + "reward": 6.371288299560547, + "reward_std": 0.33220386505126953, + "rewards/CLIPTextReward/mean": 1.060925841331482, + "rewards/CLIPTextReward/std": 0.17794457077980042, + "rewards/HPSV21TextReward/mean": 0.8472576141357422, + "rewards/HPSV21TextReward/std": 0.07239008694887161, + "rewards/MANIQAReward/mean": 0.9830461740493774, + "rewards/MANIQAReward/std": 0.1399579495191574, + "rewards/QwenFakeDiscrimReward/mean": 0.1953125, + "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03750002756714821, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 93 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.025066666666666668, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.0013897167518734932, + "learning_rate": 9.9876e-06, + "loss": 0.0001, + "num_tokens": 1631392.0, + "reward": 6.293347358703613, + "reward_std": 0.402121365070343, + "rewards/CLIPTextReward/mean": 1.0947368144989014, + "rewards/CLIPTextReward/std": 0.22335071116685867, + "rewards/HPSV21TextReward/mean": 0.8451414108276367, + "rewards/HPSV21TextReward/std": 0.09467275440692902, + "rewards/MANIQAReward/mean": 0.9737468361854553, + "rewards/MANIQAReward/std": 0.15930481255054474, + "rewards/QwenFakeDiscrimReward/mean": 0.12890625, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.1156249046325684, + "rewards/QwenTextReward/std": 0.11032959818840027, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 94 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.025333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.001365752425044775, + "learning_rate": 9.987466666666667e-06, + "loss": 0.0001, + "num_tokens": 1648864.0, + "reward": 6.3488569259643555, + "reward_std": 0.37489140033721924, + "rewards/CLIPTextReward/mean": 1.1043108701705933, + "rewards/CLIPTextReward/std": 0.20488134771585464, + "rewards/HPSV21TextReward/mean": 0.7973947525024414, + "rewards/HPSV21TextReward/std": 0.057390403002500534, + "rewards/MANIQAReward/mean": 1.0102896690368652, + "rewards/MANIQAReward/std": 0.12302688509225845, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 95 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0256, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0014814387541264296, + "learning_rate": 9.987333333333335e-06, + "loss": 0.0001, + "num_tokens": 1666120.0, + "reward": 6.278581619262695, + "reward_std": 0.3662630319595337, + "rewards/CLIPTextReward/mean": 1.075215220451355, + "rewards/CLIPTextReward/std": 0.20643755793571472, + "rewards/HPSV21TextReward/mean": 0.8416833877563477, + "rewards/HPSV21TextReward/std": 0.06616079062223434, + "rewards/MANIQAReward/mean": 0.9537688493728638, + "rewards/MANIQAReward/std": 0.1429898738861084, + "rewards/QwenFakeDiscrimReward/mean": 0.12890625, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 96 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.025866666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.001286173122934997, + "learning_rate": 9.987200000000002e-06, + "loss": 0.0001, + "num_tokens": 1683472.0, + "reward": 6.097433567047119, + "reward_std": 0.4108942747116089, + "rewards/CLIPTextReward/mean": 1.0381823182106018, + "rewards/CLIPTextReward/std": 0.1961667463183403, + "rewards/HPSV21TextReward/mean": 0.8087921142578125, + "rewards/HPSV21TextReward/std": 0.11398311704397202, + "rewards/MANIQAReward/mean": 0.9116876125335693, + "rewards/MANIQAReward/std": 0.1728503406047821, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.1441407203674316, + "rewards/QwenTextReward/std": 0.06451010704040527, + "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, + "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, + "step": 97 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.026133333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.390625, + "kl": 0.0014894043561071157, + "learning_rate": 9.987066666666668e-06, + "loss": 0.0001, + "num_tokens": 1700776.0, + "reward": 6.155697345733643, + "reward_std": 0.43312111496925354, + "rewards/CLIPTextReward/mean": 1.0117595791816711, + "rewards/CLIPTextReward/std": 0.17692303657531738, + "rewards/HPSV21TextReward/mean": 0.8317708969116211, + "rewards/HPSV21TextReward/std": 0.10141152888536453, + "rewards/MANIQAReward/mean": 0.9522302150726318, + "rewards/MANIQAReward/std": 0.15004120767116547, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1453125476837158, + "rewards/QwenTextReward/std": 0.0670931488275528, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 98 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0264, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.390625, + "kl": 0.0014492514310404658, + "learning_rate": 9.986933333333334e-06, + "loss": 0.0001, + "num_tokens": 1718136.0, + "reward": 6.55409049987793, + "reward_std": 0.4162108302116394, + "rewards/CLIPTextReward/mean": 1.1229727268218994, + "rewards/CLIPTextReward/std": 0.18806544691324234, + "rewards/HPSV21TextReward/mean": 0.8469762802124023, + "rewards/HPSV21TextReward/std": 0.07917948812246323, + "rewards/MANIQAReward/mean": 1.0454427003860474, + "rewards/MANIQAReward/std": 0.1772715151309967, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1742188930511475, + "rewards/QwenTextReward/std": 0.03590354323387146, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 99 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.02666666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0017493412597104907, + "learning_rate": 9.9868e-06, + "loss": 0.0002, + "num_tokens": 1735440.0, + "reward": 6.363776206970215, + "reward_std": 0.3973403573036194, + "rewards/CLIPTextReward/mean": 1.0763663053512573, + "rewards/CLIPTextReward/std": 0.19003498554229736, + "rewards/HPSV21TextReward/mean": 0.8241357803344727, + "rewards/HPSV21TextReward/std": 0.07310770452022552, + "rewards/MANIQAReward/mean": 1.022928237915039, + "rewards/MANIQAReward/std": 0.21806401014328003, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.1453124284744263, + "rewards/QwenTextReward/std": 0.0670931413769722, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 100 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.026933333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0013862024061381817, + "learning_rate": 9.986666666666667e-06, + "loss": 0.0001, + "num_tokens": 1752792.0, + "reward": 6.372419834136963, + "reward_std": 0.43285655975341797, + "rewards/CLIPTextReward/mean": 1.1106175184249878, + "rewards/CLIPTextReward/std": 0.19487349689006805, + "rewards/HPSV21TextReward/mean": 0.8137311935424805, + "rewards/HPSV21TextReward/std": 0.08555622398853302, + "rewards/MANIQAReward/mean": 0.944034993648529, + "rewards/MANIQAReward/std": 0.1661282777786255, + "rewards/QwenFakeDiscrimReward/mean": 0.203125, + "rewards/QwenFakeDiscrimReward/std": 0.09834947437047958, + "rewards/QwenTextReward/mean": 1.1539063453674316, + "rewards/QwenTextReward/std": 0.04633788391947746, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0272, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.390625, + "kl": 0.0015219071647152305, + "learning_rate": 9.986533333333333e-06, + "loss": 0.0002, + "num_tokens": 1770184.0, + "reward": 6.147955894470215, + "reward_std": 0.3923138380050659, + "rewards/CLIPTextReward/mean": 1.0357251167297363, + "rewards/CLIPTextReward/std": 0.18429216742515564, + "rewards/HPSV21TextReward/mean": 0.8081808090209961, + "rewards/HPSV21TextReward/std": 0.09258602559566498, + "rewards/MANIQAReward/mean": 1.004284381866455, + "rewards/MANIQAReward/std": 0.16934402287006378, + "rewards/QwenFakeDiscrimReward/mean": 0.12109375, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.1316406726837158, + "rewards/QwenTextReward/std": 0.11164733022451401, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.027466666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0014235489070415497, + "learning_rate": 9.9864e-06, + "loss": 0.0001, + "num_tokens": 1787464.0, + "reward": 6.333028793334961, + "reward_std": 0.3288081884384155, + "rewards/CLIPTextReward/mean": 1.0399454832077026, + "rewards/CLIPTextReward/std": 0.20984552800655365, + "rewards/HPSV21TextReward/mean": 0.8435735702514648, + "rewards/HPSV21TextReward/std": 0.14140213280916214, + "rewards/MANIQAReward/mean": 1.0234129428863525, + "rewards/MANIQAReward/std": 0.15938599407672882, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1441407203674316, + "rewards/QwenTextReward/std": 0.06677703559398651, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.027733333333333332, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0015638014301657677, + "learning_rate": 9.986266666666668e-06, + "loss": 0.0002, + "num_tokens": 1804784.0, + "reward": 6.247958183288574, + "reward_std": 0.39279264211654663, + "rewards/CLIPTextReward/mean": 1.0843626856803894, + "rewards/CLIPTextReward/std": 0.19982705265283585, + "rewards/HPSV21TextReward/mean": 0.7990350723266602, + "rewards/HPSV21TextReward/std": 0.07866711914539337, + "rewards/MANIQAReward/mean": 0.9483500719070435, + "rewards/MANIQAReward/std": 0.1665145456790924, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1695313453674316, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.028, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.0015782412374392152, + "learning_rate": 9.986133333333334e-06, + "loss": 0.0002, + "num_tokens": 1822224.0, + "reward": 6.490713596343994, + "reward_std": 0.43102598190307617, + "rewards/CLIPTextReward/mean": 1.1477264761924744, + "rewards/CLIPTextReward/std": 0.2057536318898201, + "rewards/HPSV21TextReward/mean": 0.8155384063720703, + "rewards/HPSV21TextReward/std": 0.05710719898343086, + "rewards/MANIQAReward/mean": 1.0063711404800415, + "rewards/MANIQAReward/std": 0.17023935914039612, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.1593749523162842, + "rewards/QwenTextReward/std": 0.06922188401222229, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.028266666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.0014377952320501208, + "learning_rate": 9.986e-06, + "loss": 0.0001, + "num_tokens": 1839608.0, + "reward": 6.183026313781738, + "reward_std": 0.32712695002555847, + "rewards/CLIPTextReward/mean": 1.0003265738487244, + "rewards/CLIPTextReward/std": 0.15935227647423744, + "rewards/HPSV21TextReward/mean": 0.8352365493774414, + "rewards/HPSV21TextReward/std": 0.0641993060708046, + "rewards/MANIQAReward/mean": 0.9435409307479858, + "rewards/MANIQAReward/std": 0.15548750758171082, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.166015625, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.028533333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.390625, + "kl": 0.0015673225279897451, + "learning_rate": 9.985866666666667e-06, + "loss": 0.0002, + "num_tokens": 1857104.0, + "reward": 6.220345973968506, + "reward_std": 0.3758386969566345, + "rewards/CLIPTextReward/mean": 1.0252848863601685, + "rewards/CLIPTextReward/std": 0.1825021132826805, + "rewards/HPSV21TextReward/mean": 0.8366527557373047, + "rewards/HPSV21TextReward/std": 0.07109877467155457, + "rewards/MANIQAReward/mean": 0.972642719745636, + "rewards/MANIQAReward/std": 0.19886445999145508, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.156640648841858, + "rewards/QwenTextReward/std": 0.037332236766815186, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0288, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3828125, + "kl": 0.0015098307048901916, + "learning_rate": 9.985733333333333e-06, + "loss": 0.0002, + "num_tokens": 1874456.0, + "reward": 6.419452667236328, + "reward_std": 0.4157279133796692, + "rewards/CLIPTextReward/mean": 1.1197414994239807, + "rewards/CLIPTextReward/std": 0.20846442133188248, + "rewards/HPSV21TextReward/mean": 0.8084573745727539, + "rewards/HPSV21TextReward/std": 0.08079753071069717, + "rewards/MANIQAReward/mean": 1.0208675861358643, + "rewards/MANIQAReward/std": 0.18414068222045898, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1554687023162842, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.029066666666666668, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.0014714575372636318, + "learning_rate": 9.985600000000001e-06, + "loss": 0.0001, + "num_tokens": 1891752.0, + "reward": 6.302655220031738, + "reward_std": 0.4730131924152374, + "rewards/CLIPTextReward/mean": 1.0720114707946777, + "rewards/CLIPTextReward/std": 0.2158217504620552, + "rewards/HPSV21TextReward/mean": 0.8254232406616211, + "rewards/HPSV21TextReward/std": 0.0781259685754776, + "rewards/MANIQAReward/mean": 0.9687229990959167, + "rewards/MANIQAReward/std": 0.1913435310125351, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 109 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.029333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.21875, + "kl": 0.0014287796802818775, + "learning_rate": 9.985466666666668e-06, + "loss": 0.0001, + "num_tokens": 1909112.0, + "reward": 6.507727146148682, + "reward_std": 0.40447428822517395, + "rewards/CLIPTextReward/mean": 1.1296402215957642, + "rewards/CLIPTextReward/std": 0.199630968272686, + "rewards/HPSV21TextReward/mean": 0.830596923828125, + "rewards/HPSV21TextReward/std": 0.05249226465821266, + "rewards/MANIQAReward/mean": 0.9966280460357666, + "rewards/MANIQAReward/std": 0.14694969356060028, + "rewards/QwenFakeDiscrimReward/mean": 0.20703125, + "rewards/QwenFakeDiscrimReward/std": 0.09506355226039886, + "rewards/QwenTextReward/mean": 1.1531250476837158, + "rewards/QwenTextReward/std": 0.036596279591321945, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 110 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0296, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.0016530327266082168, + "learning_rate": 9.985333333333334e-06, + "loss": 0.0002, + "num_tokens": 1926504.0, + "reward": 6.363095760345459, + "reward_std": 0.3255675733089447, + "rewards/CLIPTextReward/mean": 1.056698203086853, + "rewards/CLIPTextReward/std": 0.24591663479804993, + "rewards/HPSV21TextReward/mean": 0.8341121673583984, + "rewards/HPSV21TextReward/std": 0.13344483822584152, + "rewards/MANIQAReward/mean": 1.0506157875061035, + "rewards/MANIQAReward/std": 0.15129290521144867, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1519532203674316, + "rewards/QwenTextReward/std": 0.03627080097794533, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.029866666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2734375, + "kl": 0.0014856918714940548, + "learning_rate": 9.9852e-06, + "loss": 0.0001, + "num_tokens": 1943824.0, + "reward": 6.13232421875, + "reward_std": 0.3255869746208191, + "rewards/CLIPTextReward/mean": 0.9885252714157104, + "rewards/CLIPTextReward/std": 0.1493731364607811, + "rewards/HPSV21TextReward/mean": 0.795954704284668, + "rewards/HPSV21TextReward/std": 0.08864696323871613, + "rewards/MANIQAReward/mean": 1.0371923446655273, + "rewards/MANIQAReward/std": 0.1781325787305832, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.030133333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.0016064159572124481, + "learning_rate": 9.985066666666667e-06, + "loss": 0.0002, + "num_tokens": 1961264.0, + "reward": 6.490161895751953, + "reward_std": 0.4232161045074463, + "rewards/CLIPTextReward/mean": 1.1830998063087463, + "rewards/CLIPTextReward/std": 0.19798733294010162, + "rewards/HPSV21TextReward/mean": 0.8457889556884766, + "rewards/HPSV21TextReward/std": 0.06092287600040436, + "rewards/MANIQAReward/mean": 0.9694934487342834, + "rewards/MANIQAReward/std": 0.16687992215156555, + "rewards/QwenFakeDiscrimReward/mean": 0.13671875, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.154296875, + "rewards/QwenTextReward/std": 0.03688107803463936, + "rewards/QwenWeirdDiscrimReward/mean": 0.171875, + "rewards/QwenWeirdDiscrimReward/std": 0.116794154047966, + "step": 113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0304, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0017705290811136365, + "learning_rate": 9.984933333333335e-06, + "loss": 0.0002, + "num_tokens": 1978768.0, + "reward": 6.421212196350098, + "reward_std": 0.4210098087787628, + "rewards/CLIPTextReward/mean": 1.1120213866233826, + "rewards/CLIPTextReward/std": 0.189268559217453, + "rewards/HPSV21TextReward/mean": 0.8761920928955078, + "rewards/HPSV21TextReward/std": 0.07564789801836014, + "rewards/MANIQAReward/mean": 0.955332338809967, + "rewards/MANIQAReward/std": 0.20470233261585236, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1339843273162842, + "rewards/QwenTextReward/std": 0.08572373539209366, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.030666666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3203125, + "kl": 0.0017124887090176344, + "learning_rate": 9.984800000000001e-06, + "loss": 0.0002, + "num_tokens": 1996080.0, + "reward": 6.347210884094238, + "reward_std": 0.3971519470214844, + "rewards/CLIPTextReward/mean": 1.0920844078063965, + "rewards/CLIPTextReward/std": 0.18359429761767387, + "rewards/HPSV21TextReward/mean": 0.8116426467895508, + "rewards/HPSV21TextReward/std": 0.08479522168636322, + "rewards/MANIQAReward/mean": 0.9760847687721252, + "rewards/MANIQAReward/std": 0.1531572788953781, + "rewards/QwenFakeDiscrimReward/mean": 0.1953125, + "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, + "rewards/QwenTextReward/mean": 1.1496094465255737, + "rewards/QwenTextReward/std": 0.03792543709278107, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.030933333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.001727626658976078, + "learning_rate": 9.984666666666668e-06, + "loss": 0.0002, + "num_tokens": 2013344.0, + "reward": 6.798962593078613, + "reward_std": 0.4133073687553406, + "rewards/CLIPTextReward/mean": 1.1725159883499146, + "rewards/CLIPTextReward/std": 0.17704249918460846, + "rewards/HPSV21TextReward/mean": 0.8897771835327148, + "rewards/HPSV21TextReward/std": 0.15622979402542114, + "rewards/MANIQAReward/mean": 1.103672742843628, + "rewards/MANIQAReward/std": 0.16740120947360992, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.156640648841858, + "rewards/QwenTextReward/std": 0.037332240492105484, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0312, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.001998618245124817, + "learning_rate": 9.984533333333334e-06, + "loss": 0.0002, + "num_tokens": 2030592.0, + "reward": 6.627265930175781, + "reward_std": 0.36036553978919983, + "rewards/CLIPTextReward/mean": 1.1490726470947266, + "rewards/CLIPTextReward/std": 0.19648005813360214, + "rewards/HPSV21TextReward/mean": 0.8739986419677734, + "rewards/HPSV21TextReward/std": 0.06020794063806534, + "rewards/MANIQAReward/mean": 0.9924516677856445, + "rewards/MANIQAReward/std": 0.18218012154102325, + "rewards/QwenFakeDiscrimReward/mean": 0.1953125, + "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 117 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.031466666666666664, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.71875, + "kl": 0.002251950791105628, + "learning_rate": 9.9844e-06, + "loss": 0.0002, + "num_tokens": 2047896.0, + "reward": 6.450213432312012, + "reward_std": 0.3416060209274292, + "rewards/CLIPTextReward/mean": 1.0652124285697937, + "rewards/CLIPTextReward/std": 0.19462762027978897, + "rewards/HPSV21TextReward/mean": 0.8843517303466797, + "rewards/HPSV21TextReward/std": 0.08924156427383423, + "rewards/MANIQAReward/mean": 1.0217883586883545, + "rewards/MANIQAReward/std": 0.16342057287693024, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.154296875, + "rewards/QwenTextReward/std": 0.03688107803463936, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.031733333333333336, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.0017585426103323698, + "learning_rate": 9.984266666666667e-06, + "loss": 0.0002, + "num_tokens": 2065256.0, + "reward": 6.567963600158691, + "reward_std": 0.4625498354434967, + "rewards/CLIPTextReward/mean": 1.1237746477127075, + "rewards/CLIPTextReward/std": 0.22952871769666672, + "rewards/HPSV21TextReward/mean": 0.8930902481079102, + "rewards/HPSV21TextReward/std": 0.1436346434056759, + "rewards/MANIQAReward/mean": 1.0518121719360352, + "rewards/MANIQAReward/std": 0.17778155207633972, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.138671875, + "rewards/QwenTextReward/std": 0.08368139714002609, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.032, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.001958254724740982, + "learning_rate": 9.984133333333335e-06, + "loss": 0.0002, + "num_tokens": 2082568.0, + "reward": 6.361819267272949, + "reward_std": 0.39701226353645325, + "rewards/CLIPTextReward/mean": 1.0916123390197754, + "rewards/CLIPTextReward/std": 0.2180275395512581, + "rewards/HPSV21TextReward/mean": 0.8297672271728516, + "rewards/HPSV21TextReward/std": 0.08377574384212494, + "rewards/MANIQAReward/mean": 0.9979667663574219, + "rewards/MANIQAReward/std": 0.1789645552635193, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03980984166264534, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 120 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.032266666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0017855416517704725, + "learning_rate": 9.984e-06, + "loss": 0.0002, + "num_tokens": 2099888.0, + "reward": 6.2890801429748535, + "reward_std": 0.41400352120399475, + "rewards/CLIPTextReward/mean": 1.067662239074707, + "rewards/CLIPTextReward/std": 0.2106454148888588, + "rewards/HPSV21TextReward/mean": 0.8307352066040039, + "rewards/HPSV21TextReward/std": 0.0845264345407486, + "rewards/MANIQAReward/mean": 1.0333006381988525, + "rewards/MANIQAReward/std": 0.18434347212314606, + "rewards/QwenFakeDiscrimReward/mean": 0.12890625, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.123046875, + "rewards/QwenTextReward/std": 0.1576303243637085, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03253333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.0017559831030666828, + "learning_rate": 9.983866666666668e-06, + "loss": 0.0002, + "num_tokens": 2117280.0, + "reward": 6.306989669799805, + "reward_std": 0.4608774781227112, + "rewards/CLIPTextReward/mean": 1.083207905292511, + "rewards/CLIPTextReward/std": 0.20290355384349823, + "rewards/HPSV21TextReward/mean": 0.8382225036621094, + "rewards/HPSV21TextReward/std": 0.10574927181005478, + "rewards/MANIQAReward/mean": 0.9082695245742798, + "rewards/MANIQAReward/std": 0.14249961078166962, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1613280773162842, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0328, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.265625, + "kl": 0.0017224764451384544, + "learning_rate": 9.983733333333334e-06, + "loss": 0.0002, + "num_tokens": 2134640.0, + "reward": 6.381166458129883, + "reward_std": 0.34280797839164734, + "rewards/CLIPTextReward/mean": 1.1224886775016785, + "rewards/CLIPTextReward/std": 0.18830392882227898, + "rewards/HPSV21TextReward/mean": 0.8438854217529297, + "rewards/HPSV21TextReward/std": 0.14586665108799934, + "rewards/MANIQAReward/mean": 0.9597461819648743, + "rewards/MANIQAReward/std": 0.222927525639534, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.160546898841858, + "rewards/QwenTextReward/std": 0.07179734110832214, + "rewards/QwenWeirdDiscrimReward/mean": 0.16796875, + "rewards/QwenWeirdDiscrimReward/std": 0.1183105930685997, + "step": 123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03306666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.001943190349265933, + "learning_rate": 9.9836e-06, + "loss": 0.0002, + "num_tokens": 2151912.0, + "reward": 6.494083881378174, + "reward_std": 0.41056764125823975, + "rewards/CLIPTextReward/mean": 1.1126810908317566, + "rewards/CLIPTextReward/std": 0.22240617871284485, + "rewards/HPSV21TextReward/mean": 0.8803911209106445, + "rewards/HPSV21TextReward/std": 0.1202019639313221, + "rewards/MANIQAReward/mean": 0.9934860467910767, + "rewards/MANIQAReward/std": 0.1543145328760147, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, + "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, + "step": 124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.001764823216944933, + "learning_rate": 9.983466666666669e-06, + "loss": 0.0002, + "num_tokens": 2169232.0, + "reward": 6.223010063171387, + "reward_std": 0.4870622158050537, + "rewards/CLIPTextReward/mean": 1.0433948636054993, + "rewards/CLIPTextReward/std": 0.19244755059480667, + "rewards/HPSV21TextReward/mean": 0.8162832260131836, + "rewards/HPSV21TextReward/std": 0.13273374736309052, + "rewards/MANIQAReward/mean": 0.9817787408828735, + "rewards/MANIQAReward/std": 0.19965632259845734, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.1351561546325684, + "rewards/QwenTextReward/std": 0.08714142441749573, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0336, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3203125, + "kl": 0.0018342820694670081, + "learning_rate": 9.983333333333333e-06, + "loss": 0.0002, + "num_tokens": 2186584.0, + "reward": 6.176717281341553, + "reward_std": 0.29012149572372437, + "rewards/CLIPTextReward/mean": 1.020053744316101, + "rewards/CLIPTextReward/std": 0.17557596415281296, + "rewards/HPSV21TextReward/mean": 0.8094825744628906, + "rewards/HPSV21TextReward/std": 0.05560995638370514, + "rewards/MANIQAReward/mean": 1.0028008222579956, + "rewards/MANIQAReward/std": 0.20480003952980042, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1671874523162842, + "rewards/QwenTextReward/std": 0.03980984166264534, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03386666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.002029046881943941, + "learning_rate": 9.983200000000001e-06, + "loss": 0.0002, + "num_tokens": 2203984.0, + "reward": 6.353525161743164, + "reward_std": 0.39747315645217896, + "rewards/CLIPTextReward/mean": 1.0958306789398193, + "rewards/CLIPTextReward/std": 0.21575646102428436, + "rewards/HPSV21TextReward/mean": 0.8271303176879883, + "rewards/HPSV21TextReward/std": 0.07321495562791824, + "rewards/MANIQAReward/mean": 0.9904155135154724, + "rewards/MANIQAReward/std": 0.17507304251194, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.1617188453674316, + "rewards/QwenTextReward/std": 0.042717430740594864, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.034133333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0018549091182649136, + "learning_rate": 9.983066666666668e-06, + "loss": 0.0002, + "num_tokens": 2221312.0, + "reward": 6.532737731933594, + "reward_std": 0.3147696554660797, + "rewards/CLIPTextReward/mean": 1.1676284074783325, + "rewards/CLIPTextReward/std": 0.20117784291505814, + "rewards/HPSV21TextReward/mean": 0.8389949798583984, + "rewards/HPSV21TextReward/std": 0.10873442888259888, + "rewards/MANIQAReward/mean": 0.9530848860740662, + "rewards/MANIQAReward/std": 0.1483864039182663, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.06404346227645874, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0344, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.002199455164372921, + "learning_rate": 9.982933333333334e-06, + "loss": 0.0002, + "num_tokens": 2238576.0, + "reward": 6.328161239624023, + "reward_std": 0.32629311084747314, + "rewards/CLIPTextReward/mean": 1.0435682535171509, + "rewards/CLIPTextReward/std": 0.20102593302726746, + "rewards/HPSV21TextReward/mean": 0.8430194854736328, + "rewards/HPSV21TextReward/std": 0.11192534118890762, + "rewards/MANIQAReward/mean": 0.9631885290145874, + "rewards/MANIQAReward/std": 0.18930527567863464, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.166015625, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, + "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, + "step": 129 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.034666666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0019883974455296993, + "learning_rate": 9.9828e-06, + "loss": 0.0002, + "num_tokens": 2255872.0, + "reward": 6.34254264831543, + "reward_std": 0.4286326467990875, + "rewards/CLIPTextReward/mean": 1.0906871557235718, + "rewards/CLIPTextReward/std": 0.19571883976459503, + "rewards/HPSV21TextReward/mean": 0.8333959579467773, + "rewards/HPSV21TextReward/std": 0.058883652091026306, + "rewards/MANIQAReward/mean": 0.9580481052398682, + "rewards/MANIQAReward/std": 0.14943087100982666, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.1496093273162842, + "rewards/QwenTextReward/std": 0.03792543709278107, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 130 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03493333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0019190080929547548, + "learning_rate": 9.982666666666667e-06, + "loss": 0.0002, + "num_tokens": 2273336.0, + "reward": 6.276549339294434, + "reward_std": 0.3504009246826172, + "rewards/CLIPTextReward/mean": 1.0854588150978088, + "rewards/CLIPTextReward/std": 0.20299603044986725, + "rewards/HPSV21TextReward/mean": 0.8067693710327148, + "rewards/HPSV21TextReward/std": 0.08084672689437866, + "rewards/MANIQAReward/mean": 0.9799835085868835, + "rewards/MANIQAReward/std": 0.20051877200603485, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1449217796325684, + "rewards/QwenTextReward/std": 0.03836432099342346, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0352, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3984375, + "kl": 0.0020587556064128876, + "learning_rate": 9.982533333333335e-06, + "loss": 0.0002, + "num_tokens": 2290672.0, + "reward": 6.436410903930664, + "reward_std": 0.41164326667785645, + "rewards/CLIPTextReward/mean": 1.102868378162384, + "rewards/CLIPTextReward/std": 0.1942010223865509, + "rewards/HPSV21TextReward/mean": 0.8459510803222656, + "rewards/HPSV21TextReward/std": 0.0887582078576088, + "rewards/MANIQAReward/mean": 1.031740665435791, + "rewards/MANIQAReward/std": 0.14581337571144104, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.1554687023162842, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03546666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.002030662726610899, + "learning_rate": 9.982400000000001e-06, + "loss": 0.0002, + "num_tokens": 2307976.0, + "reward": 6.364256858825684, + "reward_std": 0.4065946042537689, + "rewards/CLIPTextReward/mean": 1.0987563729286194, + "rewards/CLIPTextReward/std": 0.18764332681894302, + "rewards/HPSV21TextReward/mean": 0.8777074813842773, + "rewards/HPSV21TextReward/std": 0.13593661412596703, + "rewards/MANIQAReward/mean": 0.8976576328277588, + "rewards/MANIQAReward/std": 0.15904058516025543, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.166015625, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.171875, + "rewards/QwenWeirdDiscrimReward/std": 0.116794154047966, + "step": 133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03573333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.890625, + "kl": 0.0023946776054799557, + "learning_rate": 9.982266666666668e-06, + "loss": 0.0002, + "num_tokens": 2325456.0, + "reward": 6.729565143585205, + "reward_std": 0.3610880970954895, + "rewards/CLIPTextReward/mean": 1.1710917353630066, + "rewards/CLIPTextReward/std": 0.19065489619970322, + "rewards/HPSV21TextReward/mean": 0.8526315689086914, + "rewards/HPSV21TextReward/std": 0.0684349313378334, + "rewards/MANIQAReward/mean": 1.1067278385162354, + "rewards/MANIQAReward/std": 0.13912783563137054, + "rewards/QwenFakeDiscrimReward/mean": 0.1875, + "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, + "rewards/QwenTextReward/mean": 1.1613280773162842, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.036, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.71875, + "kl": 0.0023967409506440163, + "learning_rate": 9.982133333333334e-06, + "loss": 0.0002, + "num_tokens": 2342752.0, + "reward": 6.26861572265625, + "reward_std": 0.43260377645492554, + "rewards/CLIPTextReward/mean": 1.0653359293937683, + "rewards/CLIPTextReward/std": 0.2139836996793747, + "rewards/HPSV21TextReward/mean": 0.8255853652954102, + "rewards/HPSV21TextReward/std": 0.07783577591180801, + "rewards/MANIQAReward/mean": 0.9984918236732483, + "rewards/MANIQAReward/std": 0.14973679184913635, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.125, + "rewards/QwenTextReward/std": 0.14295633137226105, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03626666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3828125, + "kl": 0.002043600659817457, + "learning_rate": 9.982e-06, + "loss": 0.0002, + "num_tokens": 2360144.0, + "reward": 6.444212436676025, + "reward_std": 0.3863571882247925, + "rewards/CLIPTextReward/mean": 1.0651435256004333, + "rewards/CLIPTextReward/std": 0.1987275332212448, + "rewards/HPSV21TextReward/mean": 0.8277597427368164, + "rewards/HPSV21TextReward/std": 0.0786108672618866, + "rewards/MANIQAReward/mean": 1.0498123168945312, + "rewards/MANIQAReward/std": 0.1474582999944687, + "rewards/QwenFakeDiscrimReward/mean": 0.20703125, + "rewards/QwenFakeDiscrimReward/std": 0.09506355226039886, + "rewards/QwenTextReward/mean": 1.1671874523162842, + "rewards/QwenTextReward/std": 0.03750002756714821, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.036533333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.004206032492220402, + "learning_rate": 9.981866666666667e-06, + "loss": 0.0004, + "num_tokens": 2377384.0, + "reward": 6.348177909851074, + "reward_std": 0.47931671142578125, + "rewards/CLIPTextReward/mean": 1.1119232177734375, + "rewards/CLIPTextReward/std": 0.19578742235898972, + "rewards/HPSV21TextReward/mean": 0.8302211761474609, + "rewards/HPSV21TextReward/std": 0.11700878292322159, + "rewards/MANIQAReward/mean": 0.9732642769813538, + "rewards/MANIQAReward/std": 0.1702820360660553, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.1429686546325684, + "rewards/QwenTextReward/std": 0.06643842905759811, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 137 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0368, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0020041209645569324, + "learning_rate": 9.981733333333335e-06, + "loss": 0.0002, + "num_tokens": 2394704.0, + "reward": 6.318357467651367, + "reward_std": 0.4292410612106323, + "rewards/CLIPTextReward/mean": 1.1253670454025269, + "rewards/CLIPTextReward/std": 0.20304681360721588, + "rewards/HPSV21TextReward/mean": 0.8212528228759766, + "rewards/HPSV21TextReward/std": 0.14200227707624435, + "rewards/MANIQAReward/mean": 0.9727740287780762, + "rewards/MANIQAReward/std": 0.22543597221374512, + "rewards/QwenFakeDiscrimReward/mean": 0.12109375, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.1281249523162842, + "rewards/QwenTextReward/std": 0.08351172506809235, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.037066666666666664, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.002243955619633198, + "learning_rate": 9.981600000000001e-06, + "loss": 0.0002, + "num_tokens": 2412072.0, + "reward": 6.313478469848633, + "reward_std": 0.4881005883216858, + "rewards/CLIPTextReward/mean": 1.1092491745948792, + "rewards/CLIPTextReward/std": 0.1954825296998024, + "rewards/HPSV21TextReward/mean": 0.8195505142211914, + "rewards/HPSV21TextReward/std": 0.13129733502864838, + "rewards/MANIQAReward/mean": 1.0058796405792236, + "rewards/MANIQAReward/std": 0.14454947412014008, + "rewards/QwenFakeDiscrimReward/mean": 0.10546875, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.1453124284744263, + "rewards/QwenTextReward/std": 0.07661943882703781, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.037333333333333336, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.296875, + "kl": 0.0019306773319840431, + "learning_rate": 9.981466666666668e-06, + "loss": 0.0002, + "num_tokens": 2429280.0, + "reward": 6.358804702758789, + "reward_std": 0.42756497859954834, + "rewards/CLIPTextReward/mean": 1.078059196472168, + "rewards/CLIPTextReward/std": 0.1667657159268856, + "rewards/HPSV21TextReward/mean": 0.8231105804443359, + "rewards/HPSV21TextReward/std": 0.07944857329130173, + "rewards/MANIQAReward/mean": 1.0154496431350708, + "rewards/MANIQAReward/std": 0.167838916182518, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.166015625, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 140 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0376, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0023419142235070467, + "learning_rate": 9.981333333333334e-06, + "loss": 0.0002, + "num_tokens": 2446632.0, + "reward": 6.513288497924805, + "reward_std": 0.3195926547050476, + "rewards/CLIPTextReward/mean": 1.1330569982528687, + "rewards/CLIPTextReward/std": 0.21059855818748474, + "rewards/HPSV21TextReward/mean": 0.8392133712768555, + "rewards/HPSV21TextReward/std": 0.07069305330514908, + "rewards/MANIQAReward/mean": 1.0140602588653564, + "rewards/MANIQAReward/std": 0.15958507359027863, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.03772257640957832, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 141 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.037866666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.002120975172147155, + "learning_rate": 9.9812e-06, + "loss": 0.0002, + "num_tokens": 2463936.0, + "reward": 6.509091377258301, + "reward_std": 0.44019967317581177, + "rewards/CLIPTextReward/mean": 1.1297532320022583, + "rewards/CLIPTextReward/std": 0.22469547390937805, + "rewards/HPSV21TextReward/mean": 0.8484363555908203, + "rewards/HPSV21TextReward/std": 0.11415313929319382, + "rewards/MANIQAReward/mean": 1.064821720123291, + "rewards/MANIQAReward/std": 0.1917555332183838, + "rewards/QwenFakeDiscrimReward/mean": 0.13671875, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.163671851158142, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.1875, + "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, + "step": 142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03813333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0021487551275640726, + "learning_rate": 9.981066666666668e-06, + "loss": 0.0002, + "num_tokens": 2481352.0, + "reward": 6.365453243255615, + "reward_std": 0.39431869983673096, + "rewards/CLIPTextReward/mean": 1.108916997909546, + "rewards/CLIPTextReward/std": 0.19837119430303574, + "rewards/HPSV21TextReward/mean": 0.8179616928100586, + "rewards/HPSV21TextReward/std": 0.14136216789484024, + "rewards/MANIQAReward/mean": 1.0003676414489746, + "rewards/MANIQAReward/std": 0.14379417896270752, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.151953101158142, + "rewards/QwenTextReward/std": 0.0631992369890213, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0384, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.002176680602133274, + "learning_rate": 9.980933333333335e-06, + "loss": 0.0002, + "num_tokens": 2498640.0, + "reward": 6.363863945007324, + "reward_std": 0.44721901416778564, + "rewards/CLIPTextReward/mean": 1.0735910534858704, + "rewards/CLIPTextReward/std": 0.18625196814537048, + "rewards/HPSV21TextReward/mean": 0.8158817291259766, + "rewards/HPSV21TextReward/std": 0.06047128513455391, + "rewards/MANIQAReward/mean": 1.0064024925231934, + "rewards/MANIQAReward/std": 0.16212058067321777, + "rewards/QwenFakeDiscrimReward/mean": 0.203125, + "rewards/QwenFakeDiscrimReward/std": 0.09834947437047958, + "rewards/QwenTextReward/mean": 1.148828148841858, + "rewards/QwenTextReward/std": 0.06790951639413834, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03866666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0020736553706228733, + "learning_rate": 9.980800000000001e-06, + "loss": 0.0002, + "num_tokens": 2516112.0, + "reward": 6.5203447341918945, + "reward_std": 0.4181431531906128, + "rewards/CLIPTextReward/mean": 1.1117887496948242, + "rewards/CLIPTextReward/std": 0.21418820321559906, + "rewards/HPSV21TextReward/mean": 0.8538103103637695, + "rewards/HPSV21TextReward/std": 0.10615929216146469, + "rewards/MANIQAReward/mean": 1.0532093048095703, + "rewards/MANIQAReward/std": 0.18675057590007782, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1453125476837158, + "rewards/QwenTextReward/std": 0.0670931488275528, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.038933333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0022541475482285023, + "learning_rate": 9.980666666666667e-06, + "loss": 0.0002, + "num_tokens": 2533504.0, + "reward": 6.394295692443848, + "reward_std": 0.5058107376098633, + "rewards/CLIPTextReward/mean": 1.1201102137565613, + "rewards/CLIPTextReward/std": 0.21684333682060242, + "rewards/HPSV21TextReward/mean": 0.8399591445922852, + "rewards/HPSV21TextReward/std": 0.13806646317243576, + "rewards/MANIQAReward/mean": 0.988610565662384, + "rewards/MANIQAReward/std": 0.1654188632965088, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1339844465255737, + "rewards/QwenTextReward/std": 0.08572372794151306, + "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, + "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, + "step": 146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0392, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0021095983684062958, + "learning_rate": 9.980533333333334e-06, + "loss": 0.0002, + "num_tokens": 2550824.0, + "reward": 6.628732204437256, + "reward_std": 0.4145030677318573, + "rewards/CLIPTextReward/mean": 1.1308990716934204, + "rewards/CLIPTextReward/std": 0.19320135563611984, + "rewards/HPSV21TextReward/mean": 0.8865270614624023, + "rewards/HPSV21TextReward/std": 0.1742558628320694, + "rewards/MANIQAReward/mean": 1.0040363073349, + "rewards/MANIQAReward/std": 0.1383267045021057, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.171875, + "rewards/QwenTextReward/std": 0.03659627586603165, + "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, + "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, + "step": 147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.039466666666666664, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.0022249275352805853, + "learning_rate": 9.980400000000002e-06, + "loss": 0.0002, + "num_tokens": 2568096.0, + "reward": 6.243812561035156, + "reward_std": 0.39003342390060425, + "rewards/CLIPTextReward/mean": 1.0446674823760986, + "rewards/CLIPTextReward/std": 0.22870275378227234, + "rewards/HPSV21TextReward/mean": 0.7968297004699707, + "rewards/HPSV21TextReward/std": 0.09016577154397964, + "rewards/MANIQAReward/mean": 1.035818099975586, + "rewards/MANIQAReward/std": 0.14643150568008423, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.1421875953674316, + "rewards/QwenTextReward/std": 0.08635878562927246, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.039733333333333336, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.25, + "kl": 0.0019969972781836987, + "learning_rate": 9.980266666666667e-06, + "loss": 0.0002, + "num_tokens": 2585368.0, + "reward": 6.457371711730957, + "reward_std": 0.38209855556488037, + "rewards/CLIPTextReward/mean": 1.1367380023002625, + "rewards/CLIPTextReward/std": 0.21426362544298172, + "rewards/HPSV21TextReward/mean": 0.8346633911132812, + "rewards/HPSV21TextReward/std": 0.1631467305123806, + "rewards/MANIQAReward/mean": 0.980975329875946, + "rewards/MANIQAReward/std": 0.22830736637115479, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.154687523841858, + "rewards/QwenTextReward/std": 0.06884465366601944, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.0022310742642730474, + "learning_rate": 9.980133333333335e-06, + "loss": 0.0002, + "num_tokens": 2603032.0, + "reward": 6.2441020011901855, + "reward_std": 0.4181395471096039, + "rewards/CLIPTextReward/mean": 1.0882028341293335, + "rewards/CLIPTextReward/std": 0.21721915900707245, + "rewards/HPSV21TextReward/mean": 0.802515983581543, + "rewards/HPSV21TextReward/std": 0.08247944712638855, + "rewards/MANIQAReward/mean": 0.9611018896102905, + "rewards/MANIQAReward/std": 0.15407003462314606, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.142187476158142, + "rewards/QwenTextReward/std": 0.0691322535276413, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 150 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.040266666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.296875, + "kl": 0.002213691361248493, + "learning_rate": 9.980000000000001e-06, + "loss": 0.0002, + "num_tokens": 2620440.0, + "reward": 6.366701602935791, + "reward_std": 0.38833650946617126, + "rewards/CLIPTextReward/mean": 1.1051047444343567, + "rewards/CLIPTextReward/std": 0.20750798285007477, + "rewards/HPSV21TextReward/mean": 0.8217039108276367, + "rewards/HPSV21TextReward/std": 0.07610554248094559, + "rewards/MANIQAReward/mean": 0.9685529470443726, + "rewards/MANIQAReward/std": 0.18813130259513855, + "rewards/QwenFakeDiscrimReward/mean": 0.1640625, + "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, + "rewards/QwenTextReward/mean": 1.1695313453674316, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04053333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0021160109899938107, + "learning_rate": 9.979866666666667e-06, + "loss": 0.0002, + "num_tokens": 2637816.0, + "reward": 6.335112571716309, + "reward_std": 0.48140987753868103, + "rewards/CLIPTextReward/mean": 1.090110182762146, + "rewards/CLIPTextReward/std": 0.20128751546144485, + "rewards/HPSV21TextReward/mean": 0.8006181716918945, + "rewards/HPSV21TextReward/std": 0.07966043055057526, + "rewards/MANIQAReward/mean": 0.9727966785430908, + "rewards/MANIQAReward/std": 0.16122058033943176, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.1628906726837158, + "rewards/QwenTextReward/std": 0.06929238885641098, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0408, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0022329348139464855, + "learning_rate": 9.979733333333335e-06, + "loss": 0.0002, + "num_tokens": 2655216.0, + "reward": 6.241914749145508, + "reward_std": 0.4123866558074951, + "rewards/CLIPTextReward/mean": 1.0632864832878113, + "rewards/CLIPTextReward/std": 0.20508232712745667, + "rewards/HPSV21TextReward/mean": 0.8128824234008789, + "rewards/HPSV21TextReward/std": 0.07496695220470428, + "rewards/MANIQAReward/mean": 0.9583266973495483, + "rewards/MANIQAReward/std": 0.14616796374320984, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.12890625, + "rewards/QwenTextReward/std": 0.10673908144235611, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04106666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.002218130510300398, + "learning_rate": 9.9796e-06, + "loss": 0.0002, + "num_tokens": 2672568.0, + "reward": 6.160798072814941, + "reward_std": 0.36988532543182373, + "rewards/CLIPTextReward/mean": 1.0214861631393433, + "rewards/CLIPTextReward/std": 0.20107388496398926, + "rewards/HPSV21TextReward/mean": 0.8351411819458008, + "rewards/HPSV21TextReward/std": 0.07012557238340378, + "rewards/MANIQAReward/mean": 0.9299652576446533, + "rewards/MANIQAReward/std": 0.2092486470937729, + "rewards/QwenFakeDiscrimReward/mean": 0.1875, + "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, + "rewards/QwenTextReward/mean": 1.123046875, + "rewards/QwenTextReward/std": 0.08811347186565399, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04133333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.0023776735179126263, + "learning_rate": 9.979466666666667e-06, + "loss": 0.0002, + "num_tokens": 2689960.0, + "reward": 6.323511123657227, + "reward_std": 0.4532202482223511, + "rewards/CLIPTextReward/mean": 1.0690406560897827, + "rewards/CLIPTextReward/std": 0.2158183827996254, + "rewards/HPSV21TextReward/mean": 0.8493661880493164, + "rewards/HPSV21TextReward/std": 0.18082763254642487, + "rewards/MANIQAReward/mean": 1.0003697872161865, + "rewards/MANIQAReward/std": 0.1615719050168991, + "rewards/QwenFakeDiscrimReward/mean": 0.12890625, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.138671875, + "rewards/QwenTextReward/std": 0.08716543763875961, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0416, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.002176026813685894, + "learning_rate": 9.979333333333335e-06, + "loss": 0.0002, + "num_tokens": 2707200.0, + "reward": 6.3571672439575195, + "reward_std": 0.3851667046546936, + "rewards/CLIPTextReward/mean": 1.1091200113296509, + "rewards/CLIPTextReward/std": 0.1840614676475525, + "rewards/HPSV21TextReward/mean": 0.8067169189453125, + "rewards/HPSV21TextReward/std": 0.07062850892543793, + "rewards/MANIQAReward/mean": 0.9915091395378113, + "rewards/MANIQAReward/std": 0.17302347719669342, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04186666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.002383410930633545, + "learning_rate": 9.979200000000001e-06, + "loss": 0.0002, + "num_tokens": 2724544.0, + "reward": 6.514974594116211, + "reward_std": 0.3913232684135437, + "rewards/CLIPTextReward/mean": 1.1650667190551758, + "rewards/CLIPTextReward/std": 0.2027941346168518, + "rewards/HPSV21TextReward/mean": 0.8568296432495117, + "rewards/HPSV21TextReward/std": 0.12092380970716476, + "rewards/MANIQAReward/mean": 0.9797755479812622, + "rewards/MANIQAReward/std": 0.20642335712909698, + "rewards/QwenFakeDiscrimReward/mean": 0.11328125, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.1554687023162842, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.042133333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0022624253761023283, + "learning_rate": 9.979066666666667e-06, + "loss": 0.0002, + "num_tokens": 2741944.0, + "reward": 6.376334190368652, + "reward_std": 0.4033576250076294, + "rewards/CLIPTextReward/mean": 1.0273826122283936, + "rewards/CLIPTextReward/std": 0.17035045847296715, + "rewards/HPSV21TextReward/mean": 0.8467292785644531, + "rewards/HPSV21TextReward/std": 0.12334073334932327, + "rewards/MANIQAReward/mean": 1.0710792541503906, + "rewards/MANIQAReward/std": 0.1529785394668579, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.162500023841858, + "rewards/QwenTextReward/std": 0.03779647499322891, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 158 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0424, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.296875, + "kl": 0.0022776061668992043, + "learning_rate": 9.978933333333334e-06, + "loss": 0.0002, + "num_tokens": 2759200.0, + "reward": 6.441936016082764, + "reward_std": 0.413552463054657, + "rewards/CLIPTextReward/mean": 1.0375493168830872, + "rewards/CLIPTextReward/std": 0.18821316957473755, + "rewards/HPSV21TextReward/mean": 0.8919734954833984, + "rewards/HPSV21TextReward/std": 0.18911729753017426, + "rewards/MANIQAReward/mean": 1.0360153913497925, + "rewards/MANIQAReward/std": 0.18743227422237396, + "rewards/QwenFakeDiscrimReward/mean": 0.1953125, + "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, + "rewards/QwenTextReward/mean": 1.1484375, + "rewards/QwenTextReward/std": 0.0625000149011612, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.042666666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3203125, + "kl": 0.0021952856332063675, + "learning_rate": 9.9788e-06, + "loss": 0.0002, + "num_tokens": 2776488.0, + "reward": 6.365789413452148, + "reward_std": 0.4353575110435486, + "rewards/CLIPTextReward/mean": 1.08657968044281, + "rewards/CLIPTextReward/std": 0.17875386402010918, + "rewards/HPSV21TextReward/mean": 0.8376073837280273, + "rewards/HPSV21TextReward/std": 0.11195782572031021, + "rewards/MANIQAReward/mean": 0.9701499342918396, + "rewards/MANIQAReward/std": 0.17033298313617706, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1566405296325684, + "rewards/QwenTextReward/std": 0.03965182974934578, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 160 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04293333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0024832834023982286, + "learning_rate": 9.978666666666668e-06, + "loss": 0.0002, + "num_tokens": 2793872.0, + "reward": 6.553257942199707, + "reward_std": 0.3911113739013672, + "rewards/CLIPTextReward/mean": 1.1585108041763306, + "rewards/CLIPTextReward/std": 0.19251200556755066, + "rewards/HPSV21TextReward/mean": 0.8372735977172852, + "rewards/HPSV21TextReward/std": 0.11238152533769608, + "rewards/MANIQAReward/mean": 1.0202827453613281, + "rewards/MANIQAReward/std": 0.13903705775737762, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1625001430511475, + "rewards/QwenTextReward/std": 0.037796471267938614, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 161 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0432, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3828125, + "kl": 0.0024976269342005253, + "learning_rate": 9.978533333333335e-06, + "loss": 0.0002, + "num_tokens": 2811160.0, + "reward": 6.327694892883301, + "reward_std": 0.3723014295101166, + "rewards/CLIPTextReward/mean": 1.0655624270439148, + "rewards/CLIPTextReward/std": 0.20780667662620544, + "rewards/HPSV21TextReward/mean": 0.8458852767944336, + "rewards/HPSV21TextReward/std": 0.11304989457130432, + "rewards/MANIQAReward/mean": 0.9915183782577515, + "rewards/MANIQAReward/std": 0.21026821434497833, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03750002756714821, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04346666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0023513694759458303, + "learning_rate": 9.978400000000001e-06, + "loss": 0.0002, + "num_tokens": 2828504.0, + "reward": 6.148275375366211, + "reward_std": 0.3822818398475647, + "rewards/CLIPTextReward/mean": 1.0270581245422363, + "rewards/CLIPTextReward/std": 0.1667494922876358, + "rewards/HPSV21TextReward/mean": 0.8215246200561523, + "rewards/HPSV21TextReward/std": 0.11641214042901993, + "rewards/MANIQAReward/mean": 0.898766279220581, + "rewards/MANIQAReward/std": 0.1385030746459961, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03750002756714821, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04373333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.002564219059422612, + "learning_rate": 9.978266666666667e-06, + "loss": 0.0003, + "num_tokens": 2845856.0, + "reward": 6.511420726776123, + "reward_std": 0.4238984286785126, + "rewards/CLIPTextReward/mean": 1.1618956923484802, + "rewards/CLIPTextReward/std": 0.19716718792915344, + "rewards/HPSV21TextReward/mean": 0.8364629745483398, + "rewards/HPSV21TextReward/std": 0.10770930349826813, + "rewards/MANIQAReward/mean": 1.012359380722046, + "rewards/MANIQAReward/std": 0.15895412862300873, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.142968773841858, + "rewards/QwenTextReward/std": 0.06643842905759811, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.044, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0024520137812942266, + "learning_rate": 9.978133333333334e-06, + "loss": 0.0002, + "num_tokens": 2863304.0, + "reward": 6.206347465515137, + "reward_std": 0.40142396092414856, + "rewards/CLIPTextReward/mean": 1.0258528590202332, + "rewards/CLIPTextReward/std": 0.21651653200387955, + "rewards/HPSV21TextReward/mean": 0.8280124664306641, + "rewards/HPSV21TextReward/std": 0.09464716911315918, + "rewards/MANIQAReward/mean": 0.970491886138916, + "rewards/MANIQAReward/std": 0.13681063055992126, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1648437976837158, + "rewards/QwenTextReward/std": 0.040019553154706955, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04426666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.002554597333073616, + "learning_rate": 9.978000000000002e-06, + "loss": 0.0003, + "num_tokens": 2880624.0, + "reward": 6.23911190032959, + "reward_std": 0.40902307629585266, + "rewards/CLIPTextReward/mean": 1.0364893674850464, + "rewards/CLIPTextReward/std": 0.1683896705508232, + "rewards/HPSV21TextReward/mean": 0.8219265937805176, + "rewards/HPSV21TextReward/std": 0.11840753257274628, + "rewards/MANIQAReward/mean": 0.9871234893798828, + "rewards/MANIQAReward/std": 0.19988511502742767, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1484375, + "rewards/QwenTextReward/std": 0.03503827005624771, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.044533333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.0023744776844978333, + "learning_rate": 9.977866666666668e-06, + "loss": 0.0002, + "num_tokens": 2898024.0, + "reward": 6.0969343185424805, + "reward_std": 0.32953351736068726, + "rewards/CLIPTextReward/mean": 1.0005998611450195, + "rewards/CLIPTextReward/std": 0.17174231261014938, + "rewards/HPSV21TextReward/mean": 0.7955303192138672, + "rewards/HPSV21TextReward/std": 0.06337456405162811, + "rewards/MANIQAReward/mean": 0.971861720085144, + "rewards/MANIQAReward/std": 0.14192479848861694, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1460936069488525, + "rewards/QwenTextReward/std": 0.03398728370666504, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0448, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.002273611957207322, + "learning_rate": 9.977733333333334e-06, + "loss": 0.0002, + "num_tokens": 2915264.0, + "reward": 6.160523414611816, + "reward_std": 0.3825192451477051, + "rewards/CLIPTextReward/mean": 1.0522464513778687, + "rewards/CLIPTextReward/std": 0.20829912275075912, + "rewards/HPSV21TextReward/mean": 0.7879104614257812, + "rewards/HPSV21TextReward/std": 0.07972346991300583, + "rewards/MANIQAReward/mean": 0.9302089810371399, + "rewards/MANIQAReward/std": 0.16054274141788483, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1476563215255737, + "rewards/QwenTextReward/std": 0.08518407493829727, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.045066666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.002443203004077077, + "learning_rate": 9.9776e-06, + "loss": 0.0002, + "num_tokens": 2932608.0, + "reward": 6.5020294189453125, + "reward_std": 0.484386682510376, + "rewards/CLIPTextReward/mean": 1.1416503190994263, + "rewards/CLIPTextReward/std": 0.1884015128016472, + "rewards/HPSV21TextReward/mean": 0.8431720733642578, + "rewards/HPSV21TextReward/std": 0.11021529138088226, + "rewards/MANIQAReward/mean": 1.0234001874923706, + "rewards/MANIQAReward/std": 0.18853236734867096, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1535155773162842, + "rewards/QwenTextReward/std": 0.06869927048683167, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 169 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04533333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.296875, + "kl": 0.0025329249911010265, + "learning_rate": 9.977466666666667e-06, + "loss": 0.0003, + "num_tokens": 2949912.0, + "reward": 6.431511878967285, + "reward_std": 0.407115638256073, + "rewards/CLIPTextReward/mean": 1.1464118957519531, + "rewards/CLIPTextReward/std": 0.20208461582660675, + "rewards/HPSV21TextReward/mean": 0.859166145324707, + "rewards/HPSV21TextReward/std": 0.12445147335529327, + "rewards/MANIQAReward/mean": 0.9699652791023254, + "rewards/MANIQAReward/std": 0.1636308878660202, + "rewards/QwenFakeDiscrimReward/mean": 0.125, + "rewards/QwenFakeDiscrimReward/std": 0.1259881556034088, + "rewards/QwenTextReward/mean": 1.1496093273162842, + "rewards/QwenTextReward/std": 0.03792543709278107, + "rewards/QwenWeirdDiscrimReward/mean": 0.17578125, + "rewards/QwenWeirdDiscrimReward/std": 0.1151231899857521, + "step": 170 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0456, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.0024203923530876637, + "learning_rate": 9.977333333333335e-06, + "loss": 0.0002, + "num_tokens": 2967104.0, + "reward": 6.227117538452148, + "reward_std": 0.3750118017196655, + "rewards/CLIPTextReward/mean": 1.0488858819007874, + "rewards/CLIPTextReward/std": 0.20298413932323456, + "rewards/HPSV21TextReward/mean": 0.8152523040771484, + "rewards/HPSV21TextReward/std": 0.06391119211912155, + "rewards/MANIQAReward/mean": 0.9804813265800476, + "rewards/MANIQAReward/std": 0.13435937464237213, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1472656726837158, + "rewards/QwenTextReward/std": 0.03453698009252548, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04586666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.002392132766544819, + "learning_rate": 9.977200000000002e-06, + "loss": 0.0002, + "num_tokens": 2984488.0, + "reward": 6.098992824554443, + "reward_std": 0.41760361194610596, + "rewards/CLIPTextReward/mean": 0.9913454055786133, + "rewards/CLIPTextReward/std": 0.17980967462062836, + "rewards/HPSV21TextReward/mean": 0.8134450912475586, + "rewards/HPSV21TextReward/std": 0.08377031236886978, + "rewards/MANIQAReward/mean": 0.9780833125114441, + "rewards/MANIQAReward/std": 0.20588435232639313, + "rewards/QwenFakeDiscrimReward/mean": 0.1640625, + "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, + "rewards/QwenTextReward/mean": 1.144140601158142, + "rewards/QwenTextReward/std": 0.06677703559398651, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04613333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0026148189790546894, + "learning_rate": 9.977066666666666e-06, + "loss": 0.0003, + "num_tokens": 3001728.0, + "reward": 6.125906467437744, + "reward_std": 0.3209711015224457, + "rewards/CLIPTextReward/mean": 0.9812254905700684, + "rewards/CLIPTextReward/std": 0.17670557647943497, + "rewards/HPSV21TextReward/mean": 0.7952060699462891, + "rewards/HPSV21TextReward/std": 0.06432045996189117, + "rewards/MANIQAReward/mean": 1.0378872156143188, + "rewards/MANIQAReward/std": 0.1683635413646698, + "rewards/QwenFakeDiscrimReward/mean": 0.1875, + "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, + "rewards/QwenTextReward/mean": 1.12890625, + "rewards/QwenTextReward/std": 0.09493301808834076, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0464, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.0021870234049856663, + "learning_rate": 9.976933333333334e-06, + "loss": 0.0002, + "num_tokens": 3019096.0, + "reward": 6.402133941650391, + "reward_std": 0.28885915875434875, + "rewards/CLIPTextReward/mean": 1.1439542174339294, + "rewards/CLIPTextReward/std": 0.20595373958349228, + "rewards/HPSV21TextReward/mean": 0.806889533996582, + "rewards/HPSV21TextReward/std": 0.1222410574555397, + "rewards/MANIQAReward/mean": 0.9363840818405151, + "rewards/MANIQAReward/std": 0.19096361100673676, + "rewards/QwenFakeDiscrimReward/mean": 0.1953125, + "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03750002756714821, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04666666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.00249657966196537, + "learning_rate": 9.9768e-06, + "loss": 0.0002, + "num_tokens": 3036536.0, + "reward": 6.335534572601318, + "reward_std": 0.4115269184112549, + "rewards/CLIPTextReward/mean": 1.0820544362068176, + "rewards/CLIPTextReward/std": 0.1995326355099678, + "rewards/HPSV21TextReward/mean": 0.8357610702514648, + "rewards/HPSV21TextReward/std": 0.07408785820007324, + "rewards/MANIQAReward/mean": 0.9698255062103271, + "rewards/MANIQAReward/std": 0.20197954773902893, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1472656726837158, + "rewards/QwenTextReward/std": 0.06222037598490715, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.046933333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.00256872596219182, + "learning_rate": 9.976666666666667e-06, + "loss": 0.0003, + "num_tokens": 3053840.0, + "reward": 6.486705780029297, + "reward_std": 0.4940839409828186, + "rewards/CLIPTextReward/mean": 1.12992525100708, + "rewards/CLIPTextReward/std": 0.19487006962299347, + "rewards/HPSV21TextReward/mean": 0.8539524078369141, + "rewards/HPSV21TextReward/std": 0.08571457117795944, + "rewards/MANIQAReward/mean": 0.9849660396575928, + "rewards/MANIQAReward/std": 0.19841398298740387, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1394531726837158, + "rewards/QwenTextReward/std": 0.0833100974559784, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0472, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.002719606040045619, + "learning_rate": 9.976533333333334e-06, + "loss": 0.0003, + "num_tokens": 3071208.0, + "reward": 6.385272979736328, + "reward_std": 0.38871097564697266, + "rewards/CLIPTextReward/mean": 1.066335141658783, + "rewards/CLIPTextReward/std": 0.20620963722467422, + "rewards/HPSV21TextReward/mean": 0.8252353668212891, + "rewards/HPSV21TextReward/std": 0.1381026692688465, + "rewards/MANIQAReward/mean": 1.0204912424087524, + "rewards/MANIQAReward/std": 0.13698863983154297, + "rewards/QwenFakeDiscrimReward/mean": 0.19140625, + "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, + "rewards/QwenTextReward/mean": 1.151953101158142, + "rewards/QwenTextReward/std": 0.03627080097794533, + "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, + "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, + "step": 177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.047466666666666664, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3984375, + "kl": 0.002723751123994589, + "learning_rate": 9.9764e-06, + "loss": 0.0003, + "num_tokens": 3088552.0, + "reward": 6.708035469055176, + "reward_std": 0.46653276681900024, + "rewards/CLIPTextReward/mean": 1.1932567358016968, + "rewards/CLIPTextReward/std": 0.2232767716050148, + "rewards/HPSV21TextReward/mean": 0.9151687622070312, + "rewards/HPSV21TextReward/std": 0.1896015703678131, + "rewards/MANIQAReward/mean": 0.9661847949028015, + "rewards/MANIQAReward/std": 0.19881469011306763, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.149999976158142, + "rewards/QwenTextReward/std": 0.06813853234052658, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.047733333333333336, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0027056399267166853, + "learning_rate": 9.976266666666668e-06, + "loss": 0.0003, + "num_tokens": 3105920.0, + "reward": 6.258008003234863, + "reward_std": 0.4084116816520691, + "rewards/CLIPTextReward/mean": 1.0594270825386047, + "rewards/CLIPTextReward/std": 0.19307219237089157, + "rewards/HPSV21TextReward/mean": 0.8365631103515625, + "rewards/HPSV21TextReward/std": 0.1051463857293129, + "rewards/MANIQAReward/mean": 0.9742305278778076, + "rewards/MANIQAReward/std": 0.17492277920246124, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1402344703674316, + "rewards/QwenTextReward/std": 0.06002930551767349, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 179 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.048, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.002679926110431552, + "learning_rate": 9.976133333333334e-06, + "loss": 0.0003, + "num_tokens": 3123304.0, + "reward": 6.29672384262085, + "reward_std": 0.35875028371810913, + "rewards/CLIPTextReward/mean": 1.0460641980171204, + "rewards/CLIPTextReward/std": 0.19920749962329865, + "rewards/HPSV21TextReward/mean": 0.8230581283569336, + "rewards/HPSV21TextReward/std": 0.07715026289224625, + "rewards/MANIQAReward/mean": 1.0112135410308838, + "rewards/MANIQAReward/std": 0.1757921427488327, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.1566405296325684, + "rewards/QwenTextReward/std": 0.037332236766815186, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 180 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.048266666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.0026333339046686888, + "learning_rate": 9.976e-06, + "loss": 0.0003, + "num_tokens": 3140632.0, + "reward": 6.529520034790039, + "reward_std": 0.37621504068374634, + "rewards/CLIPTextReward/mean": 1.1153369545936584, + "rewards/CLIPTextReward/std": 0.19007403403520584, + "rewards/HPSV21TextReward/mean": 0.8622684478759766, + "rewards/HPSV21TextReward/std": 0.056736063212156296, + "rewards/MANIQAReward/mean": 1.0161064863204956, + "rewards/MANIQAReward/std": 0.17400076985359192, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1519532203674316, + "rewards/QwenTextReward/std": 0.03627080097794533, + "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, + "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, + "step": 181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04853333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.002458418719470501, + "learning_rate": 9.975866666666667e-06, + "loss": 0.0002, + "num_tokens": 3158040.0, + "reward": 6.337980270385742, + "reward_std": 0.43443620204925537, + "rewards/CLIPTextReward/mean": 1.1128054857254028, + "rewards/CLIPTextReward/std": 0.2091159000992775, + "rewards/HPSV21TextReward/mean": 0.8239784240722656, + "rewards/HPSV21TextReward/std": 0.07653089612722397, + "rewards/MANIQAReward/mean": 0.9612877368927002, + "rewards/MANIQAReward/std": 0.1491847187280655, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.1320312023162842, + "rewards/QwenTextReward/std": 0.056250013411045074, + "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, + "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, + "step": 182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0488, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.002701894612982869, + "learning_rate": 9.975733333333333e-06, + "loss": 0.0003, + "num_tokens": 3175392.0, + "reward": 6.474644660949707, + "reward_std": 0.4059009850025177, + "rewards/CLIPTextReward/mean": 1.1510314345359802, + "rewards/CLIPTextReward/std": 0.20656609535217285, + "rewards/HPSV21TextReward/mean": 0.8551139831542969, + "rewards/HPSV21TextReward/std": 0.1395208202302456, + "rewards/MANIQAReward/mean": 0.9021977186203003, + "rewards/MANIQAReward/std": 0.2259361743927002, + "rewards/QwenFakeDiscrimReward/mean": 0.19921875, + "rewards/QwenFakeDiscrimReward/std": 0.10137632489204407, + "rewards/QwenTextReward/mean": 1.1343750953674316, + "rewards/QwenTextReward/std": 0.08813546597957611, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 183 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04906666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.002543346956372261, + "learning_rate": 9.975600000000002e-06, + "loss": 0.0003, + "num_tokens": 3192720.0, + "reward": 6.173674583435059, + "reward_std": 0.3256201446056366, + "rewards/CLIPTextReward/mean": 1.0212828516960144, + "rewards/CLIPTextReward/std": 0.15655865147709846, + "rewards/HPSV21TextReward/mean": 0.8296718597412109, + "rewards/HPSV21TextReward/std": 0.05253772437572479, + "rewards/MANIQAReward/mean": 0.9475463032722473, + "rewards/MANIQAReward/std": 0.1381036341190338, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1648437976837158, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, + "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, + "step": 184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04933333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.421875, + "kl": 0.00288868835195899, + "learning_rate": 9.975466666666668e-06, + "loss": 0.0003, + "num_tokens": 3210160.0, + "reward": 6.346151828765869, + "reward_std": 0.27956622838974, + "rewards/CLIPTextReward/mean": 1.0762921571731567, + "rewards/CLIPTextReward/std": 0.2376992329955101, + "rewards/HPSV21TextReward/mean": 0.7927217483520508, + "rewards/HPSV21TextReward/std": 0.06304909288883209, + "rewards/MANIQAReward/mean": 1.0503114461898804, + "rewards/MANIQAReward/std": 0.1577564924955368, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1593749523162842, + "rewards/QwenTextReward/std": 0.06922188401222229, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0496, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.0028415676206350327, + "learning_rate": 9.975333333333334e-06, + "loss": 0.0003, + "num_tokens": 3227456.0, + "reward": 6.274043083190918, + "reward_std": 0.3639392852783203, + "rewards/CLIPTextReward/mean": 1.0660734176635742, + "rewards/CLIPTextReward/std": 0.1816587634384632, + "rewards/HPSV21TextReward/mean": 0.8182039260864258, + "rewards/HPSV21TextReward/std": 0.07657317072153091, + "rewards/MANIQAReward/mean": 0.9797074198722839, + "rewards/MANIQAReward/std": 0.1761608123779297, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1625001430511475, + "rewards/QwenTextReward/std": 0.037796471267938614, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04986666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0028599947690963745, + "learning_rate": 9.9752e-06, + "loss": 0.0003, + "num_tokens": 3244976.0, + "reward": 6.46965217590332, + "reward_std": 0.37160903215408325, + "rewards/CLIPTextReward/mean": 1.1449741125106812, + "rewards/CLIPTextReward/std": 0.1829175353050232, + "rewards/HPSV21TextReward/mean": 0.8212175369262695, + "rewards/HPSV21TextReward/std": 0.05136280506849289, + "rewards/MANIQAReward/mean": 1.021644115447998, + "rewards/MANIQAReward/std": 0.1607440710067749, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.140625, + "rewards/QwenTextReward/std": 0.07420212030410767, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 187 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.050133333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.002837129170075059, + "learning_rate": 9.975066666666667e-06, + "loss": 0.0003, + "num_tokens": 3262384.0, + "reward": 6.557543754577637, + "reward_std": 0.4035520851612091, + "rewards/CLIPTextReward/mean": 1.1948737502098083, + "rewards/CLIPTextReward/std": 0.16932519897818565, + "rewards/HPSV21TextReward/mean": 0.8404254913330078, + "rewards/HPSV21TextReward/std": 0.12645912915468216, + "rewards/MANIQAReward/mean": 1.0502266883850098, + "rewards/MANIQAReward/std": 0.14965708553791046, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1359374523162842, + "rewards/QwenTextReward/std": 0.09808434545993805, + "rewards/QwenWeirdDiscrimReward/mean": 0.15234375, + "rewards/QwenWeirdDiscrimReward/std": 0.12293684482574463, + "step": 188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0504, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3828125, + "kl": 0.002935183933004737, + "learning_rate": 9.974933333333335e-06, + "loss": 0.0003, + "num_tokens": 3279672.0, + "reward": 6.320755958557129, + "reward_std": 0.327908992767334, + "rewards/CLIPTextReward/mean": 1.0910344123840332, + "rewards/CLIPTextReward/std": 0.19866855442523956, + "rewards/HPSV21TextReward/mean": 0.8061733245849609, + "rewards/HPSV21TextReward/std": 0.07433304190635681, + "rewards/MANIQAReward/mean": 0.9818093180656433, + "rewards/MANIQAReward/std": 0.1730826199054718, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03750002384185791, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.050666666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.002721617929637432, + "learning_rate": 9.974800000000001e-06, + "loss": 0.0003, + "num_tokens": 3296968.0, + "reward": 6.403863906860352, + "reward_std": 0.4016572833061218, + "rewards/CLIPTextReward/mean": 1.0907533168792725, + "rewards/CLIPTextReward/std": 0.2178114652633667, + "rewards/HPSV21TextReward/mean": 0.8505439758300781, + "rewards/HPSV21TextReward/std": 0.09670589119195938, + "rewards/MANIQAReward/mean": 1.00994074344635, + "rewards/MANIQAReward/std": 0.15439607203006744, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.1675782203674316, + "rewards/QwenTextReward/std": 0.0424170047044754, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 190 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05093333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0026723656337708235, + "learning_rate": 9.974666666666666e-06, + "loss": 0.0003, + "num_tokens": 3314224.0, + "reward": 6.2754364013671875, + "reward_std": 0.3998555541038513, + "rewards/CLIPTextReward/mean": 1.1050440073013306, + "rewards/CLIPTextReward/std": 0.19693446159362793, + "rewards/HPSV21TextReward/mean": 0.8342494964599609, + "rewards/HPSV21TextReward/std": 0.07819408923387527, + "rewards/MANIQAReward/mean": 0.923412024974823, + "rewards/MANIQAReward/std": 0.17528018355369568, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.1414062976837158, + "rewards/QwenTextReward/std": 0.06045772135257721, + "rewards/QwenWeirdDiscrimReward/mean": 0.1875, + "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, + "step": 191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0512, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.002696257084608078, + "learning_rate": 9.974533333333334e-06, + "loss": 0.0003, + "num_tokens": 3331568.0, + "reward": 6.29323673248291, + "reward_std": 0.2823619246482849, + "rewards/CLIPTextReward/mean": 1.008976697921753, + "rewards/CLIPTextReward/std": 0.16197307035326958, + "rewards/HPSV21TextReward/mean": 0.8356037139892578, + "rewards/HPSV21TextReward/std": 0.06890447437763214, + "rewards/MANIQAReward/mean": 1.03376305103302, + "rewards/MANIQAReward/std": 0.15510410070419312, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, + "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, + "step": 192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05146666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.296875, + "kl": 0.002745483536273241, + "learning_rate": 9.9744e-06, + "loss": 0.0003, + "num_tokens": 3349128.0, + "reward": 6.267972946166992, + "reward_std": 0.3895288109779358, + "rewards/CLIPTextReward/mean": 1.0269521474838257, + "rewards/CLIPTextReward/std": 0.1757335141301155, + "rewards/HPSV21TextReward/mean": 0.823420524597168, + "rewards/HPSV21TextReward/std": 0.07999104261398315, + "rewards/MANIQAReward/mean": 1.050821304321289, + "rewards/MANIQAReward/std": 0.18627648055553436, + "rewards/QwenFakeDiscrimReward/mean": 0.19140625, + "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, + "rewards/QwenTextReward/mean": 1.1257812976837158, + "rewards/QwenTextReward/std": 0.10331498086452484, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05173333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.002754206769168377, + "learning_rate": 9.974266666666667e-06, + "loss": 0.0003, + "num_tokens": 3366480.0, + "reward": 6.110285758972168, + "reward_std": 0.38404354453086853, + "rewards/CLIPTextReward/mean": 0.9982666969299316, + "rewards/CLIPTextReward/std": 0.1378244236111641, + "rewards/HPSV21TextReward/mean": 0.8200778961181641, + "rewards/HPSV21TextReward/std": 0.07046207785606384, + "rewards/MANIQAReward/mean": 0.9497687816619873, + "rewards/MANIQAReward/std": 0.14510276913642883, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.1292967796325684, + "rewards/QwenTextReward/std": 0.08220121264457703, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.052, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0027757398784160614, + "learning_rate": 9.974133333333335e-06, + "loss": 0.0003, + "num_tokens": 3383832.0, + "reward": 6.2371015548706055, + "reward_std": 0.40047281980514526, + "rewards/CLIPTextReward/mean": 1.050218403339386, + "rewards/CLIPTextReward/std": 0.2267727330327034, + "rewards/HPSV21TextReward/mean": 0.8354701995849609, + "rewards/HPSV21TextReward/std": 0.07300396263599396, + "rewards/MANIQAReward/mean": 0.9864278435707092, + "rewards/MANIQAReward/std": 0.16592912375926971, + "rewards/QwenFakeDiscrimReward/mean": 0.1328125, + "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, + "rewards/QwenTextReward/mean": 1.1511719226837158, + "rewards/QwenTextReward/std": 0.06834637373685837, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05226666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.002869168994948268, + "learning_rate": 9.974e-06, + "loss": 0.0003, + "num_tokens": 3401176.0, + "reward": 6.375538349151611, + "reward_std": 0.3194682002067566, + "rewards/CLIPTextReward/mean": 1.1040527820587158, + "rewards/CLIPTextReward/std": 0.21156348288059235, + "rewards/HPSV21TextReward/mean": 0.8256998062133789, + "rewards/HPSV21TextReward/std": 0.06539477407932281, + "rewards/MANIQAReward/mean": 0.9972831010818481, + "rewards/MANIQAReward/std": 0.18009570240974426, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1554687023162842, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.052533333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.002735574496909976, + "learning_rate": 9.973866666666668e-06, + "loss": 0.0003, + "num_tokens": 3418504.0, + "reward": 6.325824737548828, + "reward_std": 0.38276347517967224, + "rewards/CLIPTextReward/mean": 1.0841413140296936, + "rewards/CLIPTextReward/std": 0.19735275954008102, + "rewards/HPSV21TextReward/mean": 0.8420505523681641, + "rewards/HPSV21TextReward/std": 0.08244600147008896, + "rewards/MANIQAReward/mean": 0.967581570148468, + "rewards/MANIQAReward/std": 0.18143029510974884, + "rewards/QwenFakeDiscrimReward/mean": 0.1328125, + "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, + "rewards/QwenTextReward/mean": 1.138671875, + "rewards/QwenTextReward/std": 0.14412638545036316, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0528, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.002685937564820051, + "learning_rate": 9.973733333333334e-06, + "loss": 0.0003, + "num_tokens": 3435840.0, + "reward": 6.567035675048828, + "reward_std": 0.469005823135376, + "rewards/CLIPTextReward/mean": 1.124721884727478, + "rewards/CLIPTextReward/std": 0.2038327008485794, + "rewards/HPSV21TextReward/mean": 0.8921480178833008, + "rewards/HPSV21TextReward/std": 0.1626519188284874, + "rewards/MANIQAReward/mean": 0.9938427209854126, + "rewards/MANIQAReward/std": 0.15762895345687866, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1566405296325684, + "rewards/QwenTextReward/std": 0.037332236766815186, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.053066666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0027355619240552187, + "learning_rate": 9.9736e-06, + "loss": 0.0003, + "num_tokens": 3453160.0, + "reward": 6.437889099121094, + "reward_std": 0.38492536544799805, + "rewards/CLIPTextReward/mean": 1.1193333864212036, + "rewards/CLIPTextReward/std": 0.1936298832297325, + "rewards/HPSV21TextReward/mean": 0.8576078414916992, + "rewards/HPSV21TextReward/std": 0.13272829726338387, + "rewards/MANIQAReward/mean": 0.9707251787185669, + "rewards/MANIQAReward/std": 0.16395625472068787, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1460936069488525, + "rewards/QwenTextReward/std": 0.03398728370666504, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 199 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05333333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.002614394761621952, + "learning_rate": 9.973466666666669e-06, + "loss": 0.0003, + "num_tokens": 3470480.0, + "reward": 6.575470447540283, + "reward_std": 0.3986605405807495, + "rewards/CLIPTextReward/mean": 1.113551139831543, + "rewards/CLIPTextReward/std": 0.20553741604089737, + "rewards/HPSV21TextReward/mean": 0.8867435455322266, + "rewards/HPSV21TextReward/std": 0.1364203877747059, + "rewards/MANIQAReward/mean": 1.0623810291290283, + "rewards/MANIQAReward/std": 0.1529991775751114, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1531250476837158, + "rewards/QwenTextReward/std": 0.036596279591321945, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 200 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0536, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.002905803034082055, + "learning_rate": 9.973333333333333e-06, + "loss": 0.0003, + "num_tokens": 3487880.0, + "reward": 6.478170394897461, + "reward_std": 0.49340736865997314, + "rewards/CLIPTextReward/mean": 1.1195530891418457, + "rewards/CLIPTextReward/std": 0.20537645369768143, + "rewards/HPSV21TextReward/mean": 0.831049919128418, + "rewards/HPSV21TextReward/std": 0.08279094845056534, + "rewards/MANIQAReward/mean": 1.0441515445709229, + "rewards/MANIQAReward/std": 0.15340273082256317, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.1460938453674316, + "rewards/QwenTextReward/std": 0.061916936188936234, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 201 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05386666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.002745371311903, + "learning_rate": 9.973200000000001e-06, + "loss": 0.0003, + "num_tokens": 3505312.0, + "reward": 6.431427001953125, + "reward_std": 0.33977192640304565, + "rewards/CLIPTextReward/mean": 1.1148839592933655, + "rewards/CLIPTextReward/std": 0.2095688208937645, + "rewards/HPSV21TextReward/mean": 0.8343162536621094, + "rewards/HPSV21TextReward/std": 0.08807380497455597, + "rewards/MANIQAReward/mean": 0.9779485464096069, + "rewards/MANIQAReward/std": 0.15203088521957397, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1449220180511475, + "rewards/QwenTextReward/std": 0.06158936396241188, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05413333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.002616970334202051, + "learning_rate": 9.973066666666668e-06, + "loss": 0.0003, + "num_tokens": 3522752.0, + "reward": 6.270486831665039, + "reward_std": 0.3961990475654602, + "rewards/CLIPTextReward/mean": 1.089812994003296, + "rewards/CLIPTextReward/std": 0.2142699658870697, + "rewards/HPSV21TextReward/mean": 0.7937088012695312, + "rewards/HPSV21TextReward/std": 0.0994756817817688, + "rewards/MANIQAReward/mean": 0.9725837707519531, + "rewards/MANIQAReward/std": 0.14475348591804504, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1519532203674316, + "rewards/QwenTextReward/std": 0.0386541411280632, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0544, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.0029000602662563324, + "learning_rate": 9.972933333333334e-06, + "loss": 0.0003, + "num_tokens": 3540184.0, + "reward": 6.369531154632568, + "reward_std": 0.3710113763809204, + "rewards/CLIPTextReward/mean": 1.1070588827133179, + "rewards/CLIPTextReward/std": 0.2002292200922966, + "rewards/HPSV21TextReward/mean": 0.7954778671264648, + "rewards/HPSV21TextReward/std": 0.06739770621061325, + "rewards/MANIQAReward/mean": 1.0457077026367188, + "rewards/MANIQAReward/std": 0.12836866080760956, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1554687023162842, + "rewards/QwenTextReward/std": 0.03945784643292427, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05466666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4375, + "kl": 0.0038333996199071407, + "learning_rate": 9.972800000000002e-06, + "loss": 0.0004, + "num_tokens": 3557584.0, + "reward": 6.196996688842773, + "reward_std": 0.3954958915710449, + "rewards/CLIPTextReward/mean": 1.0345997214317322, + "rewards/CLIPTextReward/std": 0.20499616861343384, + "rewards/HPSV21TextReward/mean": 0.7918825149536133, + "rewards/HPSV21TextReward/std": 0.0736343041062355, + "rewards/MANIQAReward/mean": 1.0256726741790771, + "rewards/MANIQAReward/std": 0.1692875772714615, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1394531726837158, + "rewards/QwenTextReward/std": 0.0833100974559784, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.054933333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.0026570966001600027, + "learning_rate": 9.972666666666667e-06, + "loss": 0.0003, + "num_tokens": 3574896.0, + "reward": 6.4823150634765625, + "reward_std": 0.40404587984085083, + "rewards/CLIPTextReward/mean": 1.1078510284423828, + "rewards/CLIPTextReward/std": 0.20066715776920319, + "rewards/HPSV21TextReward/mean": 0.846653938293457, + "rewards/HPSV21TextReward/std": 0.12554003298282623, + "rewards/MANIQAReward/mean": 1.0229145288467407, + "rewards/MANIQAReward/std": 0.17728054523468018, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1636719703674316, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0552, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8046875, + "kl": 0.002860800828784704, + "learning_rate": 9.972533333333335e-06, + "loss": 0.0003, + "num_tokens": 3592240.0, + "reward": 6.497309684753418, + "reward_std": 0.4115002751350403, + "rewards/CLIPTextReward/mean": 1.1606194376945496, + "rewards/CLIPTextReward/std": 0.195783831179142, + "rewards/HPSV21TextReward/mean": 0.84130859375, + "rewards/HPSV21TextReward/std": 0.11015837639570236, + "rewards/MANIQAReward/mean": 0.9379844665527344, + "rewards/MANIQAReward/std": 0.17234882712364197, + "rewards/QwenFakeDiscrimReward/mean": 0.1953125, + "rewards/QwenFakeDiscrimReward/std": 0.1041666716337204, + "rewards/QwenTextReward/mean": 1.1531250476837158, + "rewards/QwenTextReward/std": 0.03659627586603165, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.055466666666666664, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.003817642340436578, + "learning_rate": 9.972400000000001e-06, + "loss": 0.0004, + "num_tokens": 3609680.0, + "reward": 6.229999542236328, + "reward_std": 0.3975546360015869, + "rewards/CLIPTextReward/mean": 1.076364815235138, + "rewards/CLIPTextReward/std": 0.21021181344985962, + "rewards/HPSV21TextReward/mean": 0.8293190002441406, + "rewards/HPSV21TextReward/std": 0.08222578465938568, + "rewards/MANIQAReward/mean": 0.9467570781707764, + "rewards/MANIQAReward/std": 0.14959219098091125, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.1085937023162842, + "rewards/QwenTextReward/std": 0.1296529918909073, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.055733333333333336, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.002774016000330448, + "learning_rate": 9.972266666666668e-06, + "loss": 0.0003, + "num_tokens": 3626928.0, + "reward": 6.374789237976074, + "reward_std": 0.36420223116874695, + "rewards/CLIPTextReward/mean": 1.0703842639923096, + "rewards/CLIPTextReward/std": 0.21086003631353378, + "rewards/HPSV21TextReward/mean": 0.8615245819091797, + "rewards/HPSV21TextReward/std": 0.05977732688188553, + "rewards/MANIQAReward/mean": 0.9527685642242432, + "rewards/MANIQAReward/std": 0.15766838192939758, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.163671851158142, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.056, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.002732650376856327, + "learning_rate": 9.972133333333334e-06, + "loss": 0.0003, + "num_tokens": 3644248.0, + "reward": 6.305462837219238, + "reward_std": 0.456421822309494, + "rewards/CLIPTextReward/mean": 1.0700019001960754, + "rewards/CLIPTextReward/std": 0.1975710168480873, + "rewards/HPSV21TextReward/mean": 0.8302736282348633, + "rewards/HPSV21TextReward/std": 0.0959872305393219, + "rewards/MANIQAReward/mean": 0.992802619934082, + "rewards/MANIQAReward/std": 0.15043874084949493, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.144921898841858, + "rewards/QwenTextReward/std": 0.06158936396241188, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 210 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.056266666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.002957995980978012, + "learning_rate": 9.972e-06, + "loss": 0.0003, + "num_tokens": 3661552.0, + "reward": 6.458515167236328, + "reward_std": 0.352350115776062, + "rewards/CLIPTextReward/mean": 1.1411879062652588, + "rewards/CLIPTextReward/std": 0.20142141729593277, + "rewards/HPSV21TextReward/mean": 0.8212127685546875, + "rewards/HPSV21TextReward/std": 0.0886462852358818, + "rewards/MANIQAReward/mean": 1.009104609489441, + "rewards/MANIQAReward/std": 0.15248696506023407, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1417968273162842, + "rewards/QwenTextReward/std": 0.06607697904109955, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05653333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0038779843598604202, + "learning_rate": 9.971866666666667e-06, + "loss": 0.0004, + "num_tokens": 3678864.0, + "reward": 6.417540550231934, + "reward_std": 0.43140143156051636, + "rewards/CLIPTextReward/mean": 1.106486737728119, + "rewards/CLIPTextReward/std": 0.20730093866586685, + "rewards/HPSV21TextReward/mean": 0.863398551940918, + "rewards/HPSV21TextReward/std": 0.06245674192905426, + "rewards/MANIQAReward/mean": 1.007067322731018, + "rewards/MANIQAReward/std": 0.2022874653339386, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.130859375, + "rewards/QwenTextReward/std": 0.07595193386077881, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0568, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.003234908450394869, + "learning_rate": 9.971733333333335e-06, + "loss": 0.0003, + "num_tokens": 3696200.0, + "reward": 6.502376079559326, + "reward_std": 0.4503940939903259, + "rewards/CLIPTextReward/mean": 1.117896556854248, + "rewards/CLIPTextReward/std": 0.2203962802886963, + "rewards/HPSV21TextReward/mean": 0.8834381103515625, + "rewards/HPSV21TextReward/std": 0.1342969425022602, + "rewards/MANIQAReward/mean": 1.007910132408142, + "rewards/MANIQAReward/std": 0.16741357743740082, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1636719703674316, + "rewards/QwenTextReward/std": 0.04007181152701378, + "rewards/QwenWeirdDiscrimReward/mean": 0.1796875, + "rewards/QwenWeirdDiscrimReward/std": 0.11329087615013123, + "step": 213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05706666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0028942429926246405, + "learning_rate": 9.971600000000001e-06, + "loss": 0.0003, + "num_tokens": 3713480.0, + "reward": 6.227326393127441, + "reward_std": 0.3156473636627197, + "rewards/CLIPTextReward/mean": 1.0346298217773438, + "rewards/CLIPTextReward/std": 0.18641937524080276, + "rewards/HPSV21TextReward/mean": 0.836186408996582, + "rewards/HPSV21TextReward/std": 0.10609928518533707, + "rewards/MANIQAReward/mean": 0.9368653297424316, + "rewards/MANIQAReward/std": 0.18543748557567596, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.154296875, + "rewards/QwenTextReward/std": 0.039227355271577835, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05733333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0032178538385778666, + "learning_rate": 9.971466666666668e-06, + "loss": 0.0003, + "num_tokens": 3730968.0, + "reward": 6.278270721435547, + "reward_std": 0.5345197916030884, + "rewards/CLIPTextReward/mean": 1.1357209086418152, + "rewards/CLIPTextReward/std": 0.19665353745222092, + "rewards/HPSV21TextReward/mean": 0.7758393287658691, + "rewards/HPSV21TextReward/std": 0.07874829322099686, + "rewards/MANIQAReward/mean": 0.9817126393318176, + "rewards/MANIQAReward/std": 0.1834738850593567, + "rewards/QwenFakeDiscrimReward/mean": 0.10546875, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.1570312976837158, + "rewards/QwenTextReward/std": 0.042367640882730484, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0576, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0027981791645288467, + "learning_rate": 9.971333333333334e-06, + "loss": 0.0003, + "num_tokens": 3748240.0, + "reward": 6.433413028717041, + "reward_std": 0.35207629203796387, + "rewards/CLIPTextReward/mean": 1.0457903146743774, + "rewards/CLIPTextReward/std": 0.1923365518450737, + "rewards/HPSV21TextReward/mean": 0.8758325576782227, + "rewards/HPSV21TextReward/std": 0.1333918683230877, + "rewards/MANIQAReward/mean": 1.0339173078536987, + "rewards/MANIQAReward/std": 0.15527579188346863, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03750002384185791, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.057866666666666663, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0031056725420057774, + "learning_rate": 9.9712e-06, + "loss": 0.0003, + "num_tokens": 3765528.0, + "reward": 6.364522933959961, + "reward_std": 0.3756159543991089, + "rewards/CLIPTextReward/mean": 1.0495604276657104, + "rewards/CLIPTextReward/std": 0.17222807928919792, + "rewards/HPSV21TextReward/mean": 0.8294382095336914, + "rewards/HPSV21TextReward/std": 0.07039421051740646, + "rewards/MANIQAReward/mean": 1.0276196002960205, + "rewards/MANIQAReward/std": 0.21525166928768158, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1648437976837158, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, + "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, + "step": 217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.058133333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0027923269663006067, + "learning_rate": 9.971066666666668e-06, + "loss": 0.0003, + "num_tokens": 3783032.0, + "reward": 6.405855178833008, + "reward_std": 0.3659675717353821, + "rewards/CLIPTextReward/mean": 1.1177189350128174, + "rewards/CLIPTextReward/std": 0.198662169277668, + "rewards/HPSV21TextReward/mean": 0.8444490432739258, + "rewards/HPSV21TextReward/std": 0.07149509340524673, + "rewards/MANIQAReward/mean": 0.9584724307060242, + "rewards/MANIQAReward/std": 0.1412878930568695, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.151953101158142, + "rewards/QwenTextReward/std": 0.03627080097794533, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0584, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4296875, + "kl": 0.003411591053009033, + "learning_rate": 9.970933333333335e-06, + "loss": 0.0003, + "num_tokens": 3800272.0, + "reward": 6.541419506072998, + "reward_std": 0.35087212920188904, + "rewards/CLIPTextReward/mean": 1.1012699604034424, + "rewards/CLIPTextReward/std": 0.1799757182598114, + "rewards/HPSV21TextReward/mean": 0.8397064208984375, + "rewards/HPSV21TextReward/std": 0.11745796352624893, + "rewards/MANIQAReward/mean": 1.0981385707855225, + "rewards/MANIQAReward/std": 0.14114679396152496, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.058666666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0031459140591323376, + "learning_rate": 9.970800000000001e-06, + "loss": 0.0003, + "num_tokens": 3817560.0, + "reward": 6.071300029754639, + "reward_std": 0.4156203866004944, + "rewards/CLIPTextReward/mean": 1.0444402694702148, + "rewards/CLIPTextReward/std": 0.20825284719467163, + "rewards/HPSV21TextReward/mean": 0.8236780166625977, + "rewards/HPSV21TextReward/std": 0.06731405109167099, + "rewards/MANIQAReward/mean": 0.9178757667541504, + "rewards/MANIQAReward/std": 0.142966166138649, + "rewards/QwenFakeDiscrimReward/mean": 0.125, + "rewards/QwenFakeDiscrimReward/std": 0.1259881556034088, + "rewards/QwenTextReward/mean": 1.1203124523162842, + "rewards/QwenTextReward/std": 0.12286874651908875, + "rewards/QwenWeirdDiscrimReward/mean": 0.171875, + "rewards/QwenWeirdDiscrimReward/std": 0.116794154047966, + "step": 220 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05893333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.003471190808340907, + "learning_rate": 9.970666666666668e-06, + "loss": 0.0003, + "num_tokens": 3834912.0, + "reward": 6.415256500244141, + "reward_std": 0.3701218366622925, + "rewards/CLIPTextReward/mean": 1.1057547330856323, + "rewards/CLIPTextReward/std": 0.20841475576162338, + "rewards/HPSV21TextReward/mean": 0.8342828750610352, + "rewards/HPSV21TextReward/std": 0.08610900491476059, + "rewards/MANIQAReward/mean": 1.0148690938949585, + "rewards/MANIQAReward/std": 0.15851165354251862, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1375000476837158, + "rewards/QwenTextReward/std": 0.10681167244911194, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 221 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0592, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4453125, + "kl": 0.0032579144462943077, + "learning_rate": 9.970533333333334e-06, + "loss": 0.0003, + "num_tokens": 3852312.0, + "reward": 6.209277153015137, + "reward_std": 0.425199419260025, + "rewards/CLIPTextReward/mean": 1.028854489326477, + "rewards/CLIPTextReward/std": 0.2329263687133789, + "rewards/HPSV21TextReward/mean": 0.8044242858886719, + "rewards/HPSV21TextReward/std": 0.1179187074303627, + "rewards/MANIQAReward/mean": 1.0532667636871338, + "rewards/MANIQAReward/std": 0.1624266505241394, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1535155773162842, + "rewards/QwenTextReward/std": 0.06869927048683167, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05946666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.003359306836500764, + "learning_rate": 9.970400000000002e-06, + "loss": 0.0003, + "num_tokens": 3869640.0, + "reward": 6.436337947845459, + "reward_std": 0.4217972159385681, + "rewards/CLIPTextReward/mean": 1.0638182163238525, + "rewards/CLIPTextReward/std": 0.18361417949199677, + "rewards/HPSV21TextReward/mean": 0.8837671279907227, + "rewards/HPSV21TextReward/std": 0.13798930868506432, + "rewards/MANIQAReward/mean": 1.0372610092163086, + "rewards/MANIQAReward/std": 0.1778913289308548, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.15234375, + "rewards/QwenTextReward/std": 0.06853322684764862, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05973333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.003051916602998972, + "learning_rate": 9.970266666666667e-06, + "loss": 0.0003, + "num_tokens": 3887000.0, + "reward": 6.635891437530518, + "reward_std": 0.45852988958358765, + "rewards/CLIPTextReward/mean": 1.1718319654464722, + "rewards/CLIPTextReward/std": 0.1853807494044304, + "rewards/HPSV21TextReward/mean": 0.8488368988037109, + "rewards/HPSV21TextReward/std": 0.11583320796489716, + "rewards/MANIQAReward/mean": 1.0285377502441406, + "rewards/MANIQAReward/std": 0.1676163673400879, + "rewards/QwenFakeDiscrimReward/mean": 0.19140625, + "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, + "rewards/QwenTextReward/mean": 1.1480469703674316, + "rewards/QwenTextReward/std": 0.0706130787730217, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4609375, + "kl": 0.003515611169859767, + "learning_rate": 9.970133333333335e-06, + "loss": 0.0004, + "num_tokens": 3904352.0, + "reward": 6.577434539794922, + "reward_std": 0.38679757714271545, + "rewards/CLIPTextReward/mean": 1.110971450805664, + "rewards/CLIPTextReward/std": 0.2008369341492653, + "rewards/HPSV21TextReward/mean": 0.8672590255737305, + "rewards/HPSV21TextReward/std": 0.12437398359179497, + "rewards/MANIQAReward/mean": 1.0385515689849854, + "rewards/MANIQAReward/std": 0.1615229696035385, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.168359398841858, + "rewards/QwenTextReward/std": 0.037332240492105484, + "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, + "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, + "step": 225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06026666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.0031529960688203573, + "learning_rate": 9.970000000000001e-06, + "loss": 0.0003, + "num_tokens": 3921704.0, + "reward": 6.4590959548950195, + "reward_std": 0.35303205251693726, + "rewards/CLIPTextReward/mean": 1.111771583557129, + "rewards/CLIPTextReward/std": 0.20289433002471924, + "rewards/HPSV21TextReward/mean": 0.8199443817138672, + "rewards/HPSV21TextReward/std": 0.07744725793600082, + "rewards/MANIQAReward/mean": 1.0612890720367432, + "rewards/MANIQAReward/std": 0.16553428769111633, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1476563215255737, + "rewards/QwenTextReward/std": 0.06765910983085632, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.060533333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.003157949075102806, + "learning_rate": 9.969866666666667e-06, + "loss": 0.0003, + "num_tokens": 3938992.0, + "reward": 6.377033233642578, + "reward_std": 0.3737572729587555, + "rewards/CLIPTextReward/mean": 1.0765042304992676, + "rewards/CLIPTextReward/std": 0.21037136763334274, + "rewards/HPSV21TextReward/mean": 0.8247890472412109, + "rewards/HPSV21TextReward/std": 0.07355935126543045, + "rewards/MANIQAReward/mean": 1.0334311723709106, + "rewards/MANIQAReward/std": 0.17133867740631104, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.154296875, + "rewards/QwenTextReward/std": 0.03688107430934906, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0608, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.390625, + "kl": 0.003242003731429577, + "learning_rate": 9.969733333333334e-06, + "loss": 0.0003, + "num_tokens": 3956368.0, + "reward": 6.312893867492676, + "reward_std": 0.35467755794525146, + "rewards/CLIPTextReward/mean": 1.090986728668213, + "rewards/CLIPTextReward/std": 0.2179025486111641, + "rewards/HPSV21TextReward/mean": 0.8224906921386719, + "rewards/HPSV21TextReward/std": 0.07208240032196045, + "rewards/MANIQAReward/mean": 0.9781261086463928, + "rewards/MANIQAReward/std": 0.13308070600032806, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.140625, + "rewards/QwenTextReward/std": 0.08363044261932373, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.061066666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.390625, + "kl": 0.0035423971712589264, + "learning_rate": 9.9696e-06, + "loss": 0.0004, + "num_tokens": 3973664.0, + "reward": 6.631295204162598, + "reward_std": 0.31362563371658325, + "rewards/CLIPTextReward/mean": 1.1526328921318054, + "rewards/CLIPTextReward/std": 0.18516195937991142, + "rewards/HPSV21TextReward/mean": 0.8869228363037109, + "rewards/HPSV21TextReward/std": 0.12250558286905289, + "rewards/MANIQAReward/mean": 0.9685895442962646, + "rewards/MANIQAReward/std": 0.1559518277645111, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.169531226158142, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, + "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, + "step": 229 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06133333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.40625, + "kl": 0.003313015680760145, + "learning_rate": 9.969466666666667e-06, + "loss": 0.0003, + "num_tokens": 3990984.0, + "reward": 6.636979103088379, + "reward_std": 0.31966114044189453, + "rewards/CLIPTextReward/mean": 1.1439168453216553, + "rewards/CLIPTextReward/std": 0.18489812687039375, + "rewards/HPSV21TextReward/mean": 0.8825368881225586, + "rewards/HPSV21TextReward/std": 0.12275548651814461, + "rewards/MANIQAReward/mean": 1.0493061542510986, + "rewards/MANIQAReward/std": 0.18121196329593658, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1519532203674316, + "rewards/QwenTextReward/std": 0.0631992369890213, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 230 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0616, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3828125, + "kl": 0.0030702250078320503, + "learning_rate": 9.969333333333335e-06, + "loss": 0.0003, + "num_tokens": 4008440.0, + "reward": 6.474734306335449, + "reward_std": 0.43573421239852905, + "rewards/CLIPTextReward/mean": 1.0921568274497986, + "rewards/CLIPTextReward/std": 0.17702462896704674, + "rewards/HPSV21TextReward/mean": 0.8393945693969727, + "rewards/HPSV21TextReward/std": 0.0692838579416275, + "rewards/MANIQAReward/mean": 1.0081157684326172, + "rewards/MANIQAReward/std": 0.15015669167041779, + "rewards/QwenFakeDiscrimReward/mean": 0.19140625, + "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, + "rewards/QwenTextReward/mean": 1.166015625, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.24609375, + "rewards/QwenWeirdDiscrimReward/std": 0.03125, + "step": 231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06186666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.40625, + "kl": 0.0032026702538132668, + "learning_rate": 9.969200000000001e-06, + "loss": 0.0003, + "num_tokens": 4025784.0, + "reward": 6.492210388183594, + "reward_std": 0.3398313820362091, + "rewards/CLIPTextReward/mean": 1.1206194162368774, + "rewards/CLIPTextReward/std": 0.2059924080967903, + "rewards/HPSV21TextReward/mean": 0.8321371078491211, + "rewards/HPSV21TextReward/std": 0.0851968377828598, + "rewards/MANIQAReward/mean": 1.0527129173278809, + "rewards/MANIQAReward/std": 0.12888813018798828, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.054621223360300064, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06213333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.625, + "kl": 0.00353748956695199, + "learning_rate": 9.969066666666667e-06, + "loss": 0.0004, + "num_tokens": 4043136.0, + "reward": 6.196873664855957, + "reward_std": 0.36793988943099976, + "rewards/CLIPTextReward/mean": 1.0475822687149048, + "rewards/CLIPTextReward/std": 0.19828400015830994, + "rewards/HPSV21TextReward/mean": 0.8117570877075195, + "rewards/HPSV21TextReward/std": 0.07968274503946304, + "rewards/MANIQAReward/mean": 0.9211634397506714, + "rewards/MANIQAReward/std": 0.15552812814712524, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.174218773841858, + "rewards/QwenTextReward/std": 0.03590353950858116, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0624, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.003463034052401781, + "learning_rate": 9.968933333333334e-06, + "loss": 0.0003, + "num_tokens": 4060512.0, + "reward": 6.254439353942871, + "reward_std": 0.41635259985923767, + "rewards/CLIPTextReward/mean": 1.0879863500595093, + "rewards/CLIPTextReward/std": 0.19300304353237152, + "rewards/HPSV21TextReward/mean": 0.8314504623413086, + "rewards/HPSV21TextReward/std": 0.0751388818025589, + "rewards/MANIQAReward/mean": 0.9788466095924377, + "rewards/MANIQAReward/std": 0.16533632576465607, + "rewards/QwenFakeDiscrimReward/mean": 0.125, + "rewards/QwenFakeDiscrimReward/std": 0.1259881556034088, + "rewards/QwenTextReward/mean": 1.1359374523162842, + "rewards/QwenTextReward/std": 0.06391262263059616, + "rewards/QwenWeirdDiscrimReward/mean": 0.17578125, + "rewards/QwenWeirdDiscrimReward/std": 0.1151231899857521, + "step": 234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06266666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.0030904915183782578, + "learning_rate": 9.9688e-06, + "loss": 0.0003, + "num_tokens": 4077760.0, + "reward": 6.185770034790039, + "reward_std": 0.4532530605792999, + "rewards/CLIPTextReward/mean": 1.028242588043213, + "rewards/CLIPTextReward/std": 0.18711334466934204, + "rewards/HPSV21TextReward/mean": 0.8127584457397461, + "rewards/HPSV21TextReward/std": 0.0767570361495018, + "rewards/MANIQAReward/mean": 1.0061118602752686, + "rewards/MANIQAReward/std": 0.14821553230285645, + "rewards/QwenFakeDiscrimReward/mean": 0.1328125, + "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03750002384185791, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06293333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0033108042553067207, + "learning_rate": 9.968666666666668e-06, + "loss": 0.0003, + "num_tokens": 4095136.0, + "reward": 6.450545310974121, + "reward_std": 0.34508979320526123, + "rewards/CLIPTextReward/mean": 1.1125239133834839, + "rewards/CLIPTextReward/std": 0.18488702550530434, + "rewards/HPSV21TextReward/mean": 0.83349609375, + "rewards/HPSV21TextReward/std": 0.057444117963314056, + "rewards/MANIQAReward/mean": 1.0319424867630005, + "rewards/MANIQAReward/std": 0.15041349828243256, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.1554687023162842, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0632, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.0028371165972203016, + "learning_rate": 9.968533333333335e-06, + "loss": 0.0003, + "num_tokens": 4112496.0, + "reward": 6.167689323425293, + "reward_std": 0.32739606499671936, + "rewards/CLIPTextReward/mean": 1.048931360244751, + "rewards/CLIPTextReward/std": 0.19749532639980316, + "rewards/HPSV21TextReward/mean": 0.7899513244628906, + "rewards/HPSV21TextReward/std": 0.07274974882602692, + "rewards/MANIQAReward/mean": 0.9731267690658569, + "rewards/MANIQAReward/std": 0.15662546455860138, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1378905773162842, + "rewards/QwenTextReward/std": 0.104558065533638, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06346666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.003085110802203417, + "learning_rate": 9.968400000000001e-06, + "loss": 0.0003, + "num_tokens": 4129840.0, + "reward": 6.481772422790527, + "reward_std": 0.35772961378097534, + "rewards/CLIPTextReward/mean": 1.0813804864883423, + "rewards/CLIPTextReward/std": 0.20720766484737396, + "rewards/HPSV21TextReward/mean": 0.8641490936279297, + "rewards/HPSV21TextReward/std": 0.17907562106847763, + "rewards/MANIQAReward/mean": 1.0344631671905518, + "rewards/MANIQAReward/std": 0.14367032051086426, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1500000953674316, + "rewards/QwenTextReward/std": 0.07637627422809601, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06373333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.002941595856100321, + "learning_rate": 9.968266666666667e-06, + "loss": 0.0003, + "num_tokens": 4147096.0, + "reward": 6.353859901428223, + "reward_std": 0.3981371819972992, + "rewards/CLIPTextReward/mean": 1.063906729221344, + "rewards/CLIPTextReward/std": 0.18121198564767838, + "rewards/HPSV21TextReward/mean": 0.81640625, + "rewards/HPSV21TextReward/std": 0.07464800775051117, + "rewards/MANIQAReward/mean": 0.9889370203018188, + "rewards/MANIQAReward/std": 0.15643689036369324, + "rewards/QwenFakeDiscrimReward/mean": 0.19921875, + "rewards/QwenFakeDiscrimReward/std": 0.10137632489204407, + "rewards/QwenTextReward/mean": 1.1628906726837158, + "rewards/QwenTextReward/std": 0.06929238885641098, + "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, + "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, + "step": 239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.064, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.003230172209441662, + "learning_rate": 9.968133333333334e-06, + "loss": 0.0003, + "num_tokens": 4164432.0, + "reward": 6.608020782470703, + "reward_std": 0.3431330919265747, + "rewards/CLIPTextReward/mean": 1.215427041053772, + "rewards/CLIPTextReward/std": 0.19819049537181854, + "rewards/HPSV21TextReward/mean": 0.8417644500732422, + "rewards/HPSV21TextReward/std": 0.05505716800689697, + "rewards/MANIQAReward/mean": 0.9971534013748169, + "rewards/MANIQAReward/std": 0.14450205862522125, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1449217796325684, + "rewards/QwenTextReward/std": 0.10598927736282349, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 240 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06426666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.003244362771511078, + "learning_rate": 9.968000000000002e-06, + "loss": 0.0003, + "num_tokens": 4181760.0, + "reward": 6.384057521820068, + "reward_std": 0.4403102993965149, + "rewards/CLIPTextReward/mean": 1.1254714727401733, + "rewards/CLIPTextReward/std": 0.1991724595427513, + "rewards/HPSV21TextReward/mean": 0.8270206451416016, + "rewards/HPSV21TextReward/std": 0.07428202033042908, + "rewards/MANIQAReward/mean": 1.000557541847229, + "rewards/MANIQAReward/std": 0.16089297831058502, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.142578125, + "rewards/QwenTextReward/std": 0.034698180854320526, + "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, + "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, + "step": 241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06453333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.003257544245570898, + "learning_rate": 9.967866666666668e-06, + "loss": 0.0003, + "num_tokens": 4199208.0, + "reward": 6.362607479095459, + "reward_std": 0.3074047863483429, + "rewards/CLIPTextReward/mean": 1.0759841203689575, + "rewards/CLIPTextReward/std": 0.19806966930627823, + "rewards/HPSV21TextReward/mean": 0.8266582489013672, + "rewards/HPSV21TextReward/std": 0.09093856066465378, + "rewards/MANIQAReward/mean": 1.0159164667129517, + "rewards/MANIQAReward/std": 0.14935745298862457, + "rewards/QwenFakeDiscrimReward/mean": 0.1875, + "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, + "rewards/QwenTextReward/mean": 1.123437523841858, + "rewards/QwenTextReward/std": 0.13347278535366058, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0648, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.003465704619884491, + "learning_rate": 9.967733333333335e-06, + "loss": 0.0003, + "num_tokens": 4216568.0, + "reward": 6.463565826416016, + "reward_std": 0.3935210704803467, + "rewards/CLIPTextReward/mean": 1.057560682296753, + "rewards/CLIPTextReward/std": 0.17162110283970833, + "rewards/HPSV21TextReward/mean": 0.8747730255126953, + "rewards/HPSV21TextReward/std": 0.11632852256298065, + "rewards/MANIQAReward/mean": 1.0449917316436768, + "rewards/MANIQAReward/std": 0.18094049394130707, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.1554687023162842, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06506666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.265625, + "kl": 0.0031383931636810303, + "learning_rate": 9.967600000000001e-06, + "loss": 0.0003, + "num_tokens": 4233936.0, + "reward": 6.2581281661987305, + "reward_std": 0.4083012044429779, + "rewards/CLIPTextReward/mean": 1.1018357276916504, + "rewards/CLIPTextReward/std": 0.20741251856088638, + "rewards/HPSV21TextReward/mean": 0.811309814453125, + "rewards/HPSV21TextReward/std": 0.10790691524744034, + "rewards/MANIQAReward/mean": 0.953321099281311, + "rewards/MANIQAReward/std": 0.18221963942050934, + "rewards/QwenFakeDiscrimReward/mean": 0.12109375, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.146484375, + "rewards/QwenTextReward/std": 0.039557892829179764, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06533333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0034496267326176167, + "learning_rate": 9.967466666666667e-06, + "loss": 0.0003, + "num_tokens": 4251464.0, + "reward": 6.258942127227783, + "reward_std": 0.467055082321167, + "rewards/CLIPTextReward/mean": 1.0869176387786865, + "rewards/CLIPTextReward/std": 0.22874008864164352, + "rewards/HPSV21TextReward/mean": 0.8402419090270996, + "rewards/HPSV21TextReward/std": 0.11818192154169083, + "rewards/MANIQAReward/mean": 0.9710292816162109, + "rewards/MANIQAReward/std": 0.16109530627727509, + "rewards/QwenFakeDiscrimReward/mean": 0.1171875, + "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, + "rewards/QwenTextReward/mean": 1.140625, + "rewards/QwenTextReward/std": 0.03659627214074135, + "rewards/QwenWeirdDiscrimReward/mean": 0.17578125, + "rewards/QwenWeirdDiscrimReward/std": 0.1151231899857521, + "step": 245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0656, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3984375, + "kl": 0.004035755526274443, + "learning_rate": 9.967333333333334e-06, + "loss": 0.0004, + "num_tokens": 4268856.0, + "reward": 6.789213180541992, + "reward_std": 0.35714399814605713, + "rewards/CLIPTextReward/mean": 1.1455610394477844, + "rewards/CLIPTextReward/std": 0.20604808628559113, + "rewards/HPSV21TextReward/mean": 0.9044771194458008, + "rewards/HPSV21TextReward/std": 0.13277480378746986, + "rewards/MANIQAReward/mean": 1.1317150592803955, + "rewards/MANIQAReward/std": 0.192893385887146, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06586666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4375, + "kl": 0.003506520064547658, + "learning_rate": 9.967200000000002e-06, + "loss": 0.0004, + "num_tokens": 4286104.0, + "reward": 6.166922569274902, + "reward_std": 0.3984561562538147, + "rewards/CLIPTextReward/mean": 1.0409660339355469, + "rewards/CLIPTextReward/std": 0.20897932350635529, + "rewards/HPSV21TextReward/mean": 0.8085098266601562, + "rewards/HPSV21TextReward/std": 0.07335306704044342, + "rewards/MANIQAReward/mean": 0.988283097743988, + "rewards/MANIQAReward/std": 0.1673584133386612, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.1281249523162842, + "rewards/QwenTextReward/std": 0.11104661971330643, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06613333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0034983076620846987, + "learning_rate": 9.967066666666666e-06, + "loss": 0.0004, + "num_tokens": 4303496.0, + "reward": 6.271515846252441, + "reward_std": 0.32968008518218994, + "rewards/CLIPTextReward/mean": 1.08308744430542, + "rewards/CLIPTextReward/std": 0.18308281898498535, + "rewards/HPSV21TextReward/mean": 0.827845573425293, + "rewards/HPSV21TextReward/std": 0.05251236632466316, + "rewards/MANIQAReward/mean": 0.8984781503677368, + "rewards/MANIQAReward/std": 0.16231420636177063, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.168359398841858, + "rewards/QwenTextReward/std": 0.03965182974934578, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0664, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.296875, + "kl": 0.0032820648048073053, + "learning_rate": 9.966933333333334e-06, + "loss": 0.0003, + "num_tokens": 4320896.0, + "reward": 6.3601202964782715, + "reward_std": 0.3747427463531494, + "rewards/CLIPTextReward/mean": 1.0786243677139282, + "rewards/CLIPTextReward/std": 0.18130076304078102, + "rewards/HPSV21TextReward/mean": 0.8438816070556641, + "rewards/HPSV21TextReward/std": 0.07765045017004013, + "rewards/MANIQAReward/mean": 1.0408895015716553, + "rewards/MANIQAReward/std": 0.12134683132171631, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1343750953674316, + "rewards/QwenTextReward/std": 0.0574767179787159, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06666666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.0035632310900837183, + "learning_rate": 9.966800000000001e-06, + "loss": 0.0004, + "num_tokens": 4338272.0, + "reward": 6.261956214904785, + "reward_std": 0.3757457435131073, + "rewards/CLIPTextReward/mean": 1.097538948059082, + "rewards/CLIPTextReward/std": 0.2125762403011322, + "rewards/HPSV21TextReward/mean": 0.8204212188720703, + "rewards/HPSV21TextReward/std": 0.06764323264360428, + "rewards/MANIQAReward/mean": 0.9693953990936279, + "rewards/MANIQAReward/std": 0.13938410580158234, + "rewards/QwenFakeDiscrimReward/mean": 0.12890625, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.140234351158142, + "rewards/QwenTextReward/std": 0.030412914231419563, + "rewards/QwenWeirdDiscrimReward/mean": 0.1875, + "rewards/QwenWeirdDiscrimReward/std": 0.10910894721746445, + "step": 250 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06693333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3828125, + "kl": 0.003846655134111643, + "learning_rate": 9.966666666666667e-06, + "loss": 0.0004, + "num_tokens": 4355688.0, + "reward": 6.575259208679199, + "reward_std": 0.43733108043670654, + "rewards/CLIPTextReward/mean": 1.1088823080062866, + "rewards/CLIPTextReward/std": 0.1903865486383438, + "rewards/HPSV21TextReward/mean": 0.9043560028076172, + "rewards/HPSV21TextReward/std": 0.15610383450984955, + "rewards/MANIQAReward/mean": 1.0280795097351074, + "rewards/MANIQAReward/std": 0.153267964720726, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1496093273162842, + "rewards/QwenTextReward/std": 0.06275618821382523, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0672, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.003265757579356432, + "learning_rate": 9.966533333333334e-06, + "loss": 0.0003, + "num_tokens": 4372928.0, + "reward": 6.416788101196289, + "reward_std": 0.3261052966117859, + "rewards/CLIPTextReward/mean": 1.0913305878639221, + "rewards/CLIPTextReward/std": 0.18725522607564926, + "rewards/HPSV21TextReward/mean": 0.8135738372802734, + "rewards/HPSV21TextReward/std": 0.08006454259157181, + "rewards/MANIQAReward/mean": 1.035104513168335, + "rewards/MANIQAReward/std": 0.1181311383843422, + "rewards/QwenFakeDiscrimReward/mean": 0.20703125, + "rewards/QwenFakeDiscrimReward/std": 0.09506355226039886, + "rewards/QwenTextReward/mean": 1.1304688453674316, + "rewards/QwenTextReward/std": 0.10488740354776382, + "rewards/QwenWeirdDiscrimReward/mean": 0.234375, + "rewards/QwenWeirdDiscrimReward/std": 0.06099375709891319, + "step": 252 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06746666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.003604835830628872, + "learning_rate": 9.9664e-06, + "loss": 0.0004, + "num_tokens": 4390336.0, + "reward": 6.431319236755371, + "reward_std": 0.39881986379623413, + "rewards/CLIPTextReward/mean": 1.1042168140411377, + "rewards/CLIPTextReward/std": 0.1991683542728424, + "rewards/HPSV21TextReward/mean": 0.8092203140258789, + "rewards/HPSV21TextReward/std": 0.07349780946969986, + "rewards/MANIQAReward/mean": 1.038038969039917, + "rewards/MANIQAReward/std": 0.163633793592453, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06773333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2734375, + "kl": 0.0028017221484333277, + "learning_rate": 9.966266666666668e-06, + "loss": 0.0003, + "num_tokens": 4407592.0, + "reward": 6.429580211639404, + "reward_std": 0.43548667430877686, + "rewards/CLIPTextReward/mean": 1.0889440774917603, + "rewards/CLIPTextReward/std": 0.19870372861623764, + "rewards/HPSV21TextReward/mean": 0.8702507019042969, + "rewards/HPSV21TextReward/std": 0.1553412601351738, + "rewards/MANIQAReward/mean": 0.9592373967170715, + "rewards/MANIQAReward/std": 0.19040289521217346, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1613280773162842, + "rewards/QwenTextReward/std": 0.06407613307237625, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.068, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0030375674832612276, + "learning_rate": 9.966133333333334e-06, + "loss": 0.0003, + "num_tokens": 4424856.0, + "reward": 6.504112243652344, + "reward_std": 0.44497406482696533, + "rewards/CLIPTextReward/mean": 1.1087359189987183, + "rewards/CLIPTextReward/std": 0.19022604078054428, + "rewards/HPSV21TextReward/mean": 0.8200454711914062, + "rewards/HPSV21TextReward/std": 0.13096706569194794, + "rewards/MANIQAReward/mean": 1.0363925695419312, + "rewards/MANIQAReward/std": 0.16490438580513, + "rewards/QwenFakeDiscrimReward/mean": 0.19140625, + "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, + "rewards/QwenTextReward/mean": 1.1765625476837158, + "rewards/QwenTextReward/std": 0.03503827005624771, + "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, + "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, + "step": 255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06826666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0036664570216089487, + "learning_rate": 9.966e-06, + "loss": 0.0004, + "num_tokens": 4442272.0, + "reward": 6.318208694458008, + "reward_std": 0.37837815284729004, + "rewards/CLIPTextReward/mean": 1.0883460640907288, + "rewards/CLIPTextReward/std": 0.2118919938802719, + "rewards/HPSV21TextReward/mean": 0.8206787109375, + "rewards/HPSV21TextReward/std": 0.05644918233156204, + "rewards/MANIQAReward/mean": 0.971252977848053, + "rewards/MANIQAReward/std": 0.1337847262620926, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.146093726158142, + "rewards/QwenTextReward/std": 0.06191693991422653, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06853333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6171875, + "kl": 0.0037016423884779215, + "learning_rate": 9.965866666666667e-06, + "loss": 0.0004, + "num_tokens": 4459688.0, + "reward": 6.354341506958008, + "reward_std": 0.42910268902778625, + "rewards/CLIPTextReward/mean": 1.0966795682907104, + "rewards/CLIPTextReward/std": 0.18399309739470482, + "rewards/HPSV21TextReward/mean": 0.8211269378662109, + "rewards/HPSV21TextReward/std": 0.07162735611200333, + "rewards/MANIQAReward/mean": 0.958962619304657, + "rewards/MANIQAReward/std": 0.1704968512058258, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1613280773162842, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0688, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.003262406447902322, + "learning_rate": 9.965733333333334e-06, + "loss": 0.0003, + "num_tokens": 4476952.0, + "reward": 6.296139717102051, + "reward_std": 0.3946528434753418, + "rewards/CLIPTextReward/mean": 1.0265021324157715, + "rewards/CLIPTextReward/std": 0.1562262624502182, + "rewards/HPSV21TextReward/mean": 0.8317289352416992, + "rewards/HPSV21TextReward/std": 0.13052061945199966, + "rewards/MANIQAReward/mean": 1.031239628791809, + "rewards/MANIQAReward/std": 0.15305881202220917, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.149999976158142, + "rewards/QwenTextReward/std": 0.06813853234052658, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06906666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2890625, + "kl": 0.002926542889326811, + "learning_rate": 9.965600000000002e-06, + "loss": 0.0003, + "num_tokens": 4494312.0, + "reward": 6.2506866455078125, + "reward_std": 0.42295345664024353, + "rewards/CLIPTextReward/mean": 1.095668911933899, + "rewards/CLIPTextReward/std": 0.19134847819805145, + "rewards/HPSV21TextReward/mean": 0.7793178558349609, + "rewards/HPSV21TextReward/std": 0.06817104667425156, + "rewards/MANIQAReward/mean": 0.9956351518630981, + "rewards/MANIQAReward/std": 0.19141975045204163, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1222655773162842, + "rewards/QwenTextReward/std": 0.09439840167760849, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06933333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0038751026149839163, + "learning_rate": 9.965466666666668e-06, + "loss": 0.0004, + "num_tokens": 4511528.0, + "reward": 6.591069221496582, + "reward_std": 0.4039503335952759, + "rewards/CLIPTextReward/mean": 1.0908901691436768, + "rewards/CLIPTextReward/std": 0.19581331312656403, + "rewards/HPSV21TextReward/mean": 0.9031896591186523, + "rewards/HPSV21TextReward/std": 0.19001787155866623, + "rewards/MANIQAReward/mean": 1.0818156003952026, + "rewards/MANIQAReward/std": 0.19983670115470886, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.138281226158142, + "rewards/QwenTextReward/std": 0.08297169953584671, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 260 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0696, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0037587848491966724, + "learning_rate": 9.965333333333334e-06, + "loss": 0.0004, + "num_tokens": 4528864.0, + "reward": 6.491024017333984, + "reward_std": 0.3804985284805298, + "rewards/CLIPTextReward/mean": 1.0769048929214478, + "rewards/CLIPTextReward/std": 0.1824543960392475, + "rewards/HPSV21TextReward/mean": 0.8643198013305664, + "rewards/HPSV21TextReward/std": 0.11065323650836945, + "rewards/MANIQAReward/mean": 1.0597467422485352, + "rewards/MANIQAReward/std": 0.14535771310329437, + "rewards/QwenFakeDiscrimReward/mean": 0.1640625, + "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, + "rewards/QwenTextReward/mean": 1.154296875, + "rewards/QwenTextReward/std": 0.06355144083499908, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 261 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06986666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.40625, + "kl": 0.0034617320634424686, + "learning_rate": 9.9652e-06, + "loss": 0.0003, + "num_tokens": 4546272.0, + "reward": 6.14242696762085, + "reward_std": 0.396884024143219, + "rewards/CLIPTextReward/mean": 1.0349934697151184, + "rewards/CLIPTextReward/std": 0.20142310112714767, + "rewards/HPSV21TextReward/mean": 0.7808294296264648, + "rewards/HPSV21TextReward/std": 0.08118784427642822, + "rewards/MANIQAReward/mean": 1.0135157108306885, + "rewards/MANIQAReward/std": 0.14665435254573822, + "rewards/QwenFakeDiscrimReward/mean": 0.125, + "rewards/QwenFakeDiscrimReward/std": 0.1259881556034088, + "rewards/QwenTextReward/mean": 1.1496093273162842, + "rewards/QwenTextReward/std": 0.03549320250749588, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07013333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3828125, + "kl": 0.003490220755338669, + "learning_rate": 9.965066666666667e-06, + "loss": 0.0003, + "num_tokens": 4563752.0, + "reward": 6.163957595825195, + "reward_std": 0.385267436504364, + "rewards/CLIPTextReward/mean": 1.0602093935012817, + "rewards/CLIPTextReward/std": 0.2071426585316658, + "rewards/HPSV21TextReward/mean": 0.7925214767456055, + "rewards/HPSV21TextReward/std": 0.08600976318120956, + "rewards/MANIQAReward/mean": 1.0194330215454102, + "rewards/MANIQAReward/std": 0.1785665899515152, + "rewards/QwenFakeDiscrimReward/mean": 0.109375, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.146093726158142, + "rewards/QwenTextReward/std": 0.03398728370666504, + "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, + "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, + "step": 263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0704, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.004037609323859215, + "learning_rate": 9.964933333333333e-06, + "loss": 0.0004, + "num_tokens": 4581080.0, + "reward": 6.484213352203369, + "reward_std": 0.3694634437561035, + "rewards/CLIPTextReward/mean": 1.1062145233154297, + "rewards/CLIPTextReward/std": 0.1875825934112072, + "rewards/HPSV21TextReward/mean": 0.8689737319946289, + "rewards/HPSV21TextReward/std": 0.10702138394117355, + "rewards/MANIQAReward/mean": 0.9932119846343994, + "rewards/MANIQAReward/std": 0.14993032813072205, + "rewards/QwenFakeDiscrimReward/mean": 0.1640625, + "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.06391262263059616, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07066666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.00397456856444478, + "learning_rate": 9.964800000000002e-06, + "loss": 0.0004, + "num_tokens": 4598448.0, + "reward": 6.335943222045898, + "reward_std": 0.3582858443260193, + "rewards/CLIPTextReward/mean": 1.0521622896194458, + "rewards/CLIPTextReward/std": 0.21286197006702423, + "rewards/HPSV21TextReward/mean": 0.8410053253173828, + "rewards/HPSV21TextReward/std": 0.13608239963650703, + "rewards/MANIQAReward/mean": 1.0343737602233887, + "rewards/MANIQAReward/std": 0.15465207397937775, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.144140601158142, + "rewards/QwenTextReward/std": 0.08448517322540283, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07093333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.375, + "kl": 0.0033863503485918045, + "learning_rate": 9.964666666666666e-06, + "loss": 0.0003, + "num_tokens": 4615800.0, + "reward": 6.447694778442383, + "reward_std": 0.3566947877407074, + "rewards/CLIPTextReward/mean": 1.1556851863861084, + "rewards/CLIPTextReward/std": 0.21023477613925934, + "rewards/HPSV21TextReward/mean": 0.8281173706054688, + "rewards/HPSV21TextReward/std": 0.08502472192049026, + "rewards/MANIQAReward/mean": 0.9918079972267151, + "rewards/MANIQAReward/std": 0.17592301964759827, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1484375, + "rewards/QwenTextReward/std": 0.0625000149011612, + "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, + "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, + "step": 266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0712, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0034869378432631493, + "learning_rate": 9.964533333333334e-06, + "loss": 0.0003, + "num_tokens": 4633048.0, + "reward": 6.298396110534668, + "reward_std": 0.4051980674266815, + "rewards/CLIPTextReward/mean": 1.0385345220565796, + "rewards/CLIPTextReward/std": 0.20875421911478043, + "rewards/HPSV21TextReward/mean": 0.834111213684082, + "rewards/HPSV21TextReward/std": 0.07624765485525131, + "rewards/MANIQAReward/mean": 1.0234172344207764, + "rewards/MANIQAReward/std": 0.18777620792388916, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.166406273841858, + "rewards/QwenTextReward/std": 0.07046034932136536, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07146666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.0043066563084721565, + "learning_rate": 9.9644e-06, + "loss": 0.0004, + "num_tokens": 4650384.0, + "reward": 6.33132266998291, + "reward_std": 0.3792267441749573, + "rewards/CLIPTextReward/mean": 1.0630069971084595, + "rewards/CLIPTextReward/std": 0.208414725959301, + "rewards/HPSV21TextReward/mean": 0.8625335693359375, + "rewards/HPSV21TextReward/std": 0.1617307998239994, + "rewards/MANIQAReward/mean": 0.9564133882522583, + "rewards/MANIQAReward/std": 0.1984596848487854, + "rewards/QwenFakeDiscrimReward/mean": 0.1640625, + "rewards/QwenFakeDiscrimReward/std": 0.11967839300632477, + "rewards/QwenTextReward/mean": 1.156640648841858, + "rewards/QwenTextReward/std": 0.06381431967020035, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 268 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07173333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.421875, + "kl": 0.004046045243740082, + "learning_rate": 9.964266666666667e-06, + "loss": 0.0004, + "num_tokens": 4667816.0, + "reward": 6.5365142822265625, + "reward_std": 0.3556900918483734, + "rewards/CLIPTextReward/mean": 1.0756992101669312, + "rewards/CLIPTextReward/std": 0.19139912724494934, + "rewards/HPSV21TextReward/mean": 0.8689680099487305, + "rewards/HPSV21TextReward/std": 0.05170682817697525, + "rewards/MANIQAReward/mean": 1.0948357582092285, + "rewards/MANIQAReward/std": 0.14325103163719177, + "rewards/QwenFakeDiscrimReward/mean": 0.16796875, + "rewards/QwenFakeDiscrimReward/std": 0.1183105930685997, + "rewards/QwenTextReward/mean": 1.1695313453674316, + "rewards/QwenTextReward/std": 0.03945784643292427, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.072, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3203125, + "kl": 0.003918897360563278, + "learning_rate": 9.964133333333335e-06, + "loss": 0.0004, + "num_tokens": 4685160.0, + "reward": 6.477409839630127, + "reward_std": 0.41067662835121155, + "rewards/CLIPTextReward/mean": 1.1299338340759277, + "rewards/CLIPTextReward/std": 0.20776088535785675, + "rewards/HPSV21TextReward/mean": 0.8103847503662109, + "rewards/HPSV21TextReward/std": 0.10611110925674438, + "rewards/MANIQAReward/mean": 1.0385692119598389, + "rewards/MANIQAReward/std": 0.18805089592933655, + "rewards/QwenFakeDiscrimReward/mean": 0.1875, + "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, + "rewards/QwenTextReward/mean": 1.1636719703674316, + "rewards/QwenTextReward/std": 0.037778012454509735, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 270 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07226666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.0036119483411312103, + "learning_rate": 9.964e-06, + "loss": 0.0004, + "num_tokens": 4702432.0, + "reward": 6.398820877075195, + "reward_std": 0.37300944328308105, + "rewards/CLIPTextReward/mean": 1.0580294132232666, + "rewards/CLIPTextReward/std": 0.18786108493804932, + "rewards/HPSV21TextReward/mean": 0.8465709686279297, + "rewards/HPSV21TextReward/std": 0.0728863999247551, + "rewards/MANIQAReward/mean": 1.0263386964797974, + "rewards/MANIQAReward/std": 0.15372315049171448, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1648437976837158, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07253333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3125, + "kl": 0.0035398683976382017, + "learning_rate": 9.963866666666668e-06, + "loss": 0.0004, + "num_tokens": 4719864.0, + "reward": 6.652245044708252, + "reward_std": 0.4208158552646637, + "rewards/CLIPTextReward/mean": 1.1644315719604492, + "rewards/CLIPTextReward/std": 0.1786278672516346, + "rewards/HPSV21TextReward/mean": 0.8648881912231445, + "rewards/HPSV21TextReward/std": 0.1337926834821701, + "rewards/MANIQAReward/mean": 0.9908708930015564, + "rewards/MANIQAReward/std": 0.14858315885066986, + "rewards/QwenFakeDiscrimReward/mean": 0.19140625, + "rewards/QwenFakeDiscrimReward/std": 0.10673907399177551, + "rewards/QwenTextReward/mean": 1.1730469465255737, + "rewards/QwenTextReward/std": 0.03627080097794533, + "rewards/QwenWeirdDiscrimReward/mean": 0.23828125, + "rewards/QwenWeirdDiscrimReward/std": 0.05326050892472267, + "step": 272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0728, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.004338162951171398, + "learning_rate": 9.963733333333334e-06, + "loss": 0.0004, + "num_tokens": 4737176.0, + "reward": 6.562404632568359, + "reward_std": 0.3299024701118469, + "rewards/CLIPTextReward/mean": 1.0884344577789307, + "rewards/CLIPTextReward/std": 0.18314744904637337, + "rewards/HPSV21TextReward/mean": 0.9037189483642578, + "rewards/HPSV21TextReward/std": 0.13727488741278648, + "rewards/MANIQAReward/mean": 1.048019528388977, + "rewards/MANIQAReward/std": 0.18316024541854858, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1589844226837158, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07306666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0039003107231110334, + "learning_rate": 9.9636e-06, + "loss": 0.0004, + "num_tokens": 4754480.0, + "reward": 6.383108615875244, + "reward_std": 0.3999929428100586, + "rewards/CLIPTextReward/mean": 1.079944133758545, + "rewards/CLIPTextReward/std": 0.1960008516907692, + "rewards/HPSV21TextReward/mean": 0.8467903137207031, + "rewards/HPSV21TextReward/std": 0.06765388697385788, + "rewards/MANIQAReward/mean": 1.0222177505493164, + "rewards/MANIQAReward/std": 0.19649730622768402, + "rewards/QwenFakeDiscrimReward/mean": 0.1328125, + "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, + "rewards/QwenTextReward/mean": 1.1636719703674316, + "rewards/QwenTextReward/std": 0.04007181152701378, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 274 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.28125, + "kl": 0.0034153100568801165, + "learning_rate": 9.963466666666669e-06, + "loss": 0.0003, + "num_tokens": 4771792.0, + "reward": 6.4820380210876465, + "reward_std": 0.4463214874267578, + "rewards/CLIPTextReward/mean": 1.1188676357269287, + "rewards/CLIPTextReward/std": 0.20363353937864304, + "rewards/HPSV21TextReward/mean": 0.876103401184082, + "rewards/HPSV21TextReward/std": 0.11718452721834183, + "rewards/MANIQAReward/mean": 0.973345935344696, + "rewards/MANIQAReward/std": 0.18426600098609924, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1398438215255737, + "rewards/QwenTextReward/std": 0.08748229593038559, + "rewards/QwenWeirdDiscrimReward/mean": 0.203125, + "rewards/QwenWeirdDiscrimReward/std": 0.09834947437047958, + "step": 275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0736, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.003312772372737527, + "learning_rate": 9.963333333333333e-06, + "loss": 0.0003, + "num_tokens": 4789144.0, + "reward": 6.420487403869629, + "reward_std": 0.4493219256401062, + "rewards/CLIPTextReward/mean": 1.107895851135254, + "rewards/CLIPTextReward/std": 0.2051476389169693, + "rewards/HPSV21TextReward/mean": 0.8210859298706055, + "rewards/HPSV21TextReward/std": 0.1308876797556877, + "rewards/MANIQAReward/mean": 1.0277581214904785, + "rewards/MANIQAReward/std": 0.16148918867111206, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.1324219703674316, + "rewards/QwenTextReward/std": 0.0810006633400917, + "rewards/QwenWeirdDiscrimReward/mean": 0.22265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07864411175251007, + "step": 276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07386666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.003616444068029523, + "learning_rate": 9.963200000000001e-06, + "loss": 0.0004, + "num_tokens": 4806400.0, + "reward": 6.341896057128906, + "reward_std": 0.4027109146118164, + "rewards/CLIPTextReward/mean": 1.1019320487976074, + "rewards/CLIPTextReward/std": 0.25059743225574493, + "rewards/HPSV21TextReward/mean": 0.8123483657836914, + "rewards/HPSV21TextReward/std": 0.07598406821489334, + "rewards/MANIQAReward/mean": 1.025053858757019, + "rewards/MANIQAReward/std": 0.16003963351249695, + "rewards/QwenFakeDiscrimReward/mean": 0.14453125, + "rewards/QwenFakeDiscrimReward/std": 0.12444071471691132, + "rewards/QwenTextReward/mean": 1.13671875, + "rewards/QwenTextReward/std": 0.07867565751075745, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07413333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.0036018122918903828, + "learning_rate": 9.963066666666668e-06, + "loss": 0.0004, + "num_tokens": 4823672.0, + "reward": 6.3944621086120605, + "reward_std": 0.43935614824295044, + "rewards/CLIPTextReward/mean": 1.075329065322876, + "rewards/CLIPTextReward/std": 0.19888444989919662, + "rewards/HPSV21TextReward/mean": 0.833500862121582, + "rewards/HPSV21TextReward/std": 0.06530586630105972, + "rewards/MANIQAReward/mean": 1.0189895629882812, + "rewards/MANIQAReward/std": 0.14748038351535797, + "rewards/QwenFakeDiscrimReward/mean": 0.1875, + "rewards/QwenFakeDiscrimReward/std": 0.10910894721746445, + "rewards/QwenTextReward/mean": 1.1554688215255737, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0744, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3984375, + "kl": 0.0039861504919826984, + "learning_rate": 9.962933333333334e-06, + "loss": 0.0004, + "num_tokens": 4840992.0, + "reward": 6.240785598754883, + "reward_std": 0.3987360894680023, + "rewards/CLIPTextReward/mean": 1.0433543920516968, + "rewards/CLIPTextReward/std": 0.1878664493560791, + "rewards/HPSV21TextReward/mean": 0.8436336517333984, + "rewards/HPSV21TextReward/std": 0.06514830887317657, + "rewards/MANIQAReward/mean": 0.9824346303939819, + "rewards/MANIQAReward/std": 0.16924676299095154, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.1484375, + "rewards/QwenTextReward/std": 0.0625000149011612, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 279 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07466666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3359375, + "kl": 0.0036991594824939966, + "learning_rate": 9.9628e-06, + "loss": 0.0004, + "num_tokens": 4858328.0, + "reward": 6.502524375915527, + "reward_std": 0.3771998882293701, + "rewards/CLIPTextReward/mean": 1.1242562532424927, + "rewards/CLIPTextReward/std": 0.1967948004603386, + "rewards/HPSV21TextReward/mean": 0.8170080184936523, + "rewards/HPSV21TextReward/std": 0.1085859090089798, + "rewards/MANIQAReward/mean": 1.062964677810669, + "rewards/MANIQAReward/std": 0.16206982731819153, + "rewards/QwenFakeDiscrimReward/mean": 0.18359375, + "rewards/QwenFakeDiscrimReward/std": 0.11128923296928406, + "rewards/QwenTextReward/mean": 1.142968773841858, + "rewards/QwenTextReward/std": 0.06643842905759811, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 280 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07493333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.003956177271902561, + "learning_rate": 9.962666666666667e-06, + "loss": 0.0004, + "num_tokens": 4875776.0, + "reward": 6.581701278686523, + "reward_std": 0.43749040365219116, + "rewards/CLIPTextReward/mean": 1.104413628578186, + "rewards/CLIPTextReward/std": 0.19979221373796463, + "rewards/HPSV21TextReward/mean": 0.9073686599731445, + "rewards/HPSV21TextReward/std": 0.15387574583292007, + "rewards/MANIQAReward/mean": 0.9991519451141357, + "rewards/MANIQAReward/std": 0.18088753521442413, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.168359398841858, + "rewards/QwenTextReward/std": 0.03965182974934578, + "rewards/QwenWeirdDiscrimReward/mean": 0.23046875, + "rewards/QwenWeirdDiscrimReward/std": 0.06762243062257767, + "step": 281 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0752, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.00417393259704113, + "learning_rate": 9.962533333333333e-06, + "loss": 0.0004, + "num_tokens": 4893080.0, + "reward": 6.471848487854004, + "reward_std": 0.3436872959136963, + "rewards/CLIPTextReward/mean": 1.1173217296600342, + "rewards/CLIPTextReward/std": 0.21878229081630707, + "rewards/HPSV21TextReward/mean": 0.8560466766357422, + "rewards/HPSV21TextReward/std": 0.0951707735657692, + "rewards/MANIQAReward/mean": 1.0106582641601562, + "rewards/MANIQAReward/std": 0.163259819149971, + "rewards/QwenFakeDiscrimReward/mean": 0.1796875, + "rewards/QwenFakeDiscrimReward/std": 0.11329087615013123, + "rewards/QwenTextReward/mean": 1.1433594226837158, + "rewards/QwenTextReward/std": 0.08833134919404984, + "rewards/QwenWeirdDiscrimReward/mean": 0.19140625, + "rewards/QwenWeirdDiscrimReward/std": 0.10673907399177551, + "step": 282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07546666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.003545068670064211, + "learning_rate": 9.962400000000001e-06, + "loss": 0.0004, + "num_tokens": 4910616.0, + "reward": 6.3966569900512695, + "reward_std": 0.3278886675834656, + "rewards/CLIPTextReward/mean": 1.133664846420288, + "rewards/CLIPTextReward/std": 0.20752384513616562, + "rewards/HPSV21TextReward/mean": 0.7828845977783203, + "rewards/HPSV21TextReward/std": 0.07142157107591629, + "rewards/MANIQAReward/mean": 1.0354331731796265, + "rewards/MANIQAReward/std": 0.13326719403266907, + "rewards/QwenFakeDiscrimReward/mean": 0.17578125, + "rewards/QwenFakeDiscrimReward/std": 0.1151231899857521, + "rewards/QwenTextReward/mean": 1.1414062976837158, + "rewards/QwenTextReward/std": 0.08787830919027328, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07573333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.40625, + "kl": 0.003691106103360653, + "learning_rate": 9.962266666666668e-06, + "loss": 0.0004, + "num_tokens": 4927920.0, + "reward": 6.352651596069336, + "reward_std": 0.37639120221138, + "rewards/CLIPTextReward/mean": 1.0819745659828186, + "rewards/CLIPTextReward/std": 0.21546390652656555, + "rewards/HPSV21TextReward/mean": 0.8350515365600586, + "rewards/HPSV21TextReward/std": 0.12028282135725021, + "rewards/MANIQAReward/mean": 1.0287556648254395, + "rewards/MANIQAReward/std": 0.14155684411525726, + "rewards/QwenFakeDiscrimReward/mean": 0.13671875, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.126562476158142, + "rewards/QwenTextReward/std": 0.13663147389888763, + "rewards/QwenWeirdDiscrimReward/mean": 0.2265625, + "rewards/QwenWeirdDiscrimReward/std": 0.07344620674848557, + "step": 284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.076, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0040000625886023045, + "learning_rate": 9.962133333333334e-06, + "loss": 0.0004, + "num_tokens": 4945336.0, + "reward": 6.344567775726318, + "reward_std": 0.42905884981155396, + "rewards/CLIPTextReward/mean": 1.1392810344696045, + "rewards/CLIPTextReward/std": 0.1911049485206604, + "rewards/HPSV21TextReward/mean": 0.8198728561401367, + "rewards/HPSV21TextReward/std": 0.054194964468479156, + "rewards/MANIQAReward/mean": 0.9813380241394043, + "rewards/MANIQAReward/std": 0.1418820321559906, + "rewards/QwenFakeDiscrimReward/mean": 0.12890625, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.144140601158142, + "rewards/QwenTextReward/std": 0.04974982514977455, + "rewards/QwenWeirdDiscrimReward/mean": 0.171875, + "rewards/QwenWeirdDiscrimReward/std": 0.116794154047966, + "step": 285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07626666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.0036982204765081406, + "learning_rate": 9.962e-06, + "loss": 0.0004, + "num_tokens": 4962568.0, + "reward": 6.429768085479736, + "reward_std": 0.4796310067176819, + "rewards/CLIPTextReward/mean": 1.109709620475769, + "rewards/CLIPTextReward/std": 0.22573811560869217, + "rewards/HPSV21TextReward/mean": 0.8387088775634766, + "rewards/HPSV21TextReward/std": 0.10024221241474152, + "rewards/MANIQAReward/mean": 1.0161340236663818, + "rewards/MANIQAReward/std": 0.17700877785682678, + "rewards/QwenFakeDiscrimReward/mean": 0.12890625, + "rewards/QwenFakeDiscrimReward/std": 0.12592662870883942, + "rewards/QwenTextReward/mean": 1.1730469465255737, + "rewards/QwenTextReward/std": 0.03627080097794533, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07653333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.40625, + "kl": 0.004039062652736902, + "learning_rate": 9.961866666666667e-06, + "loss": 0.0004, + "num_tokens": 4979920.0, + "reward": 6.669560432434082, + "reward_std": 0.3349897563457489, + "rewards/CLIPTextReward/mean": 1.1544265747070312, + "rewards/CLIPTextReward/std": 0.20579788833856583, + "rewards/HPSV21TextReward/mean": 0.8574304580688477, + "rewards/HPSV21TextReward/std": 0.1492508500814438, + "rewards/MANIQAReward/mean": 1.031002163887024, + "rewards/MANIQAReward/std": 0.1384875327348709, + "rewards/QwenFakeDiscrimReward/mean": 0.21484375, + "rewards/QwenFakeDiscrimReward/std": 0.0875956118106842, + "rewards/QwenTextReward/mean": 1.157812476158142, + "rewards/QwenTextReward/std": 0.03750002384185791, + "rewards/QwenWeirdDiscrimReward/mean": 0.2421875, + "rewards/QwenWeirdDiscrimReward/std": 0.043842025101184845, + "step": 287 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0768, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.004077456891536713, + "learning_rate": 9.961733333333335e-06, + "loss": 0.0004, + "num_tokens": 4997352.0, + "reward": 6.327436923980713, + "reward_std": 0.4377618432044983, + "rewards/CLIPTextReward/mean": 1.0494632124900818, + "rewards/CLIPTextReward/std": 0.17974548041820526, + "rewards/HPSV21TextReward/mean": 0.8615455627441406, + "rewards/HPSV21TextReward/std": 0.12188783288002014, + "rewards/MANIQAReward/mean": 1.0417473316192627, + "rewards/MANIQAReward/std": 0.17893892526626587, + "rewards/QwenFakeDiscrimReward/mean": 0.1328125, + "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, + "rewards/QwenTextReward/mean": 1.1472656726837158, + "rewards/QwenTextReward/std": 0.06222037598490715, + "rewards/QwenWeirdDiscrimReward/mean": 0.18359375, + "rewards/QwenWeirdDiscrimReward/std": 0.11128923296928406, + "step": 288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07706666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.34375, + "kl": 0.00398043030872941, + "learning_rate": 9.961600000000001e-06, + "loss": 0.0004, + "num_tokens": 5014744.0, + "reward": 6.2923078536987305, + "reward_std": 0.41273707151412964, + "rewards/CLIPTextReward/mean": 1.0740737915039062, + "rewards/CLIPTextReward/std": 0.20466717332601547, + "rewards/HPSV21TextReward/mean": 0.8345117568969727, + "rewards/HPSV21TextReward/std": 0.06559011340141296, + "rewards/MANIQAReward/mean": 0.963417649269104, + "rewards/MANIQAReward/std": 0.16190515458583832, + "rewards/QwenFakeDiscrimReward/mean": 0.1328125, + "rewards/QwenFakeDiscrimReward/std": 0.1257418543100357, + "rewards/QwenTextReward/mean": 1.16015625, + "rewards/QwenTextReward/std": 0.03772258013486862, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 289 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07733333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3203125, + "kl": 0.004222570452839136, + "learning_rate": 9.961466666666668e-06, + "loss": 0.0004, + "num_tokens": 5032048.0, + "reward": 6.57753849029541, + "reward_std": 0.37709754705429077, + "rewards/CLIPTextReward/mean": 1.1433212757110596, + "rewards/CLIPTextReward/std": 0.18561596423387527, + "rewards/HPSV21TextReward/mean": 0.8779010772705078, + "rewards/HPSV21TextReward/std": 0.13092831522226334, + "rewards/MANIQAReward/mean": 1.0280625820159912, + "rewards/MANIQAReward/std": 0.1869654357433319, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.1554687023162842, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.1953125, + "rewards/QwenWeirdDiscrimReward/std": 0.1041666716337204, + "step": 290 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0776, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.40625, + "kl": 0.004480007104575634, + "learning_rate": 9.961333333333334e-06, + "loss": 0.0004, + "num_tokens": 5049368.0, + "reward": 6.583841800689697, + "reward_std": 0.3834884762763977, + "rewards/CLIPTextReward/mean": 1.1373045444488525, + "rewards/CLIPTextReward/std": 0.20375385880470276, + "rewards/HPSV21TextReward/mean": 0.8537569046020508, + "rewards/HPSV21TextReward/std": 0.08211877197027206, + "rewards/MANIQAReward/mean": 1.082578420639038, + "rewards/MANIQAReward/std": 0.13105171918869019, + "rewards/QwenFakeDiscrimReward/mean": 0.1484375, + "rewards/QwenFakeDiscrimReward/std": 0.12375371158123016, + "rewards/QwenTextReward/mean": 1.1519532203674316, + "rewards/QwenTextReward/std": 0.03627080097794533, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 291 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07786666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.390625, + "kl": 0.005785368382930756, + "learning_rate": 9.9612e-06, + "loss": 0.0006, + "num_tokens": 5066744.0, + "reward": 6.347508430480957, + "reward_std": 0.36312463879585266, + "rewards/CLIPTextReward/mean": 1.019821047782898, + "rewards/CLIPTextReward/std": 0.1682324931025505, + "rewards/HPSV21TextReward/mean": 0.8669137954711914, + "rewards/HPSV21TextReward/std": 0.1736130267381668, + "rewards/MANIQAReward/mean": 1.0318512916564941, + "rewards/MANIQAReward/std": 0.15442673861980438, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.1593749523162842, + "rewards/QwenTextReward/std": 0.06922188401222229, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07813333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.328125, + "kl": 0.004109468311071396, + "learning_rate": 9.961066666666669e-06, + "loss": 0.0004, + "num_tokens": 5084104.0, + "reward": 6.442000389099121, + "reward_std": 0.48246192932128906, + "rewards/CLIPTextReward/mean": 1.1207281947135925, + "rewards/CLIPTextReward/std": 0.19325513392686844, + "rewards/HPSV21TextReward/mean": 0.8586606979370117, + "rewards/HPSV21TextReward/std": 0.1198316216468811, + "rewards/MANIQAReward/mean": 0.9582223296165466, + "rewards/MANIQAReward/std": 0.16580310463905334, + "rewards/QwenFakeDiscrimReward/mean": 0.16015625, + "rewards/QwenFakeDiscrimReward/std": 0.12090259045362473, + "rewards/QwenTextReward/mean": 1.1578125953674316, + "rewards/QwenTextReward/std": 0.03750002756714821, + "rewards/QwenWeirdDiscrimReward/mean": 0.20703125, + "rewards/QwenWeirdDiscrimReward/std": 0.09506355226039886, + "step": 293 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0784, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3671875, + "kl": 0.004067155998200178, + "learning_rate": 9.960933333333335e-06, + "loss": 0.0004, + "num_tokens": 5101344.0, + "reward": 6.1931233406066895, + "reward_std": 0.31273579597473145, + "rewards/CLIPTextReward/mean": 0.9793084859848022, + "rewards/CLIPTextReward/std": 0.17051473259925842, + "rewards/HPSV21TextReward/mean": 0.8084526062011719, + "rewards/HPSV21TextReward/std": 0.06458032876253128, + "rewards/MANIQAReward/mean": 1.121117115020752, + "rewards/MANIQAReward/std": 0.16443513333797455, + "rewards/QwenFakeDiscrimReward/mean": 0.15234375, + "rewards/QwenFakeDiscrimReward/std": 0.12293684482574463, + "rewards/QwenTextReward/mean": 1.1449217796325684, + "rewards/QwenTextReward/std": 0.03338678926229477, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07866666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.359375, + "kl": 0.003640000708401203, + "learning_rate": 9.960800000000001e-06, + "loss": 0.0004, + "num_tokens": 5118600.0, + "reward": 6.4132184982299805, + "reward_std": 0.4460734724998474, + "rewards/CLIPTextReward/mean": 1.1124546527862549, + "rewards/CLIPTextReward/std": 0.21535417437553406, + "rewards/HPSV21TextReward/mean": 0.8255329132080078, + "rewards/HPSV21TextReward/std": 0.06593691557645798, + "rewards/MANIQAReward/mean": 1.0157592296600342, + "rewards/MANIQAReward/std": 0.1600320190191269, + "rewards/QwenFakeDiscrimReward/mean": 0.171875, + "rewards/QwenFakeDiscrimReward/std": 0.116794154047966, + "rewards/QwenTextReward/mean": 1.134765625, + "rewards/QwenTextReward/std": 0.08000710606575012, + "rewards/QwenWeirdDiscrimReward/mean": 0.21484375, + "rewards/QwenWeirdDiscrimReward/std": 0.0875956118106842, + "step": 295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07893333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3828125, + "kl": 0.003906239289790392, + "learning_rate": 9.960666666666668e-06, + "loss": 0.0004, + "num_tokens": 5135992.0, + "reward": 6.093643665313721, + "reward_std": 0.3523331880569458, + "rewards/CLIPTextReward/mean": 1.0198302865028381, + "rewards/CLIPTextReward/std": 0.18580127507448196, + "rewards/HPSV21TextReward/mean": 0.7943859100341797, + "rewards/HPSV21TextReward/std": 0.061968978494405746, + "rewards/MANIQAReward/mean": 0.9851330518722534, + "rewards/MANIQAReward/std": 0.17740575969219208, + "rewards/QwenFakeDiscrimReward/mean": 0.13671875, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.1441404819488525, + "rewards/QwenTextReward/std": 0.06677703559398651, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0792, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4140625, + "kl": 0.003932563588023186, + "learning_rate": 9.960533333333334e-06, + "loss": 0.0004, + "num_tokens": 5153304.0, + "reward": 6.546805381774902, + "reward_std": 0.3591303825378418, + "rewards/CLIPTextReward/mean": 1.0995697379112244, + "rewards/CLIPTextReward/std": 0.20298991352319717, + "rewards/HPSV21TextReward/mean": 0.8044948577880859, + "rewards/HPSV21TextReward/std": 0.08719000220298767, + "rewards/MANIQAReward/mean": 1.1082077026367188, + "rewards/MANIQAReward/std": 0.14398537576198578, + "rewards/QwenFakeDiscrimReward/mean": 0.2109375, + "rewards/QwenFakeDiscrimReward/std": 0.09149063378572464, + "rewards/QwenTextReward/mean": 1.1695313453674316, + "rewards/QwenTextReward/std": 0.03712613880634308, + "rewards/QwenWeirdDiscrimReward/mean": 0.25, + "rewards/QwenWeirdDiscrimReward/std": 0.0, + "step": 297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07946666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.390625, + "kl": 0.004132324364036322, + "learning_rate": 9.9604e-06, + "loss": 0.0004, + "num_tokens": 5170608.0, + "reward": 6.462998390197754, + "reward_std": 0.4576346278190613, + "rewards/CLIPTextReward/mean": 1.1270618438720703, + "rewards/CLIPTextReward/std": 0.19543292373418808, + "rewards/HPSV21TextReward/mean": 0.8457469940185547, + "rewards/HPSV21TextReward/std": 0.12634971737861633, + "rewards/MANIQAReward/mean": 1.071286916732788, + "rewards/MANIQAReward/std": 0.14734399318695068, + "rewards/QwenFakeDiscrimReward/mean": 0.11328125, + "rewards/QwenFakeDiscrimReward/std": 0.1254332810640335, + "rewards/QwenTextReward/mean": 1.1335937976837158, + "rewards/QwenTextReward/std": 0.06287093460559845, + "rewards/QwenWeirdDiscrimReward/mean": 0.19921875, + "rewards/QwenWeirdDiscrimReward/std": 0.10137632489204407, + "step": 298 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07973333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3046875, + "kl": 0.003896570298820734, + "learning_rate": 9.960266666666667e-06, + "loss": 0.0004, + "num_tokens": 5187856.0, + "reward": 6.554800033569336, + "reward_std": 0.4156927168369293, + "rewards/CLIPTextReward/mean": 1.114656925201416, + "rewards/CLIPTextReward/std": 0.19729189574718475, + "rewards/HPSV21TextReward/mean": 0.860011100769043, + "rewards/HPSV21TextReward/std": 0.13110067322850227, + "rewards/MANIQAReward/mean": 1.0789012908935547, + "rewards/MANIQAReward/std": 0.16471092402935028, + "rewards/QwenFakeDiscrimReward/mean": 0.15625, + "rewards/QwenFakeDiscrimReward/std": 0.12198751419782639, + "rewards/QwenTextReward/mean": 1.1515625715255737, + "rewards/QwenTextReward/std": 0.04473801702260971, + "rewards/QwenWeirdDiscrimReward/mean": 0.21875, + "rewards/QwenWeirdDiscrimReward/std": 0.0833333358168602, + "step": 299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 256.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 256.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 256.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.08, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3515625, + "kl": 0.003982767462730408, + "learning_rate": 9.960133333333333e-06, + "loss": 0.0004, + "num_tokens": 5205168.0, + "reward": 6.407749176025391, + "reward_std": 0.45647531747817993, + "rewards/CLIPTextReward/mean": 1.0897849798202515, + "rewards/CLIPTextReward/std": 0.21174846589565277, + "rewards/HPSV21TextReward/mean": 0.8501195907592773, + "rewards/HPSV21TextReward/std": 0.11032982170581818, + "rewards/MANIQAReward/mean": 1.010362148284912, + "rewards/MANIQAReward/std": 0.18477247655391693, + "rewards/QwenFakeDiscrimReward/mean": 0.140625, + "rewards/QwenFakeDiscrimReward/std": 0.125, + "rewards/QwenTextReward/mean": 1.166015625, + "rewards/QwenTextReward/std": 0.03763000667095184, + "rewards/QwenWeirdDiscrimReward/mean": 0.2109375, + "rewards/QwenWeirdDiscrimReward/std": 0.09149063378572464, + "step": 300 + } + ], + "logging_steps": 1.0, + "max_steps": 75000, + "num_input_tokens_seen": 5205168, + "num_train_epochs": 20, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +}